From 0c834365d554871d66911f7fd8ea8a41e4096b7c Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Thu, 15 Oct 2026 15:00:37 +0100 Subject: [PATCH 01/48] refactor: host declaration scope shapes natively with native ordering and computed attributes --- .../src/stage_capture/constants.rs | 178 +++- .../sqlbuild-cache/src/stage_capture/tests.rs | 2 + .../src/stage_capture/tests/helpers.rs | 10 + .../tests/test_hosted_shape_types.rs | 33 + .../src/stage_capture/tests/test_types.rs | 6 + .../compile_attachment/model_inputs.rs | 4 +- .../_helpers/compile_records/constants.rs | 16 +- .../_helpers/compiled_project/attachment.rs | 4 +- .../_helpers/compiled_project/loop_classes.rs | 4 +- .../_helpers/compiler_enums/constants.rs | 337 +++++++ .../bindings/_helpers/compiler_enums/mod.rs | 1 + .../_helpers/compiler_errors/constants.rs | 344 +++++++ .../bindings/_helpers/compiler_errors/mod.rs | 1 + .../_helpers/compiler_records/constants.rs | 865 +----------------- .../compiler_records/shape_classes.rs | 52 +- .../_helpers/discovery/declaration_records.rs | 2 +- .../src/bindings/_helpers/mod.rs | 6 + .../_helpers/planner_records/constants.rs | 151 +++ .../bindings/_helpers/planner_records/mod.rs | 1 + .../_helpers/record_behaviours/constants.rs | 19 + .../_helpers/record_behaviours/mod.rs | 2 + .../_helpers/record_behaviours/scopes.rs | 30 + .../bindings/_helpers/records/constants.rs | 31 +- .../src/bindings/_helpers/records/fields.rs | 29 +- .../src/bindings/_helpers/records/models.rs | 94 +- .../bindings/_helpers/records/registration.rs | 26 +- .../src/bindings/_helpers/records/types.rs | 10 +- .../_helpers/refactor/refactor_facts.rs | 4 +- .../bindings/_helpers/render/authored_sql.rs | 2 +- .../_helpers/render/declaration_values.rs | 9 +- .../_helpers/render/macro_expansion_result.rs | 4 +- .../_helpers/rule_records/constants.rs | 177 ++++ .../src/bindings/_helpers/rule_records/mod.rs | 1 + .../_helpers/rules/custom_evaluation.rs | 2 +- .../src/bindings/_helpers/rules/lint_text.rs | 2 +- .../_helpers/scope_records/constants.rs | 450 +++++++++ .../bindings/_helpers/scope_records/mod.rs | 1 + .../_helpers/scopes/compile_usages.rs | 2 +- .../_helpers/scopes/declaration_contexts.rs | 11 +- .../bindings/_helpers/scopes/scope_command.rs | 4 +- .../bindings/_helpers/scopes/scope_objects.rs | 6 +- .../_helpers/scopes/scope_report_objects.rs | 2 +- .../_helpers/scopes/scope_validation.rs | 6 +- fensu.toml | 5 - python-boundary.toml | 2 +- .../_helpers/corpus/emitted_codes.py | 3 +- src/sqlbuild/_native.pyi | 738 ++++++++++++++- .../compile/_helpers/render/user_macros.py | 12 +- src/sqlbuild/compiler/compile/models.py | 14 +- src/sqlbuild/compiler/compile/types.py | 3 +- src/sqlbuild/compiler/discovery/models.py | 2 +- src/sqlbuild/compiler/scopes/__init__.py | 1 - src/sqlbuild/compiler/scopes/models.py | 435 --------- src/sqlbuild/compiler/scopes/types.py | 129 --- .../src/sqlbuild/compiler/compile/helpers.py | 4 +- .../pipeline/test_macro_load_count.py | 2 +- .../src/sqlbuild/compiler/scopes/helpers.py | 26 +- .../scopes/test_native_scope_parity.py | 2 +- .../commands/_helpers/entry/test_parsing.py | 2 +- .../cli/commands/_helpers/scope/helpers.py | 8 +- .../cli/commands/main/scope/test_scope.py | 3 +- .../compiler/compile/_helpers/_test_types.py | 3 +- .../compiler/compile/_helpers/helpers.py | 22 +- .../_helpers/test_declaration_scopes.py | 16 +- .../compiler/compile/_helpers/test_macros.py | 3 +- .../_helpers/test_rebind_declaration_scope.py | 4 +- .../test_scoped_named_declarations.py | 2 +- .../sqlbuild/compiler/manifest/test_main.py | 16 +- .../sqlbuild/compiler/scopes/_test_helpers.py | 26 +- .../sqlbuild/compiler/scopes/_test_types.py | 7 +- .../src/sqlbuild/compiler/scopes/helpers.py | 24 +- .../scopes/test_builder_and_visibility.py | 34 +- .../compiler/scopes/test_identities.py | 12 +- .../scopes/test_lookup_and_metadata.py | 15 +- .../compiler/scopes/test_offline_cache.py | 13 +- .../compiler/scopes/test_placement.py | 20 +- .../compiler/scopes/test_relocation.py | 6 +- .../sqlbuild/compiler/scopes/test_reports.py | 24 +- .../rule_engine/_helpers/engine/helpers.py | 2 +- .../_helpers/engine/test_scope_facts.py | 8 +- 80 files changed, 2867 insertions(+), 1722 deletions(-) create mode 100644 crates/sqlbuild-cache/src/stage_capture/tests/test_hosted_shape_types.rs create mode 100644 crates/sqlbuild-python/src/bindings/_helpers/compiler_enums/constants.rs create mode 100644 crates/sqlbuild-python/src/bindings/_helpers/compiler_enums/mod.rs create mode 100644 crates/sqlbuild-python/src/bindings/_helpers/compiler_errors/constants.rs create mode 100644 crates/sqlbuild-python/src/bindings/_helpers/compiler_errors/mod.rs create mode 100644 crates/sqlbuild-python/src/bindings/_helpers/planner_records/constants.rs create mode 100644 crates/sqlbuild-python/src/bindings/_helpers/planner_records/mod.rs create mode 100644 crates/sqlbuild-python/src/bindings/_helpers/record_behaviours/constants.rs create mode 100644 crates/sqlbuild-python/src/bindings/_helpers/record_behaviours/mod.rs create mode 100644 crates/sqlbuild-python/src/bindings/_helpers/record_behaviours/scopes.rs create mode 100644 crates/sqlbuild-python/src/bindings/_helpers/rule_records/constants.rs create mode 100644 crates/sqlbuild-python/src/bindings/_helpers/rule_records/mod.rs create mode 100644 crates/sqlbuild-python/src/bindings/_helpers/scope_records/constants.rs create mode 100644 crates/sqlbuild-python/src/bindings/_helpers/scope_records/mod.rs delete mode 100644 src/sqlbuild/compiler/scopes/__init__.py delete mode 100644 src/sqlbuild/compiler/scopes/models.py delete mode 100644 src/sqlbuild/compiler/scopes/types.py diff --git a/crates/sqlbuild-cache/src/stage_capture/constants.rs b/crates/sqlbuild-cache/src/stage_capture/constants.rs index f9d5b74f87..85a6625b34 100644 --- a/crates/sqlbuild-cache/src/stage_capture/constants.rs +++ b/crates/sqlbuild-cache/src/stage_capture/constants.rs @@ -383,7 +383,7 @@ pub const CANONICAL_TYPES: [(&str, &str); 11] = [ /// Sequence types captured as their items. pub const DECODED_SEQUENCES: [&str; 1] = ["sqlbuild.compiler.compile.models:CompactLineageFacts"]; /// The Python model type a natively hosted compiler shape is captured as. -pub const HOSTED_SHAPE_TYPES: [(&str, &str); 213] = [ +pub const HOSTED_SHAPE_TYPES: [(&str, &str); 257] = [ ( "sqlbuild._native:AnalysisCacheContext", "sqlbuild.compiler.compile.models:AnalysisCacheContext", @@ -524,6 +524,10 @@ pub const HOSTED_SHAPE_TYPES: [(&str, &str); 213] = [ "sqlbuild._native:ConfigValueTypeError", "sqlbuild.spec.contracts.exceptions:ConfigValueTypeError", ), + ( + "sqlbuild._native:ConstantMetadata", + "sqlbuild.compiler.scopes.models:ConstantMetadata", + ), ( "sqlbuild._native:ContractAdoptionError", "sqlbuild.compiler.contract_adoption.exceptions:ContractAdoptionError", @@ -604,10 +608,30 @@ pub const HOSTED_SHAPE_TYPES: [(&str, &str); 213] = [ "sqlbuild._native:DeclarationExpansionContext", "sqlbuild.compiler.compile.models:DeclarationExpansionContext", ), + ( + "sqlbuild._native:DeclarationExplanation", + "sqlbuild.compiler.scopes.models:DeclarationExplanation", + ), + ( + "sqlbuild._native:DeclarationIdentity", + "sqlbuild.compiler.scopes.models:DeclarationIdentity", + ), + ( + "sqlbuild._native:DeclarationKind", + "sqlbuild.compiler.scopes.types:DeclarationKind", + ), ( "sqlbuild._native:DeclarationParseError", "sqlbuild.compiler.discovery.exceptions:DeclarationParseError", ), + ( + "sqlbuild._native:DeclarationRecord", + "sqlbuild.compiler.scopes.models:DeclarationRecord", + ), + ( + "sqlbuild._native:DeclarationReport", + "sqlbuild.compiler.scopes.models:DeclarationReport", + ), ( "sqlbuild._native:DeclarationRuntimeProjection", "sqlbuild.compiler.compile.models:DeclarationRuntimeProjection", @@ -620,6 +644,14 @@ pub const HOSTED_SHAPE_TYPES: [(&str, &str); 213] = [ "sqlbuild._native:DeclarationScopeResolver", "sqlbuild.compiler.compile.models:DeclarationScopeResolver", ), + ( + "sqlbuild._native:DeclarationVisibility", + "sqlbuild.compiler.scopes.models:DeclarationVisibility", + ), + ( + "sqlbuild._native:DeclarationVisibilityIndex", + "sqlbuild.compiler.scopes.models:DeclarationVisibilityIndex", + ), ( "sqlbuild._native:DeferralInputs", "sqlbuild.compiler.planner.models:DeferralInputs", @@ -736,6 +768,14 @@ pub const HOSTED_SHAPE_TYPES: [(&str, &str); 213] = [ "sqlbuild._native:EnumMember", "sqlbuild.compiler.discovery.models:EnumMember", ), + ( + "sqlbuild._native:EnumMemberMetadata", + "sqlbuild.compiler.scopes.models:EnumMemberMetadata", + ), + ( + "sqlbuild._native:EnumMetadata", + "sqlbuild.compiler.scopes.models:EnumMetadata", + ), ( "sqlbuild._native:EventExporterDiscoveryError", "sqlbuild.compiler.discovery.exceptions:EventExporterDiscoveryError", @@ -768,6 +808,14 @@ pub const HOSTED_SHAPE_TYPES: [(&str, &str); 213] = [ "sqlbuild._native:FunctionReturnColumn", "sqlbuild.compiler.compile.models:FunctionReturnColumn", ), + ( + "sqlbuild._native:GrantKind", + "sqlbuild.compiler.scopes.types:GrantKind", + ), + ( + "sqlbuild._native:GrantRecord", + "sqlbuild.compiler.scopes.models:GrantRecord", + ), ( "sqlbuild._native:GraphEdgeExclusion", "sqlbuild.rule_engine.models:GraphEdgeExclusion", @@ -788,6 +836,14 @@ pub const HOSTED_SHAPE_TYPES: [(&str, &str); 213] = [ "sqlbuild._native:HookReadType", "sqlbuild.compiler.graph.types:HookReadType", ), + ( + "sqlbuild._native:InaccessibleReason", + "sqlbuild.compiler.scopes.types:InaccessibleReason", + ), + ( + "sqlbuild._native:InaccessibleRecord", + "sqlbuild.compiler.scopes.models:InaccessibleRecord", + ), ( "sqlbuild._native:InterpolationSite", "sqlbuild.lint.models:InterpolationSite", @@ -846,6 +902,10 @@ pub const HOSTED_SHAPE_TYPES: [(&str, &str); 213] = [ "sqlbuild._native:MacroExpansionResult", "sqlbuild.compiler.compile.models:MacroExpansionResult", ), + ( + "sqlbuild._native:MacroMetadata", + "sqlbuild.compiler.scopes.models:MacroMetadata", + ), ( "sqlbuild._native:ManualLocation", "sqlbuild.compiler.refactoring.models:ManualLocation", @@ -886,6 +946,10 @@ pub const HOSTED_SHAPE_TYPES: [(&str, &str); 213] = [ "sqlbuild._native:ModelValidationRequest", "sqlbuild.compiler.compile.models:ModelValidationRequest", ), + ( + "sqlbuild._native:MovePreview", + "sqlbuild.compiler.scopes.models:MovePreview", + ), ( "sqlbuild._native:NamedSqlHookEntry", "sqlbuild.compiler.discovery.models:NamedSqlHookEntry", @@ -954,6 +1018,14 @@ pub const HOSTED_SHAPE_TYPES: [(&str, &str); 213] = [ "sqlbuild._native:OldNameViewAction", "sqlbuild.compiler.migrations.types:OldNameViewAction", ), + ( + "sqlbuild._native:OwnershipRoot", + "sqlbuild.compiler.scopes.models:OwnershipRoot", + ), + ( + "sqlbuild._native:OwnershipRootKind", + "sqlbuild.compiler.scopes.types:OwnershipRootKind", + ), ( "sqlbuild._native:ParsedSelector", "sqlbuild.compiler.planner.models:ParsedSelector", @@ -1082,10 +1154,22 @@ pub const HOSTED_SHAPE_TYPES: [(&str, &str); 213] = [ "sqlbuild._native:RelatedLocation", "sqlbuild.compiler.compile.models:RelatedLocation", ), + ( + "sqlbuild._native:ResourceIdentity", + "sqlbuild.compiler.scopes.models:ResourceIdentity", + ), ( "sqlbuild._native:ResourceIdentityError", "sqlbuild.compiler.resource_names.exceptions:ResourceIdentityError", ), + ( + "sqlbuild._native:ResourceKind", + "sqlbuild.compiler.scopes.types:ResourceKind", + ), + ( + "sqlbuild._native:ResourceRecord", + "sqlbuild.compiler.scopes.models:ResourceRecord", + ), ( "sqlbuild._native:RuleExemption", "sqlbuild.rule_engine.models:RuleExemption", @@ -1118,10 +1202,78 @@ pub const HOSTED_SHAPE_TYPES: [(&str, &str); 213] = [ "sqlbuild._native:SchemaParseError", "sqlbuild.compiler.discovery.exceptions:SchemaParseError", ), + ( + "sqlbuild._native:ScopeBrowseResult", + "sqlbuild.compiler.scopes.models:ScopeBrowseResult", + ), + ( + "sqlbuild._native:ScopeChainEntry", + "sqlbuild.compiler.scopes.models:ScopeChainEntry", + ), + ( + "sqlbuild._native:ScopeCompleteness", + "sqlbuild.compiler.scopes.models:ScopeCompleteness", + ), + ( + "sqlbuild._native:ScopeDiagnostic", + "sqlbuild.compiler.scopes.models:ScopeDiagnostic", + ), + ( + "sqlbuild._native:ScopeDiagnosticCode", + "sqlbuild.compiler.scopes.types:ScopeDiagnosticCode", + ), + ( + "sqlbuild._native:ScopeDiagnosticSeverity", + "sqlbuild.compiler.scopes.types:DiagnosticSeverity", + ), ( "sqlbuild._native:ScopeError", "sqlbuild.compiler.scopes.exceptions:ScopeError", ), + ( + "sqlbuild._native:ScopeFolder", + "sqlbuild.compiler.scopes.models:ScopeFolder", + ), + ( + "sqlbuild._native:ScopeIndex", + "sqlbuild.compiler.scopes.models:ScopeIndex", + ), + ( + "sqlbuild._native:ScopeKind", + "sqlbuild.compiler.scopes.types:ScopeKind", + ), + ( + "sqlbuild._native:ScopeListResult", + "sqlbuild.compiler.scopes.models:ScopeListResult", + ), + ( + "sqlbuild._native:ScopeLookup", + "sqlbuild.compiler.scopes.models:ScopeLookup", + ), + ( + "sqlbuild._native:ScopeReport", + "sqlbuild.compiler.scopes.models:ScopeReport", + ), + ( + "sqlbuild._native:ScopeReportFilters", + "sqlbuild.compiler.scopes.models:ScopeReportFilters", + ), + ( + "sqlbuild._native:ScopeResourceReport", + "sqlbuild.compiler.scopes.models:ScopeResourceReport", + ), + ( + "sqlbuild._native:ScopeSection", + "sqlbuild.compiler.scopes.models:ScopeSection", + ), + ( + "sqlbuild._native:ScopeSourceLocation", + "sqlbuild.compiler.scopes.models:SourceLocation", + ), + ( + "sqlbuild._native:ScopeTargetQuery", + "sqlbuild.compiler.scopes.models:ScopeTargetQuery", + ), ( "sqlbuild._native:ScopeValidationError", "sqlbuild.compiler.scopes.exceptions:ScopeValidationError", @@ -1226,6 +1378,30 @@ pub const HOSTED_SHAPE_TYPES: [(&str, &str); 213] = [ "sqlbuild._native:UnsupportedLexicalSyntaxError", "sqlbuild.compiler.sql_analysis.exceptions:UnsupportedLexicalSyntaxError", ), + ( + "sqlbuild._native:UsageKind", + "sqlbuild.compiler.scopes.types:UsageKind", + ), + ( + "sqlbuild._native:UsageRecord", + "sqlbuild.compiler.scopes.models:UsageRecord", + ), + ( + "sqlbuild._native:VisibilityProvenance", + "sqlbuild.compiler.scopes.models:VisibilityProvenance", + ), + ( + "sqlbuild._native:VisibilityReason", + "sqlbuild.compiler.scopes.types:VisibilityReason", + ), + ( + "sqlbuild._native:VisibilityRecord", + "sqlbuild.compiler.scopes.models:VisibilityRecord", + ), + ( + "sqlbuild._native:VisibilityResolution", + "sqlbuild.compiler.scopes.models:VisibilityResolution", + ), ( "sqlbuild._native:VisibleModelDeclarations", "sqlbuild.compiler.compile.models:VisibleModelDeclarations", diff --git a/crates/sqlbuild-cache/src/stage_capture/tests.rs b/crates/sqlbuild-cache/src/stage_capture/tests.rs index 80c6c333f9..86958e472c 100644 --- a/crates/sqlbuild-cache/src/stage_capture/tests.rs +++ b/crates/sqlbuild-cache/src/stage_capture/tests.rs @@ -2,6 +2,8 @@ mod capture_document; #[path = "tests/helpers.rs"] mod helpers; +#[path = "tests/test_hosted_shape_types.rs"] +mod hosted_shape_types; #[path = "tests/test_mask_capture_noise.rs"] mod mask_capture_noise; #[path = "tests/test_types.rs"] diff --git a/crates/sqlbuild-cache/src/stage_capture/tests/helpers.rs b/crates/sqlbuild-cache/src/stage_capture/tests/helpers.rs index 59924b9967..cbc968651b 100644 --- a/crates/sqlbuild-cache/src/stage_capture/tests/helpers.rs +++ b/crates/sqlbuild-cache/src/stage_capture/tests/helpers.rs @@ -1,4 +1,5 @@ use sqlbuild_core::json::models::{JsonInteger, JsonValue}; +use std::collections::HashSet; pub(crate) fn text(value: &str) -> JsonValue { JsonValue::String(value.to_owned()) @@ -7,3 +8,12 @@ pub(crate) fn text(value: &str) -> JsonValue { pub(crate) fn integer(value: i64) -> JsonValue { JsonValue::Integer(JsonInteger::from(value)) } + +/// The keys of `keys` seen more than once, in first-repeat order. +pub(super) fn duplicates(keys: &[&'static str]) -> Vec<&'static str> { + let mut seen: HashSet<&str> = HashSet::new(); + keys.iter() + .copied() + .filter(|key| !seen.insert(key)) + .collect() +} diff --git a/crates/sqlbuild-cache/src/stage_capture/tests/test_hosted_shape_types.rs b/crates/sqlbuild-cache/src/stage_capture/tests/test_hosted_shape_types.rs new file mode 100644 index 0000000000..e5d828e9c7 --- /dev/null +++ b/crates/sqlbuild-cache/src/stage_capture/tests/test_hosted_shape_types.rs @@ -0,0 +1,33 @@ +use crate::stage_capture::constants::HOSTED_SHAPE_TYPES; +use crate::stage_capture::tests::helpers::duplicates; +use crate::stage_capture::tests::test_types::HostedKeyTestCase; + +#[test] +fn given_hosted_shape_table_when_keyed_then_every_native_and_python_name_is_unique() { + let test_cases = [ + HostedKeyTestCase { + description: "native class names", + keys: HOSTED_SHAPE_TYPES + .iter() + .map(|(native, _)| *native) + .collect(), + expected_duplicates: Vec::new(), + }, + HostedKeyTestCase { + description: "qualified Python shape names", + keys: HOSTED_SHAPE_TYPES + .iter() + .map(|(_, python)| *python) + .collect(), + expected_duplicates: Vec::new(), + }, + ]; + for test_case in test_cases { + assert_eq!( + duplicates(&test_case.keys), + test_case.expected_duplicates, + "{}", + test_case.description + ); + } +} diff --git a/crates/sqlbuild-cache/src/stage_capture/tests/test_types.rs b/crates/sqlbuild-cache/src/stage_capture/tests/test_types.rs index 9962247a4d..25806f4d1b 100644 --- a/crates/sqlbuild-cache/src/stage_capture/tests/test_types.rs +++ b/crates/sqlbuild-cache/src/stage_capture/tests/test_types.rs @@ -18,3 +18,9 @@ pub(crate) struct SortKeyTestCase { pub(crate) item: JsonValue, pub(crate) expected_key: &'static str, } + +pub(crate) struct HostedKeyTestCase { + pub(crate) description: &'static str, + pub(crate) keys: Vec<&'static str>, + pub(crate) expected_duplicates: Vec<&'static str>, +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compile_attachment/model_inputs.rs b/crates/sqlbuild-python/src/bindings/_helpers/compile_attachment/model_inputs.rs index 4afff18d09..12c8cd732b 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compile_attachment/model_inputs.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compile_attachment/model_inputs.rs @@ -23,8 +23,8 @@ const COMPILE_MODELS_MODULE: &str = "sqlbuild.compiler.compile.models"; const COMPILE_ERRORS_MODULE: &str = "sqlbuild.compiler.compile.exceptions"; const DISCOVERY_MODELS_MODULE: &str = "sqlbuild.compiler.discovery.models"; const DISCOVERY_CONSTANTS_MODULE: &str = "sqlbuild.compiler.discovery.constants"; -const SCOPE_MODELS_MODULE: &str = "sqlbuild.compiler.scopes.models"; -const SCOPE_TYPES_MODULE: &str = "sqlbuild.compiler.scopes.types"; +const SCOPE_MODELS_MODULE: &str = "sqlbuild._native"; +const SCOPE_TYPES_MODULE: &str = "sqlbuild._native"; const COLLECTOR_MODULE: &str = "sqlbuild._native"; const PROJECT_VARS_MODULE: &str = "sqlbuild._native"; const MACRO_BRIDGE_MODULE: &str = "sqlbuild.compiler.macro_bridge.main.active_macro_bridge"; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compile_records/constants.rs b/crates/sqlbuild-python/src/bindings/_helpers/compile_records/constants.rs index f18c81c805..b5a8bd0f08 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compile_records/constants.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compile_records/constants.rs @@ -322,7 +322,7 @@ const R_COMPILE_PROJECT_INPUTS: RecordSpec = RecordSpec { FieldSpec::new("external_sql_reference_resolver", FieldDefault::None), FieldSpec::new( "scope_index", - FieldDefault::Call("sqlbuild.compiler.scopes.models", "ScopeIndex"), + FieldDefault::Call("sqlbuild._native", "ScopeIndex"), ), FieldSpec { name: "declaration_scope", @@ -596,7 +596,7 @@ const R_DISCOVERED_SQL_HOOK_FILE: RecordSpec = RecordSpec { FieldSpec::new("description", FieldDefault::None), FieldSpec::new( "scope_kind", - FieldDefault::Member("sqlbuild.compiler.scopes.types", "ScopeKind", "GLOBAL"), + FieldDefault::Member("sqlbuild._native", "ScopeKind", "GLOBAL"), ), FieldSpec::new("ownership_root", FieldDefault::None), FieldSpec::new("owning_path", FieldDefault::None), @@ -618,7 +618,7 @@ const R_DISCOVERED_ENUM_FILE: RecordSpec = RecordSpec { FieldSpec::required("declarations"), FieldSpec::new( "scope_kind", - FieldDefault::Member("sqlbuild.compiler.scopes.types", "ScopeKind", "GLOBAL"), + FieldDefault::Member("sqlbuild._native", "ScopeKind", "GLOBAL"), ), FieldSpec::new("ownership_root", FieldDefault::None), FieldSpec::new("owning_path", FieldDefault::None), @@ -635,7 +635,7 @@ const R_DISCOVERED_CONSTANT_FILE: RecordSpec = RecordSpec { FieldSpec::required("declarations"), FieldSpec::new( "scope_kind", - FieldDefault::Member("sqlbuild.compiler.scopes.types", "ScopeKind", "GLOBAL"), + FieldDefault::Member("sqlbuild._native", "ScopeKind", "GLOBAL"), ), FieldSpec::new("ownership_root", FieldDefault::None), FieldSpec::new("owning_path", FieldDefault::None), @@ -663,7 +663,7 @@ const R_DISCOVERED_MODEL_SCHEMA_FILE: RecordSpec = RecordSpec { FieldSpec::required("declarations"), FieldSpec::new( "scope_kind", - FieldDefault::Member("sqlbuild.compiler.scopes.types", "ScopeKind", "GLOBAL"), + FieldDefault::Member("sqlbuild._native", "ScopeKind", "GLOBAL"), ), FieldSpec::new("ownership_root", FieldDefault::None), FieldSpec::new("owning_path", FieldDefault::None), @@ -739,11 +739,11 @@ const R_DISCOVERED_AUDIT_FILE: RecordSpec = RecordSpec { FieldSpec::required("blocks"), FieldSpec::new( "declaration_kind", - FieldDefault::Member("sqlbuild.compiler.scopes.types", "DeclarationKind", "AUDIT"), + FieldDefault::Member("sqlbuild._native", "DeclarationKind", "AUDIT"), ), FieldSpec::new( "scope_kind", - FieldDefault::Member("sqlbuild.compiler.scopes.types", "ScopeKind", "GLOBAL"), + FieldDefault::Member("sqlbuild._native", "ScopeKind", "GLOBAL"), ), FieldSpec::new("ownership_root", FieldDefault::None), FieldSpec::new("owning_path", FieldDefault::None), @@ -796,7 +796,7 @@ const R_TOLERANT_SCOPE_DISCOVERY: RecordSpec = RecordSpec { ], }; -/// Compile and discovery value records. +/// The compile and discovery value records. pub(crate) const COMPILE_VALUE_RECORDS: &[RecordSpec] = &[ R_RELATED_LOCATION, R_FIXTURE_COLUMN_INFERENCE, diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/attachment.rs b/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/attachment.rs index 5f9f2b902b..19b1191942 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/attachment.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/attachment.rs @@ -31,8 +31,8 @@ const NATIVE_RECORDS: &str = "sqlbuild._native"; const COMPILE_ERRORS_MODULE: &str = "sqlbuild.compiler.compile.exceptions"; const COMPILE_MODELS_MODULE: &str = "sqlbuild.compiler.compile.models"; const COMPILE_TYPES_MODULE: &str = "sqlbuild.compiler.compile.types"; -const SCOPE_TYPES_MODULE: &str = "sqlbuild.compiler.scopes.types"; -const SCOPE_MODELS_MODULE: &str = "sqlbuild.compiler.scopes.models"; +const SCOPE_TYPES_MODULE: &str = "sqlbuild._native"; +const SCOPE_MODELS_MODULE: &str = "sqlbuild._native"; const AUDITING_TYPES_MODULE: &str = "sqlbuild.compiler.auditing.types"; const CONTRACT_MODELS_MODULE: &str = "sqlbuild.spec.contracts.models"; const SQL_VALUE_ERRORS_MODULE: &str = "sqlbuild.sql_values.exceptions"; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/loop_classes.rs b/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/loop_classes.rs index 0c5ed72558..22a7629921 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/loop_classes.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/loop_classes.rs @@ -17,8 +17,8 @@ const LINEAGE_TYPES_MODULE: &str = "sqlbuild.compiler.lineage.types"; const PLANNER_TYPES_MODULE: &str = "sqlbuild.compiler.planner.types"; const REFERENCE_TYPES_MODULE: &str = "sqlbuild.compiler.references.types"; const SCOPE_ERRORS_MODULE: &str = "sqlbuild._native"; -const SCOPE_MODELS_MODULE: &str = "sqlbuild.compiler.scopes.models"; -const SCOPE_TYPES_MODULE: &str = "sqlbuild.compiler.scopes.types"; +const SCOPE_MODELS_MODULE: &str = "sqlbuild._native"; +const SCOPE_TYPES_MODULE: &str = "sqlbuild._native"; const CONTRACT_MODELS_MODULE: &str = "sqlbuild.spec.contracts.models"; const CONTRACT_TYPES_MODULE: &str = "sqlbuild.spec.contracts.types"; const NATIVE_MODULE: &str = "sqlbuild._native"; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compiler_enums/constants.rs b/crates/sqlbuild-python/src/bindings/_helpers/compiler_enums/constants.rs new file mode 100644 index 0000000000..59b859ecb9 --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/compiler_enums/constants.rs @@ -0,0 +1,337 @@ +//! The compiler, rule, lint and config string enums hosted natively. + +use crate::bindings::_helpers::records::models::EnumSpec; + +const E_CONTRACT_FINDING_KIND: EnumSpec = EnumSpec { + name: "ContractFindingKind", + doc: "Stable physical-to-authored contract difference categories.", + members: &[ + ("MISSING_DECLARATION", "missing_declaration"), + ("MISSING_PHYSICAL_COLUMN", "missing_physical_column"), + ("TYPE_MISMATCH", "type_mismatch"), + ("RELATION_MISSING", "relation_missing"), + ("INSPECTION_UNAVAILABLE", "inspection_unavailable"), + ("OWNERSHIP_CONFLICT", "ownership_conflict"), + ], +}; +const E_COMPILER_STAGE: EnumSpec = EnumSpec { + name: "CompilerStage", + doc: "Frontier objects handed from one compiler stage to the next, in pipeline order.", + members: &[ + ("DISCOVERED_PROJECT_INPUTS", "discovered_project_inputs"), + ("COMPILE_PROJECT_INPUTS", "compile_project_inputs"), + ("COMPILED_PROJECT", "compiled_project"), + ], +}; +const E_HOOK_READ_TYPE: EnumSpec = EnumSpec { + name: "HookReadType", + doc: "Kind of model hook that declares or writes a read.", + members: &[ + ("PYTHON", "Python"), + ("SQL", "SQL"), + ("INLINE_SQL", "inline SQL"), + ], +}; +const E_LINT_SEVERITY: EnumSpec = EnumSpec { + name: "LintSeverity", + doc: "Severity assigned to one lint violation.", + members: &[("FAULT", "fault"), ("WARNING", "warning")], +}; +const E_DIAGNOSTIC_PHASE: EnumSpec = EnumSpec { + name: "DiagnosticPhase", + doc: "Phase that produced a compiler diagnostic.", + members: &[ + ("COMPILE", "compile"), + ("CONTRACT", "contract"), + ("RULE", "rule"), + ("PLAN", "plan"), + ("BUILD", "build"), + ("AUDIT", "audit"), + ("TEST", "test"), + ("CONNECTION", "connection"), + ], +}; +const E_DIAGNOSTIC_SEVERITY: EnumSpec = EnumSpec { + name: "DiagnosticSeverity", + doc: "Severity for a compiler diagnostic.", + members: &[("ERROR", "error"), ("WARNING", "warning"), ("INFO", "info")], +}; +const E_COLUMN_TRANSFORM_KIND: EnumSpec = EnumSpec { + name: "ColumnTransformKind", + doc: "High-level transform classification for one output column.", + members: &[ + ("DIRECT", "direct"), + ("CAST", "cast"), + ("EXPRESSION", "expression"), + ("AGGREGATION", "aggregation"), + ("STAR", "star"), + ("CONSTANT", "constant"), + ("UNKNOWN", "unknown"), + ], +}; +const E_COLUMN_LINEAGE_CONFIDENCE: EnumSpec = EnumSpec { + name: "ColumnLineageConfidence", + doc: "Coarse confidence that a lineage edge is fully understood.", + members: &[ + ("HIGH", "high"), + ("MEDIUM", "medium"), + ("LOW", "low"), + ("UNKNOWN", "unknown"), + ], +}; +const E_COLUMN_LINEAGE_MODE: EnumSpec = EnumSpec { + name: "ColumnLineageMode", + doc: "Column lineage analyzer mode.", + members: &[("RICH", "rich"), ("FAST", "fast")], +}; +const E_POLYGLOT_ANALYSIS_DIALECT: EnumSpec = EnumSpec { + name: "PolyglotAnalysisDialect", + doc: "Dialect names accepted by Polyglot analyze_query.", + members: &[ + ("GENERIC", "generic"), + ("POSTGRESQL", "postgresql"), + ("MYSQL", "mysql"), + ("BIGQUERY", "bigquery"), + ("SNOWFLAKE", "snowflake"), + ("DUCKDB", "duckdb"), + ("SQLITE", "sqlite"), + ("HIVE", "hive"), + ("SPARK", "spark"), + ("TRINO", "trino"), + ("PRESTO", "presto"), + ("REDSHIFT", "redshift"), + ("TSQL", "tsql"), + ("ORACLE", "oracle"), + ("CLICKHOUSE", "clickhouse"), + ("DATABRICKS", "databricks"), + ("ATHENA", "athena"), + ("TERADATA", "teradata"), + ("DORIS", "doris"), + ("STARROCKS", "starrocks"), + ("MATERIALIZE", "materialize"), + ("RISINGWAVE", "risingwave"), + ("SINGLESTORE", "singlestore"), + ("COCKROACHDB", "cockroachdb"), + ("TIDB", "tidb"), + ("DRUID", "druid"), + ("SOLR", "solr"), + ("TABLEAU", "tableau"), + ("DUNE", "dune"), + ("FABRIC", "fabric"), + ("DRILL", "drill"), + ("DREMIO", "dremio"), + ("EXASOL", "exasol"), + ("DATAFUSION", "datafusion"), + ], +}; +const E_MIGRATION_DISCOVERY: EnumSpec = EnumSpec { + name: "MigrationDiscovery", + doc: "How a migration was requested.", + members: &[("MANUAL", "manual"), ("AUTOMATIC", "automatic")], +}; +const E_MIGRATION_COMPATIBILITY: EnumSpec = EnumSpec { + name: "MigrationCompatibility", + doc: "Outcome of checking the origin relation against the destination model.", + members: &[ + ("COMPATIBLE", "compatible"), + ("INCOMPATIBLE", "incompatible"), + ("NOT_CHECKED", "not_checked"), + ], +}; +const E_OLD_NAME_VIEW_ACTION: EnumSpec = EnumSpec { + name: "OldNameViewAction", + doc: "What a build does at a migrated model's old name.", + members: &[ + ("ARCHIVE_AND_VIEW", "archive_and_view"), + ("VIEW_ONLY", "view_only"), + ("LIVE", "live"), + ("NONE", "none"), + ], +}; +const E_SELECTOR_KIND: EnumSpec = EnumSpec { + name: "SelectorKind", + doc: "", + members: &[ + ("NAME", "name"), + ("SEED", "seed"), + ("SOURCE", "source"), + ("TASK", "task"), + ("ASSET", "asset"), + ("LOADER", "loader"), + ("CHECK", "check"), + ("TAG", "tag"), + ("PATH", "path"), + ("TEST", "test"), + ], +}; +const E_PYTHON_RUN_PHASE: EnumSpec = EnumSpec { + name: "PythonRunPhase", + doc: "Lifecycle-aware Python execution phase.", + members: &[ + ("PRE_SQL_INGRESS", "pre_sql_ingress"), + ("READ_SIDE", "read_side"), + ], +}; +const E_PYTHON_IDENTITY_STATUS: EnumSpec = EnumSpec { + name: "PythonIdentityStatus", + doc: "Read-only identity diff status for Python plan display.", + members: &[ + ("NEW", "new"), + ("CHANGED", "changed"), + ("UNCHANGED", "unchanged"), + ("UNKNOWN", "unknown"), + ], +}; +const E_COLLECTION_RENDERING: EnumSpec = EnumSpec { + name: "CollectionRendering", + doc: "Physical rendering choices for homogeneous SQL collections.", + members: &[("VALUE_LIST", "value_list"), ("ARRAY", "array")], +}; +const E_RESOURCE_KIND: EnumSpec = EnumSpec { + name: "ResourceKind", + doc: "Kinds of authored resources represented in a scope index.", + members: &[ + ("MODEL", "model"), + ("TEST", "test"), + ("SCENARIO", "scenario"), + ("FUNCTION", "function"), + ("SOURCE", "source"), + ("SEED", "seed"), + ], +}; +const E_DECLARATION_KIND: EnumSpec = EnumSpec { + name: "DeclarationKind", + doc: "Independent public declaration namespaces.", + members: &[ + ("MACRO", "macro"), + ("ENUM", "enum"), + ("CONSTANT", "constant"), + ("AUDIT", "audit"), + ("SINGULAR_AUDIT", "singular_audit"), + ("SCHEMA", "schema"), + ("SQL_HOOK", "sql_hook"), + ("PYTHON_HOOK", "python_hook"), + ], +}; +const E_SCOPE_KIND: EnumSpec = EnumSpec { + name: "ScopeKind", + doc: "Compiler-enforced declaration visibility tiers.", + members: &[ + ("GLOBAL", "global"), + ("INHERITED", "inherited"), + ("LOCAL", "local"), + ("PRIVATE", "private"), + ], +}; +const E_OWNERSHIP_ROOT_KIND: EnumSpec = EnumSpec { + name: "OwnershipRootKind", + doc: "How an ownership root participates in authored resource scope.", + members: &[("RESOURCE", "resource"), ("GLOBAL", "global")], +}; +const E_USAGE_KIND: EnumSpec = EnumSpec { + name: "UsageKind", + doc: "Why one indexed identity consumes a declaration.", + members: &[ + ("RUNTIME", "runtime"), + ("GENERATED", "generated"), + ("DECLARATION_DEPENDENCY", "declaration_dependency"), + ], +}; +const E_GRANT_KIND: EnumSpec = EnumSpec { + name: "GrantKind", + doc: "Compiler relationships that grant additional public vocabulary.", + members: &[ + ("EXPECTED_MODEL", "expected_model"), + ("TESTED_MACRO", "tested_macro"), + ], +}; +const E_VISIBILITY_REASON: EnumSpec = EnumSpec { + name: "VisibilityReason", + doc: "Positive explanations emitted by the canonical resolver.", + members: &[ + ("GLOBAL", "global"), + ("INHERITED_ANCESTOR", "inherited_ancestor"), + ("LOCAL_OWNER", "local_owner"), + ("PRIVATE_OWNER", "private_owner"), + ("EXPECTED_MODEL", "expected_model"), + ("TESTED_MACRO", "tested_macro"), + ], +}; +const E_INACCESSIBLE_REASON: EnumSpec = EnumSpec { + name: "InaccessibleReason", + doc: "Stable explanations for declarations outside effective scope.", + members: &[ + ("LOCAL_BOUNDARY", "local_boundary"), + ("SIBLING_SCOPE", "sibling_scope"), + ("DESCENDANT_SCOPE", "descendant_scope"), + ("UNRELATED_SCOPE", "unrelated_scope"), + ("PRIVATE_OWNER", "private_owner"), + ("UNSUPPORTED_RESOURCE_KIND", "unsupported_resource_kind"), + ], +}; +const E_SCOPE_DIAGNOSTIC_SEVERITY: EnumSpec = EnumSpec { + name: "ScopeDiagnosticSeverity", + doc: "Severity of a stable scope diagnostic.", + members: &[("ERROR", "error"), ("WARNING", "warning")], +}; +const E_SCOPE_DIAGNOSTIC_CODE: EnumSpec = EnumSpec { + name: "ScopeDiagnosticCode", + doc: "Stable diagnostic identities shared by future scope consumers.", + members: &[ + ("UNKNOWN_TARGET", "S001"), + ("UNQUALIFIED_TARGET", "S002"), + ("DUPLICATE_DECLARATION", "S003"), + ("INVALID_DECLARATION_NAME", "S004"), + ("RESERVED_DECLARATION_NAME", "S005"), + ("INACCESSIBLE_DECLARATION", "S006"), + ("LOCAL_NEEDED_BY_DESCENDANT", "S007"), + ("OVER_BROAD_INHERITED", "S008"), + ("REQUIRES_GLOBAL_PLACEMENT", "S009"), + ("UNUSED_DECLARATION", "S010"), + ("INVALID_PROSPECTIVE_PATH", "S011"), + ("NESTED_DECLARATION_ROOT", "S012"), + ("INCOMPLETE_USAGE", "S013"), + ("INVALID_CURSOR", "S014"), + ("MACRO_MODULE_IMPORT", "S015"), + ("INVALID_MACRO_DEPENDENCY", "S016"), + ("MACRO_DEPENDENCY_CYCLE", "S017"), + ("DUPLICATE_RESOURCE", "S018"), + ("RESOURCE_PARSE_ERROR", "S019"), + ("DECLARATION_PARSE_ERROR", "S020"), + ("MACRO_PARSE_ERROR", "S021"), + ("RELATIONSHIP_PARSE_ERROR", "S022"), + ("CONFIG_PARSE_ERROR", "S023"), + ("OVER_BROAD_GLOBAL", "S024"), + ], +}; + +/// Compiler, rule, lint and config string enums. +pub(crate) const COMPILER_ENUMS: &[EnumSpec] = &[ + E_CONTRACT_FINDING_KIND, + E_COMPILER_STAGE, + E_HOOK_READ_TYPE, + E_LINT_SEVERITY, + E_DIAGNOSTIC_PHASE, + E_DIAGNOSTIC_SEVERITY, + E_COLUMN_TRANSFORM_KIND, + E_COLUMN_LINEAGE_CONFIDENCE, + E_COLUMN_LINEAGE_MODE, + E_POLYGLOT_ANALYSIS_DIALECT, + E_MIGRATION_DISCOVERY, + E_MIGRATION_COMPATIBILITY, + E_OLD_NAME_VIEW_ACTION, + E_SELECTOR_KIND, + E_PYTHON_RUN_PHASE, + E_PYTHON_IDENTITY_STATUS, + E_COLLECTION_RENDERING, + E_RESOURCE_KIND, + E_DECLARATION_KIND, + E_SCOPE_KIND, + E_OWNERSHIP_ROOT_KIND, + E_USAGE_KIND, + E_GRANT_KIND, + E_VISIBILITY_REASON, + E_INACCESSIBLE_REASON, + E_SCOPE_DIAGNOSTIC_SEVERITY, + E_SCOPE_DIAGNOSTIC_CODE, +]; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compiler_enums/mod.rs b/crates/sqlbuild-python/src/bindings/_helpers/compiler_enums/mod.rs new file mode 100644 index 0000000000..61d449b063 --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/compiler_enums/mod.rs @@ -0,0 +1 @@ +pub(crate) mod constants; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compiler_errors/constants.rs b/crates/sqlbuild-python/src/bindings/_helpers/compiler_errors/constants.rs new file mode 100644 index 0000000000..78d9a96fef --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/compiler_errors/constants.rs @@ -0,0 +1,344 @@ +//! The compiler, rule, lint and config expected-error classes hosted natively. + +use crate::bindings::_helpers::records::constants::{EXCEPTION_PARENT, VALUE_ERROR_PARENT}; +use crate::bindings::_helpers::records::models::{ClassValue, ErrorSpec}; + +/// Compiler, rule, lint and config expected-error classes, each after its parent. +pub(crate) const COMPILER_ERRORS: &[ErrorSpec] = &[ + ErrorSpec { + name: "DiscoveryError", + parent: VALUE_ERROR_PARENT, + doc: "Base error for expected discovery failures.", + class_attributes: &[("code", ClassValue::Text("D000"))], + required: &[], + fixed: &[], + }, + ErrorSpec { + name: "ProjectConfigError", + parent: "DiscoveryError", + doc: "Raised when shared or local project config files are invalid.", + class_attributes: &[("code", ClassValue::Text("D001"))], + required: &[], + fixed: &[], + }, + ErrorSpec { + name: "ModelSqlParseError", + parent: "DiscoveryError", + doc: "Raised when an authored SQL model file has an invalid shape.", + class_attributes: &[("code", ClassValue::Text("D002"))], + required: &[], + fixed: &[], + }, + ErrorSpec { + name: "SqlTestParseError", + parent: "DiscoveryError", + doc: "Raised when an authored SQL-native test file has an invalid shape.", + class_attributes: &[("code", ClassValue::Text("D003"))], + required: &[], + fixed: &[], + }, + ErrorSpec { + name: "SqlAuditParseError", + parent: "DiscoveryError", + doc: "Raised when an authored SQL audit file has an invalid shape.", + class_attributes: &[("code", ClassValue::Text("D004"))], + required: &[], + fixed: &[], + }, + ErrorSpec { + name: "SchemaParseError", + parent: "DiscoveryError", + doc: "Raised when a schema.yml file has an invalid shape.", + class_attributes: &[("code", ClassValue::Text("D005"))], + required: &[], + fixed: &[], + }, + ErrorSpec { + name: "SourceParseError", + parent: "DiscoveryError", + doc: "Raised when a sources/*.yml file has an invalid shape.", + class_attributes: &[("code", ClassValue::Text("D006"))], + required: &[], + fixed: &[], + }, + ErrorSpec { + name: "DiscoveryConflictError", + parent: "DiscoveryError", + doc: "Raised when discovered project inputs conflict across files.", + class_attributes: &[("code", ClassValue::Text("D007"))], + required: &[], + fixed: &[], + }, + ErrorSpec { + name: "SeedDiscoveryError", + parent: "DiscoveryError", + doc: "Raised when declared seed metadata does not match local seed files.", + class_attributes: &[("code", ClassValue::Text("D008"))], + required: &[], + fixed: &[], + }, + ErrorSpec { + name: "SqlScenarioParseError", + parent: "DiscoveryError", + doc: "Raised when an authored SQL scenario file has an invalid shape.", + class_attributes: &[("code", ClassValue::Text("D009"))], + required: &[], + fixed: &[], + }, + ErrorSpec { + name: "PythonNodeDiscoveryError", + parent: "DiscoveryError", + doc: "Raised when project Python nodes cannot be discovered.", + class_attributes: &[("code", ClassValue::Text("D011"))], + required: &[], + fixed: &[], + }, + ErrorSpec { + name: "ProviderDiscoveryError", + parent: "DiscoveryError", + doc: "Raised when project providers cannot be discovered.", + class_attributes: &[("code", ClassValue::Text("D012"))], + required: &[], + fixed: &[], + }, + ErrorSpec { + name: "DeclarationParseError", + parent: "DiscoveryError", + doc: "Raised when an authored public SQL declaration is invalid.", + class_attributes: &[("code", ClassValue::Text("D013"))], + required: &[], + fixed: &[], + }, + ErrorSpec { + name: "SqlHookParseError", + parent: "DiscoveryError", + doc: "Raised when an authored SQL hook resource has an invalid shape.", + class_attributes: &[("code", ClassValue::Text("D014"))], + required: &[], + fixed: &[], + }, + ErrorSpec { + name: "EventExporterDiscoveryError", + parent: "DiscoveryError", + doc: "Raised when project lifecycle event exporters are invalid.", + class_attributes: &[("code", ClassValue::Text("D015"))], + required: &[], + fixed: &[], + }, + ErrorSpec { + name: "ProjectPythonPathError", + parent: "DiscoveryError", + doc: "Raised when project-owned Python lives outside a supported extension root.", + class_attributes: &[("code", ClassValue::Text("D016"))], + required: &[], + fixed: &[], + }, + ErrorSpec { + name: "ProjectPathError", + parent: "DiscoveryError", + doc: "Raised when an authored project path cannot be read as valid UTF-8.", + class_attributes: &[("code", ClassValue::Text("D016"))], + required: &[], + fixed: &[], + }, + ErrorSpec { + name: "ResourceIdentityError", + parent: "DiscoveryError", + doc: "Raised when an authored resource identity is not canonical snake_case.", + class_attributes: &[("code", ClassValue::Text("D016"))], + required: &[], + fixed: &[], + }, + ErrorSpec { + name: "MeasurementAuditError", + parent: "builtins.ValueError", + doc: "Raised when a measurement audit contract or value is invalid.", + class_attributes: &[], + required: &[], + fixed: &[], + }, + ErrorSpec { + name: "ContractAdoptionError", + parent: "builtins.RuntimeError", + doc: "Raised when safe contract source editing cannot continue.", + class_attributes: &[], + required: &[], + fixed: &[], + }, + ErrorSpec { + name: "FingerprintInputError", + parent: VALUE_ERROR_PARENT, + doc: "Raised when fingerprint storage inputs or rows are invalid.", + class_attributes: &[("code", ClassValue::Text("F000"))], + required: &[], + fixed: &[], + }, + ErrorSpec { + name: "QueryFingerprintError", + parent: "builtins.ValueError", + doc: "Raised when SQL cannot be tokenized into a change fingerprint.", + class_attributes: &[], + required: &[], + fixed: &[], + }, + ErrorSpec { + name: "NativeStageMismatchError", + parent: "builtins.RuntimeError", + doc: "Raised when native state the compiler replays is inconsistent with its inputs.", + class_attributes: &[], + required: &[], + fixed: &[], + }, + ErrorSpec { + name: "UnknownLineageDialectError", + parent: "builtins.ValueError", + doc: "Raised, as Python's parser raised `ValueError`, for a dialect Polyglot does not know.", + class_attributes: &[], + required: &[], + fixed: &[], + }, + ErrorSpec { + name: "RefactorValueError", + parent: "builtins.ValueError", + doc: "An unhandled value failure the Python planner also leaves to the caller.", + class_attributes: &[], + required: &[], + fixed: &[], + }, + ErrorSpec { + name: "RefactorIOError", + parent: "builtins.OSError", + doc: "A file read or write failure the Python planner also leaves to the caller.", + class_attributes: &[], + required: &[], + fixed: &[], + }, + ErrorSpec { + name: "ScopeError", + parent: VALUE_ERROR_PARENT, + doc: "Base error for invalid scope-domain input.", + class_attributes: &[("code", ClassValue::Text("S000"))], + required: &[], + fixed: &[], + }, + ErrorSpec { + name: "InvalidScopePathError", + parent: "ScopeError", + doc: "Raised when a project-relative scope path is invalid.", + class_attributes: &[("code", ClassValue::Text("S011"))], + required: &[], + fixed: &[], + }, + ErrorSpec { + name: "InvalidQualifiedIdentityError", + parent: "ScopeError", + doc: "Raised when a qualified scope identity cannot be parsed.", + class_attributes: &[("code", ClassValue::Text("S002"))], + required: &[], + fixed: &[], + }, + ErrorSpec { + name: "UnindexableScopeError", + parent: "ScopeError", + doc: "Raised when the native scope index cannot place a discovered path or resource kind.", + class_attributes: &[], + required: &[], + fixed: &[], + }, + ErrorSpec { + name: "ScopeValidationError", + parent: "ScopeError", + doc: "Aggregate error raised only when strict scope validation is requested.", + class_attributes: &[], + required: &["diagnostics"], + fixed: &[], + }, + ErrorSpec { + name: "UnsupportedLexicalSyntaxError", + parent: "builtins.ValueError", + doc: "Raised when an adapter declares lexical rules the native SQL scanners do not support.", + class_attributes: &[], + required: &[], + fixed: &[], + }, + ErrorSpec { + name: "LintError", + parent: EXCEPTION_PARENT, + doc: "A lint or format run could not proceed.", + class_attributes: &[("code", ClassValue::Text("L001"))], + required: &[], + fixed: &[], + }, + ErrorSpec { + name: "NativeLintError", + parent: "LintError", + doc: "The SQLBuild-owned native lint engine rejected a request or response.", + class_attributes: &[("code", ClassValue::Text("L003"))], + required: &[], + fixed: &[], + }, + ErrorSpec { + name: "ProjectCompileError", + parent: "LintError", + doc: "The project could not be compiled, so its SQL cannot be linted.", + class_attributes: &[("code", ClassValue::Text("L005"))], + required: &[], + fixed: &[], + }, + ErrorSpec { + name: "LintStoppedError", + parent: "builtins.Exception", + doc: "A speculative lint run was stopped because its result is no longer needed.", + class_attributes: &[], + required: &[], + fixed: &[], + }, + ErrorSpec { + name: "SpecConfigError", + parent: "builtins.RuntimeError", + doc: "Raised when project specification configuration is invalid.", + class_attributes: &[], + required: &[], + fixed: &[], + }, + ErrorSpec { + name: "ConfigValueTypeError", + parent: VALUE_ERROR_PARENT, + doc: "Raised when an authored config value has the wrong runtime type.", + class_attributes: &[], + required: &["key", "expected", "actual_type"], + fixed: &[], + }, + ErrorSpec { + name: "DbtInteropError", + parent: VALUE_ERROR_PARENT, + doc: "Base class for expected dbt interop errors.", + class_attributes: &[("code", ClassValue::Text("C240"))], + required: &[], + fixed: &[], + }, + ErrorSpec { + name: "DbtInteropConfigError", + parent: "DbtInteropError", + doc: "Raised when dbt project configuration is missing or invalid.", + class_attributes: &[("code", ClassValue::Text("C240"))], + required: &[], + fixed: &[], + }, + ErrorSpec { + name: "DbtInteropRuntimeError", + parent: "DbtInteropError", + doc: "Raised when dbt command execution fails.", + class_attributes: &[("code", ClassValue::Text("C241"))], + required: &[], + fixed: &[], + }, + ErrorSpec { + name: "DbtInteropArgumentError", + parent: "DbtInteropError", + doc: "Raised when `sqb dbt` arguments cannot be routed safely.", + class_attributes: &[("code", ClassValue::Text("C230"))], + required: &[], + fixed: &[], + }, +]; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compiler_errors/mod.rs b/crates/sqlbuild-python/src/bindings/_helpers/compiler_errors/mod.rs new file mode 100644 index 0000000000..61d449b063 --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/compiler_errors/mod.rs @@ -0,0 +1 @@ +pub(crate) mod constants; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compiler_records/constants.rs b/crates/sqlbuild-python/src/bindings/_helpers/compiler_records/constants.rs index 048b70cc3d..9d37a9f40f 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compiler_records/constants.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compiler_records/constants.rs @@ -1,9 +1,6 @@ -//! Layouts of the compiler, rule, lint and config value records hosted natively. +//! Layouts of the compiler, lint and config value records hosted natively. -use crate::bindings::_helpers::records::constants::{EXCEPTION_PARENT, VALUE_ERROR_PARENT}; -use crate::bindings::_helpers::records::models::{ - ClassValue, EnumSpec, ErrorSpec, FieldDefault, FieldSpec, RecordSpec, -}; +use crate::bindings::_helpers::records::models::{FieldDefault, FieldSpec, RecordSpec}; const R_NATIVE_MODEL_ANALYSIS_REQUEST: RecordSpec = RecordSpec { name: "NativeModelAnalysisRequest", @@ -575,124 +572,6 @@ const R_CLONE_PIPELINE_RESULT: RecordSpec = RecordSpec { FieldSpec::new("origin_source_entries", FieldDefault::EmptyTuple), ], }; -const R_PARSED_SELECTOR: RecordSpec = RecordSpec { - name: "ParsedSelector", - doc: "One parsed selector token before graph resolution.", - fields: &[ - FieldSpec::required("kind"), - FieldSpec::required("value"), - FieldSpec::new("upstream", FieldDefault::False), - FieldSpec::new("downstream", FieldDefault::False), - ], -}; -const R_PATH_SELECTOR: RecordSpec = RecordSpec { - name: "PathSelector", - doc: "A directed path selector between two model names with optional endpoint expansion.", - fields: &[ - FieldSpec::required("start_name"), - FieldSpec::required("end_name"), - FieldSpec::new("upstream", FieldDefault::False), - FieldSpec::new("downstream", FieldDefault::False), - ], -}; -const R_SQL_TEST_SELECTION: RecordSpec = RecordSpec { - name: "SqlTestSelection", - doc: "Unit tests selected or excluded by name, beside the tests of the selected resources.", - fields: &[ - FieldSpec::new("names", FieldDefault::EmptyFrozenset), - FieldSpec::new("excluded_names", FieldDefault::EmptyFrozenset), - FieldSpec::new("models_selected", FieldDefault::True), - ], -}; -const R_PLANNER_SCOPE: RecordSpec = RecordSpec { - name: "PlannerScope", - doc: "Resolved graph scope for one planner invocation.", - fields: &[ - FieldSpec::required("upstream_deps"), - FieldSpec::required("downstream_deps"), - FieldSpec::required("all_keys"), - FieldSpec::required("models_by_name"), - FieldSpec::required("selected_keys"), - FieldSpec::required("execution_order"), - FieldSpec::new("user_selected_keys", FieldDefault::EmptyFrozenset), - FieldSpec::new("python_read_source_names", FieldDefault::EmptyFrozenset), - FieldSpec::new( - "sql_test_selection", - FieldDefault::Call("sqlbuild._native", "SqlTestSelection"), - ), - ], -}; -const R_PLANNER_RELATIONS_CONTEXT: RecordSpec = RecordSpec { - name: "PlannerRelationsContext", - doc: "Resolved relation and source inputs for plan entry construction.", - fields: &[ - FieldSpec::required("model_locations"), - FieldSpec::required("seed_locations"), - FieldSpec::required("function_locations"), - FieldSpec::required("source_map"), - FieldSpec::required("source_read_map"), - FieldSpec::required("python_source_read_map"), - FieldSpec::required("source_warehouse_columns"), - FieldSpec::required("star_exclude_keyword"), - FieldSpec::new("listed_source_names", FieldDefault::EmptyFrozenset), - ], -}; -const R_SQL_TEST_ASSERTION_STEP: RecordSpec = RecordSpec { - name: "SqlTestAssertionStep", - doc: "One zero-row assertion step in a SQL-native unit test.", - fields: &[ - FieldSpec::required("name"), - FieldSpec::required("resolved_sql"), - FieldSpec::new("lifted_ctes", FieldDefault::EmptyTuple), - FieldSpec::new("comparison_body_sql", FieldDefault::None), - ], -}; -const R_PLANNER_SELECTION: RecordSpec = RecordSpec { - name: "PlannerSelection", - doc: "User-facing selection inputs for one planner invocation.", - fields: &[ - FieldSpec::new("select", FieldDefault::EmptyTuple), - FieldSpec::new("exclude", FieldDefault::EmptyTuple), - FieldSpec::new("selected_keys", FieldDefault::None), - FieldSpec::new("python_read_source_names", FieldDefault::EmptyFrozenset), - FieldSpec::new( - "sql_test_selection", - FieldDefault::Call("sqlbuild._native", "SqlTestSelection"), - ), - ], -}; -const R_PLANNER_OVERRIDES: RecordSpec = RecordSpec { - name: "PlannerOverrides", - doc: "Explicit user overrides that force or bound planner decisions.", - fields: &[ - FieldSpec::new("cursor_overrides", FieldDefault::None), - FieldSpec::new("full_refresh", FieldDefault::False), - FieldSpec::new("reload_sources", FieldDefault::False), - FieldSpec::new("forced_stale_model_names", FieldDefault::EmptyTuple), - FieldSpec::new("external_blocked_model_names", FieldDefault::EmptyTuple), - FieldSpec::new("max_microbatches", FieldDefault::None), - ], -}; -const R_DEFERRAL_INPUTS: RecordSpec = RecordSpec { - name: "DeferralInputs", - doc: "Deferred relation and source-deferral inputs for planning.", - fields: &[ - FieldSpec::new("deferred_locations", FieldDefault::None), - FieldSpec::new("deferred_relations", FieldDefault::None), - FieldSpec::new("defer_sources_to", FieldDefault::None), - FieldSpec::new("source_deferral_enabled", FieldDefault::True), - ], -}; -const R_PLANNER_POLICIES: RecordSpec = RecordSpec { - name: "PlannerPolicies", - doc: "Behavior policies selected by the caller for one planner invocation.", - fields: &[ - FieldSpec::new("auto_load_sources", FieldDefault::False), - FieldSpec::new("selection_diagnostics", FieldDefault::False), - FieldSpec::new("plan_sql_tests", FieldDefault::True), - FieldSpec::new("record_migration_fingerprints", FieldDefault::True), - ], -}; const R_DISCOVERED_PYTHON_LOADER_METADATA: RecordSpec = RecordSpec { name: "DiscoveredPythonLoaderMetadata", doc: "Loader-specific metadata carried by an internal Python loader node.", @@ -767,20 +646,6 @@ const R_LITERAL_SQL_RELATION: RecordSpec = RecordSpec { FieldSpec::required("relation"), ], }; -const R_DIRECT_SOURCE_FRESHNESS_PLANNING_RESULT: RecordSpec = RecordSpec { - name: "DirectSourceFreshnessPlanningResult", - doc: "Direct planning-time source freshness observations and comparisons.", - fields: &[ - FieldSpec::new("observed_records", FieldDefault::EmptyTuple), - FieldSpec::new("previous_records", FieldDefault::EmptyTuple), - FieldSpec::new("changed_identities", FieldDefault::EmptyFrozenset), - FieldSpec::new("unchanged_identities", FieldDefault::EmptyFrozenset), - FieldSpec::new("unknown_source_names", FieldDefault::EmptyTuple), - FieldSpec::new("unknown_sources", FieldDefault::EmptyDict), - FieldSpec::new("age_statuses", FieldDefault::EmptyDict), - FieldSpec::new("propagation", FieldDefault::None), - ], -}; const R_SQL_BINDING_DIAGNOSTIC: RecordSpec = RecordSpec { name: "SqlBindingDiagnostic", doc: "One native binding error or non-blocking runtime-conversion warning.", @@ -916,346 +781,8 @@ const R_DBT_LS_RESULT: RecordSpec = RecordSpec { doc: "Parsed result from one dbt ls invocation.", fields: &[FieldSpec::required("nodes"), FieldSpec::required("command")], }; -const R_RULE_GUIDANCE: RecordSpec = RecordSpec { - name: "RuleGuidance", - doc: "Structured authoring guidance shared by diagnostics and generated documentation.", - fields: &[ - FieldSpec::required("good_example"), - FieldSpec::required("anti_tautology"), - FieldSpec::required("mutation_check"), - ], -}; -const R_RULE_EXEMPTION: RecordSpec = RecordSpec { - name: "RuleExemption", - doc: "", - fields: &[ - FieldSpec::required("rule"), - FieldSpec::required("path"), - FieldSpec::required("reason"), - ], -}; -const R_GRAPH_EDGE_EXCLUSION: RecordSpec = RecordSpec { - name: "GraphEdgeExclusion", - doc: "One exact, temporary internal-layer dependency exception.", - fields: &[ - FieldSpec::required("consumer"), - FieldSpec::required("dependency"), - FieldSpec::required("reason"), - ], -}; -const R_RULE_IGNORE: RecordSpec = RecordSpec { - name: "RuleIgnore", - doc: "", - fields: &[ - FieldSpec::required("rules"), - FieldSpec::required("paths"), - FieldSpec::required("reason"), - FieldSpec::new("selectors", FieldDefault::EmptyTuple), - ], -}; -const R_SELECT_STAR_ALLOW: RecordSpec = RecordSpec { - name: "SelectStarAllow", - doc: "", - fields: &[FieldSpec::required("paths"), FieldSpec::required("reason")], -}; -const R_THRESHOLD_OVERRIDE: RecordSpec = RecordSpec { - name: "ThresholdOverride", - doc: "", - fields: &[ - FieldSpec::required("paths"), - FieldSpec::required("thresholds"), - FieldSpec::required("reason"), - ], -}; -const R_RULES_CACHE_CONFIG: RecordSpec = RecordSpec { - name: "RulesCacheConfig", - doc: "", - fields: &[FieldSpec::new("enabled", FieldDefault::True)], -}; -const R_SQL_TEST_RULES_CONFIG: RecordSpec = RecordSpec { - name: "SqlTestRulesConfig", - doc: "", - fields: &[FieldSpec::new( - "pipeline_directory", - FieldDefault::Text("pipelines"), - )], -}; -const R_PREPARED_SQL_LINT_RESULT: RecordSpec = RecordSpec { - name: "PreparedSqlLintResult", - doc: "Reusable expansion context and completed preliminary SQL checks.", - fields: &[ - FieldSpec::required("result"), - FieldSpec::required("context"), - ], -}; -const R_SQL_EXPANSION_REUSE: RecordSpec = RecordSpec { - name: "SqlExpansionReuse", - doc: "Compiler declaration scope for post-assembly SQL checks; assembly still runs its macros.", - fields: &[ - FieldSpec::required("discovered_inputs"), - FieldSpec::required("declaration_scope"), - ], -}; -const R_PREPARED_SQL_LINT: RecordSpec = RecordSpec { - name: "PreparedSqlLint", - doc: "Process-local SQL checks started after compiler expansion has completed.", - fields: &[ - FieldSpec::required("codes"), - FieldSpec::required("future"), - FieldSpec::new("project_dir", FieldDefault::None), - ], -}; -const R_SQL_RULES_EVALUATION: RecordSpec = RecordSpec { - name: "SqlRulesEvaluation", - doc: "SQL rule findings of one run, their cache accounting and elapsed time.", - fields: &[ - FieldSpec::required("findings"), - FieldSpec::required("cache_hits"), - FieldSpec::required("cache_misses"), - FieldSpec::new("sql_ms", FieldDefault::Int(0)), - ], -}; -const R_RULE_RUN_PLAN: RecordSpec = RecordSpec { - name: "RuleRunPlan", - doc: "The catalogue, selected rules, project slice and settings one rules run evaluates.", - fields: &[ - FieldSpec::required("effective_config"), - FieldSpec::required("project_dir"), - FieldSpec::required("catalogue"), - FieldSpec::required("custom_payloads"), - FieldSpec::required("selected"), - FieldSpec::required("skipped_type_proof_rules"), - FieldSpec::required("model_paths"), - FieldSpec::required("project"), - FieldSpec::required("config"), - ], -}; -const R_RULES_RUN_RESULT: RecordSpec = RecordSpec { - name: "RulesRunResult", - doc: "Unified findings and phase measurements for configured rules.", - fields: &[ - FieldSpec::required("findings"), - FieldSpec::required("evaluated_models"), - FieldSpec::required("built_in_ms"), - FieldSpec::required("custom_ms"), - FieldSpec::new("custom_cpu_ms", FieldDefault::Int(0)), - FieldSpec::new("unevaluated_resources", FieldDefault::Int(0)), - FieldSpec::new("cache_hits", FieldDefault::Int(0)), - FieldSpec::new("cache_misses", FieldDefault::Int(0)), - FieldSpec::new("skipped_type_proof_rules", FieldDefault::EmptyTuple), - ], -}; -const R_NATIVE_RULES_EVALUATION: RecordSpec = RecordSpec { - name: "NativeRulesEvaluation", - doc: "The built-in rules response, decoded natively.", - fields: &[ - FieldSpec::required("findings"), - FieldSpec::required("selected_codes"), - FieldSpec::required("evaluated_models"), - FieldSpec::required("cache_hits"), - FieldSpec::required("cache_misses"), - FieldSpec::required("built_in_ms"), - FieldSpec::required("reused"), - ], -}; -const R_CUSTOM_RULES_OUTCOME: RecordSpec = RecordSpec { - name: "CustomRulesOutcome", - doc: "Custom-rule findings in canonical order and their cache accounting.", - fields: &[ - FieldSpec::required("findings"), - FieldSpec::required("cache_hits"), - FieldSpec::required("cache_misses"), - FieldSpec::required("custom_ms"), - FieldSpec::new("custom_cpu_ms", FieldDefault::Int(0)), - ], -}; -const E_CONTRACT_FINDING_KIND: EnumSpec = EnumSpec { - name: "ContractFindingKind", - doc: "Stable physical-to-authored contract difference categories.", - members: &[ - ("MISSING_DECLARATION", "missing_declaration"), - ("MISSING_PHYSICAL_COLUMN", "missing_physical_column"), - ("TYPE_MISMATCH", "type_mismatch"), - ("RELATION_MISSING", "relation_missing"), - ("INSPECTION_UNAVAILABLE", "inspection_unavailable"), - ("OWNERSHIP_CONFLICT", "ownership_conflict"), - ], -}; -const E_COMPILER_STAGE: EnumSpec = EnumSpec { - name: "CompilerStage", - doc: "Frontier objects handed from one compiler stage to the next, in pipeline order.", - members: &[ - ("DISCOVERED_PROJECT_INPUTS", "discovered_project_inputs"), - ("COMPILE_PROJECT_INPUTS", "compile_project_inputs"), - ("COMPILED_PROJECT", "compiled_project"), - ], -}; -const E_HOOK_READ_TYPE: EnumSpec = EnumSpec { - name: "HookReadType", - doc: "Kind of model hook that declares or writes a read.", - members: &[ - ("PYTHON", "Python"), - ("SQL", "SQL"), - ("INLINE_SQL", "inline SQL"), - ], -}; -const E_LINT_SEVERITY: EnumSpec = EnumSpec { - name: "LintSeverity", - doc: "Severity assigned to one lint violation.", - members: &[("FAULT", "fault"), ("WARNING", "warning")], -}; -const E_DIAGNOSTIC_PHASE: EnumSpec = EnumSpec { - name: "DiagnosticPhase", - doc: "Phase that produced a compiler diagnostic.", - members: &[ - ("COMPILE", "compile"), - ("CONTRACT", "contract"), - ("RULE", "rule"), - ("PLAN", "plan"), - ("BUILD", "build"), - ("AUDIT", "audit"), - ("TEST", "test"), - ("CONNECTION", "connection"), - ], -}; -const E_DIAGNOSTIC_SEVERITY: EnumSpec = EnumSpec { - name: "DiagnosticSeverity", - doc: "Severity for a compiler diagnostic.", - members: &[("ERROR", "error"), ("WARNING", "warning"), ("INFO", "info")], -}; -const E_COLUMN_TRANSFORM_KIND: EnumSpec = EnumSpec { - name: "ColumnTransformKind", - doc: "High-level transform classification for one output column.", - members: &[ - ("DIRECT", "direct"), - ("CAST", "cast"), - ("EXPRESSION", "expression"), - ("AGGREGATION", "aggregation"), - ("STAR", "star"), - ("CONSTANT", "constant"), - ("UNKNOWN", "unknown"), - ], -}; -const E_COLUMN_LINEAGE_CONFIDENCE: EnumSpec = EnumSpec { - name: "ColumnLineageConfidence", - doc: "Coarse confidence that a lineage edge is fully understood.", - members: &[ - ("HIGH", "high"), - ("MEDIUM", "medium"), - ("LOW", "low"), - ("UNKNOWN", "unknown"), - ], -}; -const E_COLUMN_LINEAGE_MODE: EnumSpec = EnumSpec { - name: "ColumnLineageMode", - doc: "Column lineage analyzer mode.", - members: &[("RICH", "rich"), ("FAST", "fast")], -}; -const E_POLYGLOT_ANALYSIS_DIALECT: EnumSpec = EnumSpec { - name: "PolyglotAnalysisDialect", - doc: "Dialect names accepted by Polyglot analyze_query.", - members: &[ - ("GENERIC", "generic"), - ("POSTGRESQL", "postgresql"), - ("MYSQL", "mysql"), - ("BIGQUERY", "bigquery"), - ("SNOWFLAKE", "snowflake"), - ("DUCKDB", "duckdb"), - ("SQLITE", "sqlite"), - ("HIVE", "hive"), - ("SPARK", "spark"), - ("TRINO", "trino"), - ("PRESTO", "presto"), - ("REDSHIFT", "redshift"), - ("TSQL", "tsql"), - ("ORACLE", "oracle"), - ("CLICKHOUSE", "clickhouse"), - ("DATABRICKS", "databricks"), - ("ATHENA", "athena"), - ("TERADATA", "teradata"), - ("DORIS", "doris"), - ("STARROCKS", "starrocks"), - ("MATERIALIZE", "materialize"), - ("RISINGWAVE", "risingwave"), - ("SINGLESTORE", "singlestore"), - ("COCKROACHDB", "cockroachdb"), - ("TIDB", "tidb"), - ("DRUID", "druid"), - ("SOLR", "solr"), - ("TABLEAU", "tableau"), - ("DUNE", "dune"), - ("FABRIC", "fabric"), - ("DRILL", "drill"), - ("DREMIO", "dremio"), - ("EXASOL", "exasol"), - ("DATAFUSION", "datafusion"), - ], -}; -const E_MIGRATION_DISCOVERY: EnumSpec = EnumSpec { - name: "MigrationDiscovery", - doc: "How a migration was requested.", - members: &[("MANUAL", "manual"), ("AUTOMATIC", "automatic")], -}; -const E_MIGRATION_COMPATIBILITY: EnumSpec = EnumSpec { - name: "MigrationCompatibility", - doc: "Outcome of checking the origin relation against the destination model.", - members: &[ - ("COMPATIBLE", "compatible"), - ("INCOMPATIBLE", "incompatible"), - ("NOT_CHECKED", "not_checked"), - ], -}; -const E_OLD_NAME_VIEW_ACTION: EnumSpec = EnumSpec { - name: "OldNameViewAction", - doc: "What a build does at a migrated model's old name.", - members: &[ - ("ARCHIVE_AND_VIEW", "archive_and_view"), - ("VIEW_ONLY", "view_only"), - ("LIVE", "live"), - ("NONE", "none"), - ], -}; -const E_SELECTOR_KIND: EnumSpec = EnumSpec { - name: "SelectorKind", - doc: "", - members: &[ - ("NAME", "name"), - ("SEED", "seed"), - ("SOURCE", "source"), - ("TASK", "task"), - ("ASSET", "asset"), - ("LOADER", "loader"), - ("CHECK", "check"), - ("TAG", "tag"), - ("PATH", "path"), - ("TEST", "test"), - ], -}; -const E_PYTHON_RUN_PHASE: EnumSpec = EnumSpec { - name: "PythonRunPhase", - doc: "Lifecycle-aware Python execution phase.", - members: &[ - ("PRE_SQL_INGRESS", "pre_sql_ingress"), - ("READ_SIDE", "read_side"), - ], -}; -const E_PYTHON_IDENTITY_STATUS: EnumSpec = EnumSpec { - name: "PythonIdentityStatus", - doc: "Read-only identity diff status for Python plan display.", - members: &[ - ("NEW", "new"), - ("CHANGED", "changed"), - ("UNCHANGED", "unchanged"), - ("UNKNOWN", "unknown"), - ], -}; -const E_COLLECTION_RENDERING: EnumSpec = EnumSpec { - name: "CollectionRendering", - doc: "Physical rendering choices for homogeneous SQL collections.", - members: &[("VALUE_LIST", "value_list"), ("ARRAY", "array")], -}; -/// Compiler, rule, lint and config value records. +/// The compiler, lint and config value records. pub(crate) const COMPILER_RECORDS: &[RecordSpec] = &[ R_NATIVE_MODEL_ANALYSIS_REQUEST, R_NATIVE_MODEL_ANALYSES, @@ -1304,16 +831,6 @@ pub(crate) const COMPILER_RECORDS: &[RecordSpec] = &[ R_CLONE_PIPELINE_OPTIONS, R_CLONE_PIPELINE_CONNECTION, R_CLONE_PIPELINE_RESULT, - R_PARSED_SELECTOR, - R_PATH_SELECTOR, - R_SQL_TEST_SELECTION, - R_PLANNER_SCOPE, - R_PLANNER_RELATIONS_CONTEXT, - R_SQL_TEST_ASSERTION_STEP, - R_PLANNER_SELECTION, - R_PLANNER_OVERRIDES, - R_DEFERRAL_INPUTS, - R_PLANNER_POLICIES, R_DISCOVERED_PYTHON_LOADER_METADATA, R_DISCOVERED_PYTHON_TASK_METADATA, R_DISCOVERED_PYTHON_ASSET_METADATA, @@ -1321,7 +838,6 @@ pub(crate) const COMPILER_RECORDS: &[RecordSpec] = &[ R_PYTHON_NODE_DEPENDENCY_EDGE, R_PYTHON_SQL_RUN_SELECTION, R_LITERAL_SQL_RELATION, - R_DIRECT_SOURCE_FRESHNESS_PLANNING_RESULT, R_SQL_BINDING_DIAGNOSTIC, R_SQL_BINDING_RESULT, R_SQL_SCHEMA_VALIDATION_REQUEST, @@ -1333,379 +849,4 @@ pub(crate) const COMPILER_RECORDS: &[RecordSpec] = &[ R_DBT_COMMAND_RESULT, R_DBT_NODE_EXECUTION_RESULT, R_DBT_LS_RESULT, - R_RULE_GUIDANCE, - R_RULE_EXEMPTION, - R_GRAPH_EDGE_EXCLUSION, - R_RULE_IGNORE, - R_SELECT_STAR_ALLOW, - R_THRESHOLD_OVERRIDE, - R_RULES_CACHE_CONFIG, - R_SQL_TEST_RULES_CONFIG, - R_PREPARED_SQL_LINT_RESULT, - R_SQL_EXPANSION_REUSE, - R_PREPARED_SQL_LINT, - R_SQL_RULES_EVALUATION, - R_RULE_RUN_PLAN, - R_RULES_RUN_RESULT, - R_NATIVE_RULES_EVALUATION, - R_CUSTOM_RULES_OUTCOME, -]; -/// Compiler, rule, lint and config string enums. -pub(crate) const COMPILER_ENUMS: &[EnumSpec] = &[ - E_CONTRACT_FINDING_KIND, - E_COMPILER_STAGE, - E_HOOK_READ_TYPE, - E_LINT_SEVERITY, - E_DIAGNOSTIC_PHASE, - E_DIAGNOSTIC_SEVERITY, - E_COLUMN_TRANSFORM_KIND, - E_COLUMN_LINEAGE_CONFIDENCE, - E_COLUMN_LINEAGE_MODE, - E_POLYGLOT_ANALYSIS_DIALECT, - E_MIGRATION_DISCOVERY, - E_MIGRATION_COMPATIBILITY, - E_OLD_NAME_VIEW_ACTION, - E_SELECTOR_KIND, - E_PYTHON_RUN_PHASE, - E_PYTHON_IDENTITY_STATUS, - E_COLLECTION_RENDERING, -]; -/// Compiler, rule, lint and config expected-error classes, each after its parent. -pub(crate) const COMPILER_ERRORS: &[ErrorSpec] = &[ - ErrorSpec { - name: "DiscoveryError", - parent: VALUE_ERROR_PARENT, - doc: "Base error for expected discovery failures.", - class_attributes: &[("code", ClassValue::Text("D000"))], - required: &[], - fixed: &[], - }, - ErrorSpec { - name: "ProjectConfigError", - parent: "DiscoveryError", - doc: "Raised when shared or local project config files are invalid.", - class_attributes: &[("code", ClassValue::Text("D001"))], - required: &[], - fixed: &[], - }, - ErrorSpec { - name: "ModelSqlParseError", - parent: "DiscoveryError", - doc: "Raised when an authored SQL model file has an invalid shape.", - class_attributes: &[("code", ClassValue::Text("D002"))], - required: &[], - fixed: &[], - }, - ErrorSpec { - name: "SqlTestParseError", - parent: "DiscoveryError", - doc: "Raised when an authored SQL-native test file has an invalid shape.", - class_attributes: &[("code", ClassValue::Text("D003"))], - required: &[], - fixed: &[], - }, - ErrorSpec { - name: "SqlAuditParseError", - parent: "DiscoveryError", - doc: "Raised when an authored SQL audit file has an invalid shape.", - class_attributes: &[("code", ClassValue::Text("D004"))], - required: &[], - fixed: &[], - }, - ErrorSpec { - name: "SchemaParseError", - parent: "DiscoveryError", - doc: "Raised when a schema.yml file has an invalid shape.", - class_attributes: &[("code", ClassValue::Text("D005"))], - required: &[], - fixed: &[], - }, - ErrorSpec { - name: "SourceParseError", - parent: "DiscoveryError", - doc: "Raised when a sources/*.yml file has an invalid shape.", - class_attributes: &[("code", ClassValue::Text("D006"))], - required: &[], - fixed: &[], - }, - ErrorSpec { - name: "DiscoveryConflictError", - parent: "DiscoveryError", - doc: "Raised when discovered project inputs conflict across files.", - class_attributes: &[("code", ClassValue::Text("D007"))], - required: &[], - fixed: &[], - }, - ErrorSpec { - name: "SeedDiscoveryError", - parent: "DiscoveryError", - doc: "Raised when declared seed metadata does not match local seed files.", - class_attributes: &[("code", ClassValue::Text("D008"))], - required: &[], - fixed: &[], - }, - ErrorSpec { - name: "SqlScenarioParseError", - parent: "DiscoveryError", - doc: "Raised when an authored SQL scenario file has an invalid shape.", - class_attributes: &[("code", ClassValue::Text("D009"))], - required: &[], - fixed: &[], - }, - ErrorSpec { - name: "PythonNodeDiscoveryError", - parent: "DiscoveryError", - doc: "Raised when project Python nodes cannot be discovered.", - class_attributes: &[("code", ClassValue::Text("D011"))], - required: &[], - fixed: &[], - }, - ErrorSpec { - name: "ProviderDiscoveryError", - parent: "DiscoveryError", - doc: "Raised when project providers cannot be discovered.", - class_attributes: &[("code", ClassValue::Text("D012"))], - required: &[], - fixed: &[], - }, - ErrorSpec { - name: "DeclarationParseError", - parent: "DiscoveryError", - doc: "Raised when an authored public SQL declaration is invalid.", - class_attributes: &[("code", ClassValue::Text("D013"))], - required: &[], - fixed: &[], - }, - ErrorSpec { - name: "SqlHookParseError", - parent: "DiscoveryError", - doc: "Raised when an authored SQL hook resource has an invalid shape.", - class_attributes: &[("code", ClassValue::Text("D014"))], - required: &[], - fixed: &[], - }, - ErrorSpec { - name: "EventExporterDiscoveryError", - parent: "DiscoveryError", - doc: "Raised when project lifecycle event exporters are invalid.", - class_attributes: &[("code", ClassValue::Text("D015"))], - required: &[], - fixed: &[], - }, - ErrorSpec { - name: "ProjectPythonPathError", - parent: "DiscoveryError", - doc: "Raised when project-owned Python lives outside a supported extension root.", - class_attributes: &[("code", ClassValue::Text("D016"))], - required: &[], - fixed: &[], - }, - ErrorSpec { - name: "ProjectPathError", - parent: "DiscoveryError", - doc: "Raised when an authored project path cannot be read as valid UTF-8.", - class_attributes: &[("code", ClassValue::Text("D016"))], - required: &[], - fixed: &[], - }, - ErrorSpec { - name: "ResourceIdentityError", - parent: "DiscoveryError", - doc: "Raised when an authored resource identity is not canonical snake_case.", - class_attributes: &[("code", ClassValue::Text("D016"))], - required: &[], - fixed: &[], - }, - ErrorSpec { - name: "MeasurementAuditError", - parent: "builtins.ValueError", - doc: "Raised when a measurement audit contract or value is invalid.", - class_attributes: &[], - required: &[], - fixed: &[], - }, - ErrorSpec { - name: "ContractAdoptionError", - parent: "builtins.RuntimeError", - doc: "Raised when safe contract source editing cannot continue.", - class_attributes: &[], - required: &[], - fixed: &[], - }, - ErrorSpec { - name: "FingerprintInputError", - parent: VALUE_ERROR_PARENT, - doc: "Raised when fingerprint storage inputs or rows are invalid.", - class_attributes: &[("code", ClassValue::Text("F000"))], - required: &[], - fixed: &[], - }, - ErrorSpec { - name: "QueryFingerprintError", - parent: "builtins.ValueError", - doc: "Raised when SQL cannot be tokenized into a change fingerprint.", - class_attributes: &[], - required: &[], - fixed: &[], - }, - ErrorSpec { - name: "NativeStageMismatchError", - parent: "builtins.RuntimeError", - doc: "Raised when native state the compiler replays is inconsistent with its inputs.", - class_attributes: &[], - required: &[], - fixed: &[], - }, - ErrorSpec { - name: "UnknownLineageDialectError", - parent: "builtins.ValueError", - doc: "Raised, as Python's parser raised `ValueError`, for a dialect Polyglot does not know.", - class_attributes: &[], - required: &[], - fixed: &[], - }, - ErrorSpec { - name: "RefactorValueError", - parent: "builtins.ValueError", - doc: "An unhandled value failure the Python planner also leaves to the caller.", - class_attributes: &[], - required: &[], - fixed: &[], - }, - ErrorSpec { - name: "RefactorIOError", - parent: "builtins.OSError", - doc: "A file read or write failure the Python planner also leaves to the caller.", - class_attributes: &[], - required: &[], - fixed: &[], - }, - ErrorSpec { - name: "ScopeError", - parent: VALUE_ERROR_PARENT, - doc: "Base error for invalid scope-domain input.", - class_attributes: &[("code", ClassValue::Text("S000"))], - required: &[], - fixed: &[], - }, - ErrorSpec { - name: "InvalidScopePathError", - parent: "ScopeError", - doc: "Raised when a project-relative scope path is invalid.", - class_attributes: &[("code", ClassValue::Text("S011"))], - required: &[], - fixed: &[], - }, - ErrorSpec { - name: "InvalidQualifiedIdentityError", - parent: "ScopeError", - doc: "Raised when a qualified scope identity cannot be parsed.", - class_attributes: &[("code", ClassValue::Text("S002"))], - required: &[], - fixed: &[], - }, - ErrorSpec { - name: "UnindexableScopeError", - parent: "ScopeError", - doc: "Raised when the native scope index cannot place a discovered path or resource kind.", - class_attributes: &[], - required: &[], - fixed: &[], - }, - ErrorSpec { - name: "ScopeValidationError", - parent: "ScopeError", - doc: "Aggregate error raised only when strict scope validation is requested.", - class_attributes: &[], - required: &["diagnostics"], - fixed: &[], - }, - ErrorSpec { - name: "UnsupportedLexicalSyntaxError", - parent: "builtins.ValueError", - doc: "Raised when an adapter declares lexical rules the native SQL scanners do not support.", - class_attributes: &[], - required: &[], - fixed: &[], - }, - ErrorSpec { - name: "LintError", - parent: EXCEPTION_PARENT, - doc: "A lint or format run could not proceed.", - class_attributes: &[("code", ClassValue::Text("L001"))], - required: &[], - fixed: &[], - }, - ErrorSpec { - name: "NativeLintError", - parent: "LintError", - doc: "The SQLBuild-owned native lint engine rejected a request or response.", - class_attributes: &[("code", ClassValue::Text("L003"))], - required: &[], - fixed: &[], - }, - ErrorSpec { - name: "ProjectCompileError", - parent: "LintError", - doc: "The project could not be compiled, so its SQL cannot be linted.", - class_attributes: &[("code", ClassValue::Text("L005"))], - required: &[], - fixed: &[], - }, - ErrorSpec { - name: "LintStoppedError", - parent: "builtins.Exception", - doc: "A speculative lint run was stopped because its result is no longer needed.", - class_attributes: &[], - required: &[], - fixed: &[], - }, - ErrorSpec { - name: "SpecConfigError", - parent: "builtins.RuntimeError", - doc: "Raised when project specification configuration is invalid.", - class_attributes: &[], - required: &[], - fixed: &[], - }, - ErrorSpec { - name: "ConfigValueTypeError", - parent: VALUE_ERROR_PARENT, - doc: "Raised when an authored config value has the wrong runtime type.", - class_attributes: &[], - required: &["key", "expected", "actual_type"], - fixed: &[], - }, - ErrorSpec { - name: "DbtInteropError", - parent: VALUE_ERROR_PARENT, - doc: "Base class for expected dbt interop errors.", - class_attributes: &[("code", ClassValue::Text("C240"))], - required: &[], - fixed: &[], - }, - ErrorSpec { - name: "DbtInteropConfigError", - parent: "DbtInteropError", - doc: "Raised when dbt project configuration is missing or invalid.", - class_attributes: &[("code", ClassValue::Text("C240"))], - required: &[], - fixed: &[], - }, - ErrorSpec { - name: "DbtInteropRuntimeError", - parent: "DbtInteropError", - doc: "Raised when dbt command execution fails.", - class_attributes: &[("code", ClassValue::Text("C241"))], - required: &[], - fixed: &[], - }, - ErrorSpec { - name: "DbtInteropArgumentError", - parent: "DbtInteropError", - doc: "Raised when `sqb dbt` arguments cannot be routed safely.", - class_attributes: &[("code", ClassValue::Text("C230"))], - required: &[], - fixed: &[], - }, ]; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compiler_records/shape_classes.rs b/crates/sqlbuild-python/src/bindings/_helpers/compiler_records/shape_classes.rs index 1c9fbfd410..fed51c5c23 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compiler_records/shape_classes.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compiler_records/shape_classes.rs @@ -1,41 +1,59 @@ -//! Resolve a shape class by name: the native class once it is hosted natively, else the Python one. +//! Resolve a shape class by its Python module and name: the native class once it is hosted natively. +use pyo3::exceptions::PyLookupError; use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult, Python}; +use sqlbuild_cache::stage_capture::constants::HOSTED_SHAPE_TYPES; + use crate::bindings::_helpers::compile_driver::python_calls::imported; -use crate::bindings::_helpers::compile_records::constants::COMPILE_VALUE_RECORDS; -use crate::bindings::_helpers::compiler_records::constants::{ - COMPILER_ENUMS, COMPILER_ERRORS, COMPILER_RECORDS, -}; use crate::bindings::_helpers::records::constants::NATIVE_MODULE; -/// Whether `class` is one of the compiler shapes hosted natively. -fn hosted(class: &str) -> bool { - COMPILER_RECORDS.iter().any(|spec| spec.name == class) - || COMPILE_VALUE_RECORDS.iter().any(|spec| spec.name == class) - || COMPILER_ENUMS.iter().any(|spec| spec.name == class) - || COMPILER_ERRORS.iter().any(|spec| spec.name == class) +/// The native class name hosting `module:class`, failing when the qualified key is ambiguous. +pub(crate) fn hosted_name(module: &str, class: &str) -> PyResult> { + let mut found: Option<&'static str> = None; + for (native, python) in HOSTED_SHAPE_TYPES { + let Some((python_module, python_class)) = python.split_once(':') else { + continue; + }; + if python_module != module || python_class != class { + continue; + } + let Some(native_name) = native + .strip_prefix(NATIVE_MODULE) + .and_then(|rest| rest.strip_prefix(':')) + else { + continue; + }; + if found.is_some_and(|existing| existing != native_name) { + return Err(PyLookupError::new_err(format!( + "{module}:{class} names more than one natively hosted shape" + ))); + } + found = Some(native_name); + } + Ok(found) } -/// `class` from the native module when hosted natively, else from the imported `module`. +/// `class` of the Python module object `module`, or the native class hosting it. pub(crate) fn shape_class<'py, T>( module: &Bound<'py, T>, class: &str, ) -> PyResult> { - if hosted(class) { - return imported(module.py(), NATIVE_MODULE, class); + let module_path: String = module.as_any().getattr("__name__")?.extract()?; + if let Some(native) = hosted_name(&module_path, class)? { + return imported(module.py(), NATIVE_MODULE, native); } module.as_any().getattr(class) } -/// `class` from the native module when hosted natively, else from the module at `module_path`. +/// `class` of the module at `module_path`, or the native class hosting it. pub(crate) fn shape_class_at<'py>( py: Python<'py>, module_path: &str, class: &str, ) -> PyResult> { - if hosted(class) { - return imported(py, NATIVE_MODULE, class); + if let Some(native) = hosted_name(module_path, class)? { + return imported(py, NATIVE_MODULE, native); } imported(py, module_path, class) } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/discovery/declaration_records.rs b/crates/sqlbuild-python/src/bindings/_helpers/discovery/declaration_records.rs index 667afa6b83..126c18bc21 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/discovery/declaration_records.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/discovery/declaration_records.rs @@ -34,7 +34,7 @@ use crate::bindings::models::{NativeDiscoverySession, NativeProjectTree}; const NATIVE_RECORDS_MODULE: &str = "sqlbuild._native"; const MODELS_MODULE: &str = "sqlbuild.compiler.discovery.models"; -const SCOPE_TYPES_MODULE: &str = "sqlbuild.compiler.scopes.types"; +const SCOPE_TYPES_MODULE: &str = "sqlbuild._native"; const AUDITING_MODELS_MODULE: &str = "sqlbuild.compiler.auditing.models"; const AUDITING_TYPES_MODULE: &str = "sqlbuild.compiler.auditing.types"; const CONTRACT_MODELS_MODULE: &str = "sqlbuild.spec.contracts.models"; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/mod.rs b/crates/sqlbuild-python/src/bindings/_helpers/mod.rs index f59238f6b8..962c1d5bad 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/mod.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/mod.rs @@ -18,6 +18,8 @@ pub(crate) mod compile_driver; pub(crate) mod compile_outputs; pub(crate) mod compile_records; pub(crate) mod compiled_project; +pub(crate) mod compiler_enums; +pub(crate) mod compiler_errors; pub(crate) mod compiler_records; pub(crate) mod compute_logs; pub(crate) mod contract_command; @@ -60,6 +62,7 @@ pub(crate) mod planner_identity; pub(crate) mod planner_migrations; pub(crate) mod planner_models; pub(crate) mod planner_phases; +pub(crate) mod planner_records; pub(crate) mod planner_registration; pub(crate) mod planner_resolution; pub(crate) mod planner_scenario; @@ -71,14 +74,17 @@ pub(crate) mod presentation_text; pub(crate) mod project_assembly; pub(crate) mod python_values; pub(crate) mod query_command; +pub(crate) mod record_behaviours; pub(crate) mod records; pub(crate) mod refactor; pub(crate) mod references; pub(crate) mod relation_naming; pub(crate) mod render; pub(crate) mod result_output; +pub(crate) mod rule_records; pub(crate) mod rules; pub(crate) mod scenario_command; +pub(crate) mod scope_records; pub(crate) mod scopes; pub(crate) mod seed_command; pub(crate) mod semantic_checks; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_records/constants.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_records/constants.rs new file mode 100644 index 0000000000..c2f70f51ba --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_records/constants.rs @@ -0,0 +1,151 @@ +//! Layouts of the planner, migration, cursor and fingerprint value records hosted natively. + +use crate::bindings::_helpers::records::models::{FieldDefault, FieldSpec, RecordSpec}; + +const R_PARSED_SELECTOR: RecordSpec = RecordSpec { + name: "ParsedSelector", + doc: "One parsed selector token before graph resolution.", + fields: &[ + FieldSpec::required("kind"), + FieldSpec::required("value"), + FieldSpec::new("upstream", FieldDefault::False), + FieldSpec::new("downstream", FieldDefault::False), + ], +}; +const R_PATH_SELECTOR: RecordSpec = RecordSpec { + name: "PathSelector", + doc: "A directed path selector between two model names with optional endpoint expansion.", + fields: &[ + FieldSpec::required("start_name"), + FieldSpec::required("end_name"), + FieldSpec::new("upstream", FieldDefault::False), + FieldSpec::new("downstream", FieldDefault::False), + ], +}; +const R_SQL_TEST_SELECTION: RecordSpec = RecordSpec { + name: "SqlTestSelection", + doc: "Unit tests selected or excluded by name, beside the tests of the selected resources.", + fields: &[ + FieldSpec::new("names", FieldDefault::EmptyFrozenset), + FieldSpec::new("excluded_names", FieldDefault::EmptyFrozenset), + FieldSpec::new("models_selected", FieldDefault::True), + ], +}; +const R_PLANNER_SCOPE: RecordSpec = RecordSpec { + name: "PlannerScope", + doc: "Resolved graph scope for one planner invocation.", + fields: &[ + FieldSpec::required("upstream_deps"), + FieldSpec::required("downstream_deps"), + FieldSpec::required("all_keys"), + FieldSpec::required("models_by_name"), + FieldSpec::required("selected_keys"), + FieldSpec::required("execution_order"), + FieldSpec::new("user_selected_keys", FieldDefault::EmptyFrozenset), + FieldSpec::new("python_read_source_names", FieldDefault::EmptyFrozenset), + FieldSpec::new( + "sql_test_selection", + FieldDefault::Call("sqlbuild._native", "SqlTestSelection"), + ), + ], +}; +const R_PLANNER_RELATIONS_CONTEXT: RecordSpec = RecordSpec { + name: "PlannerRelationsContext", + doc: "Resolved relation and source inputs for plan entry construction.", + fields: &[ + FieldSpec::required("model_locations"), + FieldSpec::required("seed_locations"), + FieldSpec::required("function_locations"), + FieldSpec::required("source_map"), + FieldSpec::required("source_read_map"), + FieldSpec::required("python_source_read_map"), + FieldSpec::required("source_warehouse_columns"), + FieldSpec::required("star_exclude_keyword"), + FieldSpec::new("listed_source_names", FieldDefault::EmptyFrozenset), + ], +}; +const R_SQL_TEST_ASSERTION_STEP: RecordSpec = RecordSpec { + name: "SqlTestAssertionStep", + doc: "One zero-row assertion step in a SQL-native unit test.", + fields: &[ + FieldSpec::required("name"), + FieldSpec::required("resolved_sql"), + FieldSpec::new("lifted_ctes", FieldDefault::EmptyTuple), + FieldSpec::new("comparison_body_sql", FieldDefault::None), + ], +}; +const R_PLANNER_SELECTION: RecordSpec = RecordSpec { + name: "PlannerSelection", + doc: "User-facing selection inputs for one planner invocation.", + fields: &[ + FieldSpec::new("select", FieldDefault::EmptyTuple), + FieldSpec::new("exclude", FieldDefault::EmptyTuple), + FieldSpec::new("selected_keys", FieldDefault::None), + FieldSpec::new("python_read_source_names", FieldDefault::EmptyFrozenset), + FieldSpec::new( + "sql_test_selection", + FieldDefault::Call("sqlbuild._native", "SqlTestSelection"), + ), + ], +}; +const R_PLANNER_OVERRIDES: RecordSpec = RecordSpec { + name: "PlannerOverrides", + doc: "Explicit user overrides that force or bound planner decisions.", + fields: &[ + FieldSpec::new("cursor_overrides", FieldDefault::None), + FieldSpec::new("full_refresh", FieldDefault::False), + FieldSpec::new("reload_sources", FieldDefault::False), + FieldSpec::new("forced_stale_model_names", FieldDefault::EmptyTuple), + FieldSpec::new("external_blocked_model_names", FieldDefault::EmptyTuple), + FieldSpec::new("max_microbatches", FieldDefault::None), + ], +}; +const R_DEFERRAL_INPUTS: RecordSpec = RecordSpec { + name: "DeferralInputs", + doc: "Deferred relation and source-deferral inputs for planning.", + fields: &[ + FieldSpec::new("deferred_locations", FieldDefault::None), + FieldSpec::new("deferred_relations", FieldDefault::None), + FieldSpec::new("defer_sources_to", FieldDefault::None), + FieldSpec::new("source_deferral_enabled", FieldDefault::True), + ], +}; +const R_PLANNER_POLICIES: RecordSpec = RecordSpec { + name: "PlannerPolicies", + doc: "Behavior policies selected by the caller for one planner invocation.", + fields: &[ + FieldSpec::new("auto_load_sources", FieldDefault::False), + FieldSpec::new("selection_diagnostics", FieldDefault::False), + FieldSpec::new("plan_sql_tests", FieldDefault::True), + FieldSpec::new("record_migration_fingerprints", FieldDefault::True), + ], +}; +const R_DIRECT_SOURCE_FRESHNESS_PLANNING_RESULT: RecordSpec = RecordSpec { + name: "DirectSourceFreshnessPlanningResult", + doc: "Direct planning-time source freshness observations and comparisons.", + fields: &[ + FieldSpec::new("observed_records", FieldDefault::EmptyTuple), + FieldSpec::new("previous_records", FieldDefault::EmptyTuple), + FieldSpec::new("changed_identities", FieldDefault::EmptyFrozenset), + FieldSpec::new("unchanged_identities", FieldDefault::EmptyFrozenset), + FieldSpec::new("unknown_source_names", FieldDefault::EmptyTuple), + FieldSpec::new("unknown_sources", FieldDefault::EmptyDict), + FieldSpec::new("age_statuses", FieldDefault::EmptyDict), + FieldSpec::new("propagation", FieldDefault::None), + ], +}; + +/// The planner, migration, cursor and fingerprint value records. +pub(crate) const PLANNER_RECORDS: &[RecordSpec] = &[ + R_PARSED_SELECTOR, + R_PATH_SELECTOR, + R_SQL_TEST_SELECTION, + R_PLANNER_SCOPE, + R_PLANNER_RELATIONS_CONTEXT, + R_SQL_TEST_ASSERTION_STEP, + R_PLANNER_SELECTION, + R_PLANNER_OVERRIDES, + R_DEFERRAL_INPUTS, + R_PLANNER_POLICIES, + R_DIRECT_SOURCE_FRESHNESS_PLANNING_RESULT, +]; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_records/mod.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_records/mod.rs new file mode 100644 index 0000000000..61d449b063 --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_records/mod.rs @@ -0,0 +1 @@ +pub(crate) mod constants; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/record_behaviours/constants.rs b/crates/sqlbuild-python/src/bindings/_helpers/record_behaviours/constants.rs new file mode 100644 index 0000000000..f449aa3b85 --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/record_behaviours/constants.rs @@ -0,0 +1,19 @@ +//! Native behaviour of the hosted shapes: computed attributes, methods, ordering and validation. + +use crate::bindings::_helpers::record_behaviours::scopes::{ + completeness_complete, target_query_unknown, +}; +use crate::bindings::_helpers::records::types::{ClassMethod, ClassProperty, RecordFinish}; + +/// Compiler records whose bound values are normalized or validated natively. +pub(crate) const BEHAVIOUR_FINISHERS: &[(&str, RecordFinish)] = &[]; +/// Computed attributes of compiler shapes. +pub(crate) const BEHAVIOUR_PROPERTIES: &[(&str, &str, ClassProperty)] = &[ + ("ScopeCompleteness", "complete", completeness_complete), + ("ScopeTargetQuery", "unknown", target_query_unknown), +]; +/// Methods of compiler shapes. +pub(crate) const BEHAVIOUR_METHODS: &[(&str, &str, ClassMethod)] = &[]; +/// Compiler records ordered by their compared fields. +pub(crate) const BEHAVIOUR_ORDERED: &[&str] = + &["ResourceIdentity", "DeclarationIdentity", "OwnershipRoot"]; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/record_behaviours/mod.rs b/crates/sqlbuild-python/src/bindings/_helpers/record_behaviours/mod.rs new file mode 100644 index 0000000000..39ca7ba2c2 --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/record_behaviours/mod.rs @@ -0,0 +1,2 @@ +pub(crate) mod constants; +pub(crate) mod scopes; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/record_behaviours/scopes.rs b/crates/sqlbuild-python/src/bindings/_helpers/record_behaviours/scopes.rs new file mode 100644 index 0000000000..9634a620d9 --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/record_behaviours/scopes.rs @@ -0,0 +1,30 @@ +//! Computed attributes of the natively hosted scope records. + +use pyo3::IntoPyObjectExt; +use pyo3::prelude::{Bound, Py, PyAny, PyAnyMethods, PyResult}; + +/// The completeness sections of a `ScopeCompleteness`. +const COMPLETENESS_SECTIONS: [&str; 6] = [ + "discovery", + "static_visibility", + "runtime_usage", + "relationships", + "placement", + "promotion_impact", +]; + +/// `ScopeCompleteness.complete`: whether every index section is complete. +pub(crate) fn completeness_complete(instance: &Bound<'_, PyAny>) -> PyResult> { + let mut complete: bool = true; + for section in COMPLETENESS_SECTIONS { + complete = complete && instance.getattr(section)?.is_truthy()?; + } + complete.into_py_any(instance.py()) +} + +/// `ScopeTargetQuery.unknown`: whether the target is absent rather than merely inaccessible. +pub(crate) fn target_query_unknown(instance: &Bound<'_, PyAny>) -> PyResult> { + let unknown: bool = !instance.getattr("matches")?.is_truthy()? + && !instance.getattr("declaration_matches")?.is_truthy()?; + unknown.into_py_any(instance.py()) +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/records/constants.rs b/crates/sqlbuild-python/src/bindings/_helpers/records/constants.rs index f86c402d4d..7e0a8c2d0e 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/records/constants.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/records/constants.rs @@ -1,14 +1,20 @@ //! The layouts of every native record class. use crate::bindings::_helpers::compile_records::constants::COMPILE_VALUE_RECORDS; -use crate::bindings::_helpers::compiler_records::constants::{ - COMPILER_ENUMS, COMPILER_ERRORS, COMPILER_RECORDS, -}; +use crate::bindings::_helpers::compiler_enums::constants::COMPILER_ENUMS; +use crate::bindings::_helpers::compiler_errors::constants::COMPILER_ERRORS; +use crate::bindings::_helpers::compiler_records::constants::COMPILER_RECORDS; use crate::bindings::_helpers::freshness::observation::normalize_observation; +use crate::bindings::_helpers::planner_records::constants::PLANNER_RECORDS; +use crate::bindings::_helpers::record_behaviours::constants::{ + BEHAVIOUR_FINISHERS, BEHAVIOUR_METHODS, BEHAVIOUR_ORDERED, BEHAVIOUR_PROPERTIES, +}; use crate::bindings::_helpers::records::models::{ ClassValue, EnumSpec, ErrorSpec, FieldDefault, FieldSpec, RecordSpec, }; -use crate::bindings::_helpers::records::types::RecordFinish; +use crate::bindings::_helpers::records::types::{ClassMethod, ClassProperty, RecordFinish}; +use crate::bindings::_helpers::rule_records::constants::RULE_RECORDS; +use crate::bindings::_helpers::scope_records::constants::SCOPE_RECORDS; pub(crate) const NATIVE_MODULE: &str = "sqlbuild._native"; /// The class attribute holding a record class's index into `RECORDS`. @@ -137,11 +143,22 @@ pub(crate) const RECORDS: &[&[RecordSpec]] = &[ DIAGNOSTIC_RECORDS, COMPILER_RECORDS, COMPILE_VALUE_RECORDS, + SCOPE_RECORDS, + PLANNER_RECORDS, + RULE_RECORDS, ]; -/// Records whose bound values are normalized before the record is created. -pub(crate) const RECORD_FINISHERS: &[(&str, RecordFinish)] = - &[("SourceFreshnessObservation", normalize_observation)]; +/// Records whose bound values are normalized or validated before the record is created. +pub(crate) const RECORD_FINISHERS: &[&[(&str, RecordFinish)]] = &[ + &[("SourceFreshnessObservation", normalize_observation)], + BEHAVIOUR_FINISHERS, +]; +/// `(class, attribute, getter)` computed attributes set on native classes after registration. +pub(crate) const CLASS_PROPERTIES: &[&[(&str, &str, ClassProperty)]] = &[BEHAVIOUR_PROPERTIES]; +/// `(class, attribute, method)` methods set on native classes after registration. +pub(crate) const CLASS_METHODS: &[&[(&str, &str, ClassMethod)]] = &[BEHAVIOUR_METHODS]; +/// Records ordered by their compared fields, as `order=True` dataclasses are. +pub(crate) const ORDERED_RECORDS: &[&[&str]] = &[BEHAVIOUR_ORDERED]; /// `SourceFreshnessObservation` field positions. pub(crate) const DATA_VERSION_FIELD: usize = 2; pub(crate) const OBSERVED_AT_FIELD: usize = 4; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/records/fields.rs b/crates/sqlbuild-python/src/bindings/_helpers/records/fields.rs index def82d50c4..5126891bda 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/records/fields.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/records/fields.rs @@ -3,6 +3,7 @@ use pyo3::IntoPyObjectExt; use pyo3::exceptions::{PyAttributeError, PyTypeError}; use pyo3::prelude::{Bound, Py, PyAny, PyAnyMethods, PyErr, PyResult, Python}; +use pyo3::pyclass::CompareOp; use pyo3::types::{ PyBool, PyDict, PyDictMethods, PyFrozenSet, PyString, PyTuple, PyTupleMethods, PyType, PyTypeMethods, @@ -10,7 +11,7 @@ use pyo3::types::{ use crate::bindings::_helpers::compile_driver::python_calls::imported; use crate::bindings::_helpers::records::constants::{ - FROZEN_INSTANCE_ERROR, RECORD_FINISHERS, RECORD_SPEC_ATTRIBUTE, RECORDS, + FROZEN_INSTANCE_ERROR, ORDERED_RECORDS, RECORD_FINISHERS, RECORD_SPEC_ATTRIBUTE, RECORDS, }; use crate::bindings::_helpers::records::models::{FieldDefault, NativeRecord, RecordSpec}; @@ -81,7 +82,11 @@ pub(crate) fn build_values( }; values.push(value.unbind()); } - match RECORD_FINISHERS.iter().find(|(record, _)| *record == name) { + match RECORD_FINISHERS + .iter() + .flat_map(|group| group.iter()) + .find(|(record, _)| *record == name) + { Some((_, finish)) => finish(py, values), None => Ok(values), } @@ -102,6 +107,7 @@ fn default_value<'py>( FieldDefault::True => Ok(PyBool::new(py, true).to_owned().into_any()), FieldDefault::Int(value) => value.into_bound_py_any(py), FieldDefault::Text(value) => Ok(PyString::new(py, value).into_any()), + FieldDefault::TextTuple(texts) => Ok(PyTuple::new(py, texts)?.into_any()), FieldDefault::EmptyTuple => Ok(PyTuple::empty(py).into_any()), FieldDefault::EmptyDict => Ok(PyDict::new(py).into_any()), FieldDefault::EmptyFrozenset => Ok(PyFrozenSet::empty(py)?.into_any()), @@ -150,3 +156,22 @@ pub(crate) fn frozen_error(py: Python<'_>, name: &str) -> PyErr { Err(_) => PyAttributeError::new_err(message), } } + +/// `slf other` over the compared fields, as an `order=True` dataclass compares. +pub(crate) fn ordered_comparison( + slf: &Bound<'_, NativeRecord>, + other: &Bound<'_, PyAny>, + op: CompareOp, +) -> PyResult> { + let py: Python<'_> = slf.py(); + let name: &str = record_spec(&slf.get_type())?.name; + let ordered: bool = ORDERED_RECORDS.iter().any(|group| group.contains(&name)); + if !ordered || !other.get_type().is(slf.get_type()) { + return Ok(py.NotImplemented()); + } + let other: &Bound<'_, NativeRecord> = other.cast::()?; + compared_values(slf)? + .as_any() + .rich_compare(compared_values(other)?, op)? + .into_py_any(py) +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/records/models.rs b/crates/sqlbuild-python/src/bindings/_helpers/records/models.rs index 4d5df04eb9..59a6bb37f1 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/records/models.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/records/models.rs @@ -2,12 +2,14 @@ use pyo3::exceptions::PyAttributeError; use pyo3::prelude::{Bound, Py, PyAny, PyAnyMethods, PyResult, Python}; +use pyo3::pyclass::CompareOp; use pyo3::types::{PyDict, PyTuple, PyType}; use pyo3::{IntoPyObjectExt, pyclass, pymethods}; use crate::bindings::_helpers::records::fields::{ - build_values, compared_values, frozen_error, record_spec, repr_text, + build_values, compared_values, frozen_error, ordered_comparison, record_spec, repr_text, }; +use crate::bindings::_helpers::records::types::{ClassMethod, ClassProperty}; /// How a record field is filled when the caller does not pass it. #[derive(Debug, Clone, Copy)] @@ -18,6 +20,8 @@ pub(crate) enum FieldDefault { True, Int(i64), Text(&'static str), + /// A tuple of texts. + TextTuple(&'static [&'static str]), EmptyTuple, EmptyDict, EmptyFrozenset, @@ -107,6 +111,22 @@ impl NativeRecord { .into_py_any(py) } + fn __lt__(slf: &Bound<'_, Self>, other: &Bound<'_, PyAny>) -> PyResult> { + ordered_comparison(slf, other, CompareOp::Lt) + } + + fn __le__(slf: &Bound<'_, Self>, other: &Bound<'_, PyAny>) -> PyResult> { + ordered_comparison(slf, other, CompareOp::Le) + } + + fn __gt__(slf: &Bound<'_, Self>, other: &Bound<'_, PyAny>) -> PyResult> { + ordered_comparison(slf, other, CompareOp::Gt) + } + + fn __ge__(slf: &Bound<'_, Self>, other: &Bound<'_, PyAny>) -> PyResult> { + ordered_comparison(slf, other, CompareOp::Ge) + } + fn __hash__(slf: &Bound<'_, Self>) -> PyResult { compared_values(slf)?.as_any().hash() } @@ -122,6 +142,78 @@ impl NativeRecord { } } +/// A computed attribute of a native class, evaluated by its native function. +#[derive(Debug)] +#[pyclass(module = "sqlbuild._native", frozen)] +pub(crate) struct NativeProperty { + pub(crate) property: ClassProperty, + pub(crate) name: &'static str, +} + +#[pymethods] +impl NativeProperty { + fn __get__( + slf: &Bound<'_, Self>, + instance: &Bound<'_, PyAny>, + _owner: &Bound<'_, PyAny>, + ) -> PyResult> { + if instance.is_none() { + return Ok(slf.clone().into_any().unbind()); + } + (slf.get().property)(instance) + } + + fn __set__(&self, instance: &Bound<'_, PyAny>, _value: &Bound<'_, PyAny>) -> PyResult<()> { + Err(frozen_error(instance.py(), self.name)) + } +} + +/// A method of a native class, run by its native function. +#[derive(Debug)] +#[pyclass(module = "sqlbuild._native", frozen)] +pub(crate) struct NativeMethod { + pub(crate) method: ClassMethod, +} + +#[pymethods] +impl NativeMethod { + fn __get__( + slf: &Bound<'_, Self>, + instance: &Bound<'_, PyAny>, + owner: &Bound<'_, PyAny>, + ) -> PyResult> { + let receiver: &Bound<'_, PyAny> = if instance.is_none() { owner } else { instance }; + Ok(Py::new( + slf.py(), + BoundNativeMethod { + method: slf.get().method, + receiver: receiver.clone().unbind(), + }, + )? + .into_any()) + } +} + +/// A `NativeMethod` bound to the instance or class it was read from. +#[derive(Debug)] +#[pyclass(module = "sqlbuild._native", frozen)] +pub(crate) struct BoundNativeMethod { + method: ClassMethod, + receiver: Py, +} + +#[pymethods] +impl BoundNativeMethod { + #[pyo3(signature = (*args, **kwargs))] + fn __call__( + &self, + args: &Bound<'_, PyTuple>, + kwargs: Option<&Bound<'_, PyDict>>, + ) -> PyResult> { + (self.method)(self.receiver.bind(args.py()), args, kwargs) + } +} + /// Read-only access to one field of a native record. #[derive(Debug)] #[pyclass(module = "sqlbuild._native", frozen)] diff --git a/crates/sqlbuild-python/src/bindings/_helpers/records/registration.rs b/crates/sqlbuild-python/src/bindings/_helpers/records/registration.rs index 3705508d4e..49cfdbaf7e 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/records/registration.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/records/registration.rs @@ -5,22 +5,26 @@ use pyo3::prelude::{Bound, Py, PyAny, PyAnyMethods, PyModule, PyModuleMethods, P use pyo3::types::{PyDict, PyDictMethods, PyString, PyTuple}; use crate::bindings::_helpers::records::constants::{ - BUILTIN_PARENT_PREFIX, BUILTINS_MODULE, DATACLASS_FIELD_KIND, DATACLASS_FIELDS_ATTRIBUTE, - DATACLASSES_MODULE, ENUM_MODULE, ENUMS, ERROR_SPEC_ATTRIBUTE, ERRORS, EXCEPTION_PARENT, - NATIVE_MODULE, RECORD_SPEC_ATTRIBUTE, STR_ENUM, VALUE_ERROR_PARENT, + BUILTIN_PARENT_PREFIX, BUILTINS_MODULE, CLASS_METHODS, CLASS_PROPERTIES, DATACLASS_FIELD_KIND, + DATACLASS_FIELDS_ATTRIBUTE, DATACLASSES_MODULE, ENUM_MODULE, ENUMS, ERROR_SPEC_ATTRIBUTE, + ERRORS, EXCEPTION_PARENT, NATIVE_MODULE, RECORD_SPEC_ATTRIBUTE, STR_ENUM, VALUE_ERROR_PARENT, }; use crate::bindings::_helpers::records::error_classes::{ BoundErrorInitializer, ErrorInitializer, class_value, }; use crate::bindings::_helpers::records::fields::all_specs; use crate::bindings::_helpers::records::models::{ - EnumSpec, ErrorSpec, NativeRecord, RecordAttribute, RecordSpec, + BoundNativeMethod, EnumSpec, ErrorSpec, NativeMethod, NativeProperty, NativeRecord, + RecordAttribute, RecordSpec, }; pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { let py: Python<'_> = module.py(); module.add_class::()?; module.add_class::()?; + module.add_class::()?; + module.add_class::()?; + module.add_class::()?; let base: Bound<'_, PyAny> = py.get_type::().into_any(); let metaclass: Bound<'_, PyAny> = base.get_type().into_any(); let str_enum: Bound<'_, PyAny> = py.import(ENUM_MODULE)?.getattr(STR_ENUM)?; @@ -49,6 +53,20 @@ pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { )?; module.add(spec.name, class)?; } + for (class, name, property) in CLASS_PROPERTIES.iter().flat_map(|group| group.iter()) { + let descriptor: Py = Py::new( + py, + NativeProperty { + property: *property, + name, + }, + )?; + module.getattr(*class)?.setattr(*name, descriptor)?; + } + for (class, name, method) in CLASS_METHODS.iter().flat_map(|group| group.iter()) { + let descriptor: Py = Py::new(py, NativeMethod { method: *method })?; + module.getattr(*class)?.setattr(*name, descriptor)?; + } Ok(()) } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/records/types.rs b/crates/sqlbuild-python/src/bindings/_helpers/records/types.rs index 4b90f1cc6a..afd480ce0d 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/records/types.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/records/types.rs @@ -1,6 +1,14 @@ //! Hooks applied while building native records. -use pyo3::prelude::{Py, PyAny, PyResult, Python}; +use pyo3::prelude::{Bound, Py, PyAny, PyResult, Python}; +use pyo3::types::{PyDict, PyTuple}; /// Normalizes the bound field values of one record before it is created. pub(crate) type RecordFinish = fn(Python<'_>, Vec>) -> PyResult>>; + +/// Computes one read-only attribute of a native class instance. +pub(crate) type ClassProperty = fn(&Bound<'_, PyAny>) -> PyResult>; + +/// Runs one method of a native class with its receiver (the instance, or the class when unbound). +pub(crate) type ClassMethod = + fn(&Bound<'_, PyAny>, &Bound<'_, PyTuple>, Option<&Bound<'_, PyDict>>) -> PyResult>; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/refactor/refactor_facts.rs b/crates/sqlbuild-python/src/bindings/_helpers/refactor/refactor_facts.rs index 205b8f1d28..4bc156e557 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/refactor/refactor_facts.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/refactor/refactor_facts.rs @@ -17,8 +17,8 @@ use crate::bindings::_helpers::scopes::scope_validation::scope_relocated_declara const NATIVE_MODULE: &str = "sqlbuild._native"; const CONFIG_MODULE: &str = "sqlbuild._native"; -const SCOPE_MODELS: &str = "sqlbuild.compiler.scopes.models"; -const SCOPE_TYPES: &str = "sqlbuild.compiler.scopes.types"; +const SCOPE_MODELS: &str = "sqlbuild._native"; +const SCOPE_TYPES: &str = "sqlbuild._native"; const GENERIC_DIALECT: &str = "generic"; const MATERIALIZED_KEY: &str = "materialized"; const MIGRATE_FROM_KEY: &str = "migrate_from"; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/render/authored_sql.rs b/crates/sqlbuild-python/src/bindings/_helpers/render/authored_sql.rs index fcfa29b3a3..fcba3cb7c2 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/render/authored_sql.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/render/authored_sql.rs @@ -24,7 +24,7 @@ const NATIVE_RECORDS_MODULE: &str = "sqlbuild._native"; const NATIVE_MODULE: &str = "sqlbuild._native"; const COMPILE_MODELS_MODULE: &str = "sqlbuild.compiler.compile.models"; const COMPILE_ERRORS_MODULE: &str = "sqlbuild.compiler.compile.exceptions"; -const SCOPE_MODELS_MODULE: &str = "sqlbuild.compiler.scopes.models"; +const SCOPE_MODELS_MODULE: &str = "sqlbuild._native"; const FUNCTOOLS_MODULE: &str = "functools"; const DECLARATION_FIELDS: [&str; 4] = [ "enums", diff --git a/crates/sqlbuild-python/src/bindings/_helpers/render/declaration_values.rs b/crates/sqlbuild-python/src/bindings/_helpers/render/declaration_values.rs index 1d40296f1f..c5d56ad028 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/render/declaration_values.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/render/declaration_values.rs @@ -13,12 +13,11 @@ const NATIVE_RECORDS_MODULE: &str = "sqlbuild._native"; const COMPILE_MODELS_MODULE: &str = "sqlbuild.compiler.compile.models"; const COMPILE_ERRORS_MODULE: &str = "sqlbuild.compiler.compile.exceptions"; -const SCOPE_MODELS_MODULE: &str = "sqlbuild.compiler.scopes.models"; -const SCOPE_TYPES_MODULE: &str = "sqlbuild.compiler.scopes.types"; +const SCOPE_MODELS_MODULE: &str = "sqlbuild._native"; +const SCOPE_TYPES_MODULE: &str = "sqlbuild._native"; const NATIVE_MODULE: &str = "sqlbuild._native"; const DISCOVERY_MODELS_MODULE: &str = "sqlbuild.compiler.discovery.models"; const SCOPE_ERRORS_MODULE: &str = "sqlbuild._native"; -const BUILTINS_MODULE: &str = "builtins"; const NO_DECLARATION_KIND: u8 = 255; /// Declaration kinds by their native code: enum 0, constant 1, macro 2. const DECLARATION_KINDS: [&str; 3] = ["ENUM", "CONSTANT", "MACRO"]; @@ -255,10 +254,6 @@ pub(crate) fn native_declaration_contexts<'py>( "resolution_context", imported(py, COMPILE_MODELS_MODULE, "DeclarationResolutionContext")?, )?; - classes.set_item( - "allocate", - imported(py, BUILTINS_MODULE, "object")?.getattr("__new__")?, - )?; classes.set_item("reasons", reasons)?; let keys: Bound<'py, PyDict> = PyDict::new(py); keys.set_item("identity_keys", identity_keys)?; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/render/macro_expansion_result.rs b/crates/sqlbuild-python/src/bindings/_helpers/render/macro_expansion_result.rs index 957e496407..005ee28870 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/render/macro_expansion_result.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/render/macro_expansion_result.rs @@ -22,8 +22,8 @@ const NATIVE_MODULE: &str = "sqlbuild._native"; const COMPILE_MODELS_MODULE: &str = "sqlbuild.compiler.compile.models"; const COMPILE_EXCEPTIONS_MODULE: &str = "sqlbuild.compiler.compile.exceptions"; const FRONTIER_EXCEPTIONS_MODULE: &str = "sqlbuild._native"; -const SCOPES_MODELS_MODULE: &str = "sqlbuild.compiler.scopes.models"; -const SCOPES_TYPES_MODULE: &str = "sqlbuild.compiler.scopes.types"; +const SCOPES_MODELS_MODULE: &str = "sqlbuild._native"; +const SCOPES_TYPES_MODULE: &str = "sqlbuild._native"; const REF_MODELS_MODULE: &str = "sqlbuild.python_nodes.models"; const REF_TYPES_MODULE: &str = "sqlbuild.python_nodes.types"; const TYPED_REFERENCE_FUNCTIONS: [(&str, &str); 3] = [ diff --git a/crates/sqlbuild-python/src/bindings/_helpers/rule_records/constants.rs b/crates/sqlbuild-python/src/bindings/_helpers/rule_records/constants.rs new file mode 100644 index 0000000000..9ba81b59be --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/rule_records/constants.rs @@ -0,0 +1,177 @@ +//! Layouts of the rule engine value records hosted natively. + +use crate::bindings::_helpers::records::models::{FieldDefault, FieldSpec, RecordSpec}; + +const R_RULE_GUIDANCE: RecordSpec = RecordSpec { + name: "RuleGuidance", + doc: "Structured authoring guidance shared by diagnostics and generated documentation.", + fields: &[ + FieldSpec::required("good_example"), + FieldSpec::required("anti_tautology"), + FieldSpec::required("mutation_check"), + ], +}; +const R_RULE_EXEMPTION: RecordSpec = RecordSpec { + name: "RuleExemption", + doc: "", + fields: &[ + FieldSpec::required("rule"), + FieldSpec::required("path"), + FieldSpec::required("reason"), + ], +}; +const R_GRAPH_EDGE_EXCLUSION: RecordSpec = RecordSpec { + name: "GraphEdgeExclusion", + doc: "One exact, temporary internal-layer dependency exception.", + fields: &[ + FieldSpec::required("consumer"), + FieldSpec::required("dependency"), + FieldSpec::required("reason"), + ], +}; +const R_RULE_IGNORE: RecordSpec = RecordSpec { + name: "RuleIgnore", + doc: "", + fields: &[ + FieldSpec::required("rules"), + FieldSpec::required("paths"), + FieldSpec::required("reason"), + FieldSpec::new("selectors", FieldDefault::EmptyTuple), + ], +}; +const R_SELECT_STAR_ALLOW: RecordSpec = RecordSpec { + name: "SelectStarAllow", + doc: "", + fields: &[FieldSpec::required("paths"), FieldSpec::required("reason")], +}; +const R_THRESHOLD_OVERRIDE: RecordSpec = RecordSpec { + name: "ThresholdOverride", + doc: "", + fields: &[ + FieldSpec::required("paths"), + FieldSpec::required("thresholds"), + FieldSpec::required("reason"), + ], +}; +const R_RULES_CACHE_CONFIG: RecordSpec = RecordSpec { + name: "RulesCacheConfig", + doc: "", + fields: &[FieldSpec::new("enabled", FieldDefault::True)], +}; +const R_SQL_TEST_RULES_CONFIG: RecordSpec = RecordSpec { + name: "SqlTestRulesConfig", + doc: "", + fields: &[FieldSpec::new( + "pipeline_directory", + FieldDefault::Text("pipelines"), + )], +}; +const R_PREPARED_SQL_LINT_RESULT: RecordSpec = RecordSpec { + name: "PreparedSqlLintResult", + doc: "Reusable expansion context and completed preliminary SQL checks.", + fields: &[ + FieldSpec::required("result"), + FieldSpec::required("context"), + ], +}; +const R_SQL_EXPANSION_REUSE: RecordSpec = RecordSpec { + name: "SqlExpansionReuse", + doc: "Compiler declaration scope for post-assembly SQL checks; assembly still runs its macros.", + fields: &[ + FieldSpec::required("discovered_inputs"), + FieldSpec::required("declaration_scope"), + ], +}; +const R_PREPARED_SQL_LINT: RecordSpec = RecordSpec { + name: "PreparedSqlLint", + doc: "Process-local SQL checks started after compiler expansion has completed.", + fields: &[ + FieldSpec::required("codes"), + FieldSpec::required("future"), + FieldSpec::new("project_dir", FieldDefault::None), + ], +}; +const R_SQL_RULES_EVALUATION: RecordSpec = RecordSpec { + name: "SqlRulesEvaluation", + doc: "SQL rule findings of one run, their cache accounting and elapsed time.", + fields: &[ + FieldSpec::required("findings"), + FieldSpec::required("cache_hits"), + FieldSpec::required("cache_misses"), + FieldSpec::new("sql_ms", FieldDefault::Int(0)), + ], +}; +const R_RULE_RUN_PLAN: RecordSpec = RecordSpec { + name: "RuleRunPlan", + doc: "The catalogue, selected rules, project slice and settings one rules run evaluates.", + fields: &[ + FieldSpec::required("effective_config"), + FieldSpec::required("project_dir"), + FieldSpec::required("catalogue"), + FieldSpec::required("custom_payloads"), + FieldSpec::required("selected"), + FieldSpec::required("skipped_type_proof_rules"), + FieldSpec::required("model_paths"), + FieldSpec::required("project"), + FieldSpec::required("config"), + ], +}; +const R_RULES_RUN_RESULT: RecordSpec = RecordSpec { + name: "RulesRunResult", + doc: "Unified findings and phase measurements for configured rules.", + fields: &[ + FieldSpec::required("findings"), + FieldSpec::required("evaluated_models"), + FieldSpec::required("built_in_ms"), + FieldSpec::required("custom_ms"), + FieldSpec::new("custom_cpu_ms", FieldDefault::Int(0)), + FieldSpec::new("unevaluated_resources", FieldDefault::Int(0)), + FieldSpec::new("cache_hits", FieldDefault::Int(0)), + FieldSpec::new("cache_misses", FieldDefault::Int(0)), + FieldSpec::new("skipped_type_proof_rules", FieldDefault::EmptyTuple), + ], +}; +const R_NATIVE_RULES_EVALUATION: RecordSpec = RecordSpec { + name: "NativeRulesEvaluation", + doc: "The built-in rules response, decoded natively.", + fields: &[ + FieldSpec::required("findings"), + FieldSpec::required("selected_codes"), + FieldSpec::required("evaluated_models"), + FieldSpec::required("cache_hits"), + FieldSpec::required("cache_misses"), + FieldSpec::required("built_in_ms"), + FieldSpec::required("reused"), + ], +}; +const R_CUSTOM_RULES_OUTCOME: RecordSpec = RecordSpec { + name: "CustomRulesOutcome", + doc: "Custom-rule findings in canonical order and their cache accounting.", + fields: &[ + FieldSpec::required("findings"), + FieldSpec::required("cache_hits"), + FieldSpec::required("cache_misses"), + FieldSpec::required("custom_ms"), + FieldSpec::new("custom_cpu_ms", FieldDefault::Int(0)), + ], +}; + +/// The rule engine value records. +pub(crate) const RULE_RECORDS: &[RecordSpec] = &[ + R_RULE_GUIDANCE, + R_RULE_EXEMPTION, + R_GRAPH_EDGE_EXCLUSION, + R_RULE_IGNORE, + R_SELECT_STAR_ALLOW, + R_THRESHOLD_OVERRIDE, + R_RULES_CACHE_CONFIG, + R_SQL_TEST_RULES_CONFIG, + R_PREPARED_SQL_LINT_RESULT, + R_SQL_EXPANSION_REUSE, + R_PREPARED_SQL_LINT, + R_SQL_RULES_EVALUATION, + R_RULE_RUN_PLAN, + R_RULES_RUN_RESULT, + R_NATIVE_RULES_EVALUATION, + R_CUSTOM_RULES_OUTCOME, +]; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/rule_records/mod.rs b/crates/sqlbuild-python/src/bindings/_helpers/rule_records/mod.rs new file mode 100644 index 0000000000..61d449b063 --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/rule_records/mod.rs @@ -0,0 +1 @@ +pub(crate) mod constants; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/rules/custom_evaluation.rs b/crates/sqlbuild-python/src/bindings/_helpers/rules/custom_evaluation.rs index 76232eeaa2..b436ff7265 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/rules/custom_evaluation.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/rules/custom_evaluation.rs @@ -31,7 +31,7 @@ const RULES_CONSTANTS_MODULE: &str = "sqlbuild.rule_engine.constants"; const RULES_ERRORS_MODULE: &str = "sqlbuild.rule_engine.exceptions"; const RULE_CONTEXT_MODULE: &str = "sqlbuild.rule_engine.classes.rule_context"; const FACT_REPLAY_MODULE: &str = "sqlbuild.rule_engine._helpers.engine.fact_replay"; -const SCOPE_MODELS_MODULE: &str = "sqlbuild.compiler.scopes.models"; +const SCOPE_MODELS_MODULE: &str = "sqlbuild._native"; const PROJECT_SUBJECT: &str = "project"; const MODEL_SUBJECT: &str = "model"; /// Project fields no rule fact reads, cleared before digests and host payloads use the project. diff --git a/crates/sqlbuild-python/src/bindings/_helpers/rules/lint_text.rs b/crates/sqlbuild-python/src/bindings/_helpers/rules/lint_text.rs index a748bb5224..ccc75757da 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/rules/lint_text.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/rules/lint_text.rs @@ -28,7 +28,7 @@ use crate::bindings::types::CompilerDetach; const LINT_MODELS_MODULE: &str = "sqlbuild._native"; const LINT_TYPES_MODULE: &str = "sqlbuild._native"; -const SCOPE_TYPES_MODULE: &str = "sqlbuild.compiler.scopes.types"; +const SCOPE_TYPES_MODULE: &str = "sqlbuild._native"; const NATIVE_ENGINE: &str = "native"; const FORMAT_UNSAFE_CODE: &str = "format-unsafe"; const FORMAT_UNSAFE_REMEDIATION: &str = "Rewrite the unsupported SQL shape or report a minimal formatter reproduction. No header, SQL body, or fixture changes were written to this file."; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/scope_records/constants.rs b/crates/sqlbuild-python/src/bindings/_helpers/scope_records/constants.rs new file mode 100644 index 0000000000..8cdc552555 --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/scope_records/constants.rs @@ -0,0 +1,450 @@ +//! Layouts of the declaration scope value records hosted natively. + +use crate::bindings::_helpers::records::models::{FieldDefault, FieldSpec, RecordSpec}; + +const R_RESOURCE_IDENTITY: RecordSpec = RecordSpec { + name: "ResourceIdentity", + doc: "Stable kind-qualified identity of one authored resource.", + fields: &[FieldSpec::required("kind"), FieldSpec::required("name")], +}; +const R_DECLARATION_IDENTITY: RecordSpec = RecordSpec { + name: "DeclarationIdentity", + doc: "Stable declaration identity, including a private declaration owner.", + fields: &[ + FieldSpec::required("kind"), + FieldSpec::required("name"), + FieldSpec::new("owner", FieldDefault::None), + ], +}; +const R_OWNERSHIP_ROOT: RecordSpec = RecordSpec { + name: "OwnershipRoot", + doc: "Normalized project-relative directory that owns resource scope.", + fields: &[ + FieldSpec::required("path"), + FieldSpec::new( + "kind", + FieldDefault::Member("sqlbuild._native", "OwnershipRootKind", "RESOURCE"), + ), + FieldSpec::new("resource_kind", FieldDefault::None), + ], +}; +const R_RESOURCE_RECORD: RecordSpec = RecordSpec { + name: "ResourceRecord", + doc: "One resource participating in lexical declaration scope.", + fields: &[ + FieldSpec::required("identity"), + FieldSpec::required("path"), + FieldSpec::required("ownership_root"), + ], +}; +const R_MACRO_METADATA: RecordSpec = RecordSpec { + name: "MacroMetadata", + doc: "Safe macro metadata required by scope and dependency inspection.", + fields: &[ + FieldSpec::new("parameters", FieldDefault::EmptyTuple), + FieldSpec::new("dependencies", FieldDefault::EmptyTuple), + FieldSpec::new("source_digest", FieldDefault::Text("")), + ], +}; +const R_ENUM_MEMBER_METADATA: RecordSpec = RecordSpec { + name: "EnumMemberMetadata", + doc: "One value-free authored enum member identity.", + fields: &[FieldSpec::required("name")], +}; +const R_ENUM_METADATA: RecordSpec = RecordSpec { + name: "EnumMetadata", + doc: "Safe enum metadata containing member names and their shared scalar type.", + fields: &[ + FieldSpec::required("members"), + FieldSpec::required("scalar_type"), + ], +}; +const R_CONSTANT_METADATA: RecordSpec = RecordSpec { + name: "ConstantMetadata", + doc: "Value-free normalized metadata for one typed constant.", + fields: &[ + FieldSpec::required("logical_type"), + FieldSpec::new("collection_kind", FieldDefault::None), + FieldSpec::new("item_count", FieldDefault::None), + FieldSpec::new("nullable", FieldDefault::False), + FieldSpec::new("render_as", FieldDefault::None), + ], +}; +const R_DECLARATION_RECORD: RecordSpec = RecordSpec { + name: "DeclarationRecord", + doc: "One declaration definition and its compiler-owned scope metadata.", + fields: &[ + FieldSpec::required("identity"), + FieldSpec::required("path"), + FieldSpec::required("line"), + FieldSpec::required("column"), + FieldSpec::required("scope"), + FieldSpec::required("ownership_root"), + FieldSpec::new("owning_path", FieldDefault::None), + FieldSpec::new("macro", FieldDefault::None), + FieldSpec::new("enum", FieldDefault::None), + FieldSpec::new("constant", FieldDefault::None), + ], +}; +const R_USAGE_RECORD: RecordSpec = RecordSpec { + name: "UsageRecord", + doc: "A runtime or declaration dependency edge.", + fields: &[ + FieldSpec::required("consumer"), + FieldSpec::required("declaration"), + FieldSpec::new( + "kind", + FieldDefault::Member("sqlbuild._native", "UsageKind", "RUNTIME"), + ), + FieldSpec::new("through", FieldDefault::None), + FieldSpec::new("enum_member", FieldDefault::None), + ], +}; +const R_GRANT_RECORD: RecordSpec = RecordSpec { + name: "GrantRecord", + doc: "A compiler relationship granting a public declaration to a resource.", + fields: &[ + FieldSpec::required("resource"), + FieldSpec::required("declaration"), + FieldSpec::required("through"), + FieldSpec::new( + "kind", + FieldDefault::Member("sqlbuild._native", "GrantKind", "EXPECTED_MODEL"), + ), + ], +}; +const R_VISIBILITY_RECORD: RecordSpec = RecordSpec { + name: "VisibilityRecord", + doc: "A resolved positive visibility fact with stable provenance.", + fields: &[ + FieldSpec::required("resource"), + FieldSpec::required("declaration"), + FieldSpec::required("reason"), + FieldSpec::new("through", FieldDefault::None), + ], +}; +const R_INACCESSIBLE_RECORD: RecordSpec = RecordSpec { + name: "InaccessibleRecord", + doc: "A resolved negative visibility fact with stable provenance.", + fields: &[ + FieldSpec::required("resource"), + FieldSpec::required("declaration"), + FieldSpec::required("reason"), + ], +}; +const R_SCOPE_DIAGNOSTIC: RecordSpec = RecordSpec { + name: "ScopeDiagnostic", + doc: "Stable diagnostic retained even when an index is partial.", + fields: &[ + FieldSpec::required("code"), + FieldSpec::required("message"), + FieldSpec::new( + "severity", + FieldDefault::Member("sqlbuild._native", "ScopeDiagnosticSeverity", "ERROR"), + ), + FieldSpec::new("path", FieldDefault::None), + FieldSpec::new("line", FieldDefault::None), + FieldSpec::new("column", FieldDefault::None), + FieldSpec::new("declaration", FieldDefault::None), + FieldSpec::new("resource", FieldDefault::None), + ], +}; +const R_SCOPE_COMPLETENESS: RecordSpec = RecordSpec { + name: "ScopeCompleteness", + doc: "Independent completeness facts for partially invalid projects.", + fields: &[ + FieldSpec::new("discovery", FieldDefault::True), + FieldSpec::new("static_visibility", FieldDefault::True), + FieldSpec::new("runtime_usage", FieldDefault::True), + FieldSpec::new("relationships", FieldDefault::True), + FieldSpec::new("placement", FieldDefault::True), + FieldSpec::new("promotion_impact", FieldDefault::True), + ], +}; +const R_SCOPE_INDEX: RecordSpec = RecordSpec { + name: "ScopeIndex", + doc: "Canonical deterministic facts used by every future scope consumer.", + fields: &[ + FieldSpec::new("ownership_roots", FieldDefault::EmptyTuple), + FieldSpec::new("resources", FieldDefault::EmptyTuple), + FieldSpec::new("declarations", FieldDefault::EmptyTuple), + FieldSpec::new("usages", FieldDefault::EmptyTuple), + FieldSpec::new("grants", FieldDefault::EmptyTuple), + FieldSpec::new("visibility", FieldDefault::EmptyTuple), + FieldSpec::new("inaccessible", FieldDefault::EmptyTuple), + FieldSpec::new("diagnostics", FieldDefault::EmptyTuple), + FieldSpec::new( + "completeness", + FieldDefault::Call("sqlbuild._native", "ScopeCompleteness"), + ), + ], +}; +const R_DECLARATION_VISIBILITY_INDEX: RecordSpec = RecordSpec { + name: "DeclarationVisibilityIndex", + doc: "Declaration positions grouped by the scope facts that make them visible.", + fields: &[ + FieldSpec::required("identities"), + FieldSpec::required("global_positions"), + FieldSpec::required("private_positions"), + FieldSpec::required("local_positions"), + FieldSpec::required("inherited_positions"), + ], +}; +const R_SCOPE_LOOKUP: RecordSpec = RecordSpec { + name: "ScopeLookup", + doc: "Immutable indexes over one canonical ``ScopeIndex``.", + fields: &[ + FieldSpec::required("index"), + FieldSpec::required("resources"), + FieldSpec::required("resources_by_path"), + FieldSpec::required("declarations"), + FieldSpec::required("usages_by_consumer"), + FieldSpec::required("usages_by_declaration"), + FieldSpec::required("grants_by_resource"), + FieldSpec::required("visibility_index"), + FieldSpec { + name: "native", + default: FieldDefault::None, + compared: false, + }, + ], +}; +const R_SCOPE_TARGET_QUERY: RecordSpec = RecordSpec { + name: "ScopeTargetQuery", + doc: "Non-throwing qualified-identity or path lookup result.", + fields: &[ + FieldSpec::required("value"), + FieldSpec::new("matches", FieldDefault::EmptyTuple), + FieldSpec::new("declaration_matches", FieldDefault::EmptyTuple), + ], +}; +const R_VISIBILITY_RESOLUTION: RecordSpec = RecordSpec { + name: "VisibilityResolution", + doc: "Complete static visibility classification for one queried target.", + fields: &[ + FieldSpec::required("target"), + FieldSpec::new("visible", FieldDefault::EmptyTuple), + FieldSpec::new("inaccessible", FieldDefault::EmptyTuple), + ], +}; +const R_DECLARATION_VISIBILITY: RecordSpec = RecordSpec { + name: "DeclarationVisibility", + doc: "Visible facts and inaccessible declaration identities for one queried target.", + fields: &[ + FieldSpec::required("target"), + FieldSpec::new("visible", FieldDefault::EmptyTuple), + FieldSpec::new("inaccessible", FieldDefault::EmptyTuple), + ], +}; +const R_SCOPE_REPORT_FILTERS: RecordSpec = RecordSpec { + name: "ScopeReportFilters", + doc: "Orthogonal declaration filters applied in their documented fixed order.", + fields: &[ + FieldSpec::new( + "sections", + FieldDefault::TextTuple(&["available", "used", "relationship_scope"]), + ), + FieldSpec::new("include_nearby", FieldDefault::False), + FieldSpec::new("defined_under", FieldDefault::None), + FieldSpec::new("kinds", FieldDefault::EmptyTuple), + FieldSpec::new("glob", FieldDefault::None), + FieldSpec::new("used_only", FieldDefault::False), + FieldSpec::new("dependency_depth", FieldDefault::Int(0)), + FieldSpec::new("sort", FieldDefault::Text("identity")), + FieldSpec::new("cursor", FieldDefault::None), + FieldSpec::new("page_size", FieldDefault::Int(100)), + FieldSpec::new("nearby_depth", FieldDefault::Int(1)), + FieldSpec::new("globals", FieldDefault::Text("summary")), + ], +}; +const R_SCOPE_SOURCE_LOCATION: RecordSpec = RecordSpec { + name: "ScopeSourceLocation", + doc: "Safe project-relative source location.", + fields: &[ + FieldSpec::required("path"), + FieldSpec::required("line"), + FieldSpec::required("column"), + ], +}; +const R_VISIBILITY_PROVENANCE: RecordSpec = RecordSpec { + name: "VisibilityProvenance", + doc: "Why a declaration is visible or inaccessible.", + fields: &[ + FieldSpec::required("reason"), + FieldSpec::new("through", FieldDefault::None), + ], +}; +const R_DECLARATION_REPORT: RecordSpec = RecordSpec { + name: "DeclarationReport", + doc: "Value-free declaration projection used by every report section.", + fields: &[ + FieldSpec::required("identity"), + FieldSpec::required("kind"), + FieldSpec::required("name"), + FieldSpec::required("owner"), + FieldSpec::required("definition"), + FieldSpec::required("scope"), + FieldSpec::required("owning_path"), + FieldSpec::new("visibility", FieldDefault::None), + FieldSpec::new("inaccessible_reason", FieldDefault::None), + FieldSpec::new("metadata", FieldDefault::EmptyTuple), + FieldSpec::new("consumers", FieldDefault::EmptyTuple), + FieldSpec::new("dependencies", FieldDefault::EmptyTuple), + FieldSpec::new("grants", FieldDefault::EmptyTuple), + FieldSpec::new("required_scope", FieldDefault::None), + FieldSpec::new("required_path", FieldDefault::None), + FieldSpec::new("promotion_impact", FieldDefault::EmptyTuple), + ], +}; +const R_SCOPE_CHAIN_ENTRY: RecordSpec = RecordSpec { + name: "ScopeChainEntry", + doc: "One exact or inherited lexical scope in deterministic path order.", + fields: &[ + FieldSpec::required("kind"), + FieldSpec::required("path"), + FieldSpec::required("declaration_count"), + ], +}; +const R_SCOPE_SECTION: RecordSpec = RecordSpec { + name: "ScopeSection", + doc: "Pagination and completeness metadata for one report section.", + fields: &[ + FieldSpec::required("name"), + FieldSpec::required("total"), + FieldSpec::required("returned"), + FieldSpec::new("collapsed", FieldDefault::False), + FieldSpec::new("collapsed_count", FieldDefault::Int(0)), + FieldSpec::new("truncated", FieldDefault::False), + FieldSpec::new("complete", FieldDefault::True), + FieldSpec::new("next_cursor", FieldDefault::None), + FieldSpec::new("cursor", FieldDefault::None), + FieldSpec::new("page_size", FieldDefault::Int(100)), + ], +}; +const R_SCOPE_RESOURCE_REPORT: RecordSpec = RecordSpec { + name: "ScopeResourceReport", + doc: "Resolved existing resource or prospective authored path.", + fields: &[ + FieldSpec::required("target"), + FieldSpec::required("identity"), + FieldSpec::required("path"), + FieldSpec::new("prospective", FieldDefault::False), + FieldSpec::new("directory", FieldDefault::False), + FieldSpec::new("duplicate_count", FieldDefault::Int(0)), + ], +}; +const R_DECLARATION_EXPLANATION: RecordSpec = RecordSpec { + name: "DeclarationExplanation", + doc: "Detailed explanation for one qualified declaration.", + fields: &[ + FieldSpec::required("declaration"), + FieldSpec::required("complete"), + ], +}; +const R_MOVE_PREVIEW: RecordSpec = RecordSpec { + name: "MovePreview", + doc: "Pure visibility delta for moving one existing resource.", + fields: &[ + FieldSpec::required("resource"), + FieldSpec::required("destination"), + FieldSpec::required("new_ownership_root"), + FieldSpec::new("retained", FieldDefault::EmptyTuple), + FieldSpec::new("gained", FieldDefault::EmptyTuple), + FieldSpec::new("lost", FieldDefault::EmptyTuple), + FieldSpec::new("invalidated_usages", FieldDefault::EmptyTuple), + FieldSpec::new("private_retained", FieldDefault::EmptyTuple), + FieldSpec::new("relationship_retained", FieldDefault::EmptyTuple), + FieldSpec::new("complete", FieldDefault::True), + ], +}; +const R_SCOPE_FOLDER: RecordSpec = RecordSpec { + name: "ScopeFolder", + doc: "One declaration-definition folder with exact recursive counts.", + fields: &[ + FieldSpec::required("path"), + FieldSpec::required("name"), + FieldSpec::required("descendant_count"), + FieldSpec::required("child_count"), + FieldSpec::required("used_count"), + FieldSpec::required("kind_counts"), + ], +}; +const R_SCOPE_BROWSE_RESULT: RecordSpec = RecordSpec { + name: "ScopeBrowseResult", + doc: "Folder-first browse result.", + fields: &[ + FieldSpec::required("folder"), + FieldSpec::required("folders"), + FieldSpec::new("diagnostics", FieldDefault::EmptyTuple), + FieldSpec::new("complete", FieldDefault::True), + ], +}; +const R_SCOPE_LIST_RESULT: RecordSpec = RecordSpec { + name: "ScopeListResult", + doc: "Recursive declaration list beneath one definition folder.", + fields: &[ + FieldSpec::required("folder"), + FieldSpec::required("declarations"), + FieldSpec::required("section"), + FieldSpec::new("diagnostics", FieldDefault::EmptyTuple), + ], +}; +const R_SCOPE_REPORT: RecordSpec = RecordSpec { + name: "ScopeReport", + doc: "Canonical payload shared by all non-rendering scope consumers.", + fields: &[ + FieldSpec::required("resource"), + FieldSpec::new("scope_chain", FieldDefault::EmptyTuple), + FieldSpec::new("available", FieldDefault::EmptyTuple), + FieldSpec::new("used", FieldDefault::EmptyTuple), + FieldSpec::new("relationship_scope", FieldDefault::EmptyTuple), + FieldSpec::new("nearby_unavailable", FieldDefault::EmptyTuple), + FieldSpec::new( + "filters", + FieldDefault::Call("sqlbuild._native", "ScopeReportFilters"), + ), + FieldSpec::new("sections", FieldDefault::EmptyTuple), + FieldSpec::new("explanation", FieldDefault::None), + FieldSpec::new("move_preview", FieldDefault::None), + FieldSpec::new("diagnostics", FieldDefault::EmptyTuple), + FieldSpec::new("complete", FieldDefault::True), + ], +}; + +/// The declaration scope value records. +pub(crate) const SCOPE_RECORDS: &[RecordSpec] = &[ + R_RESOURCE_IDENTITY, + R_DECLARATION_IDENTITY, + R_OWNERSHIP_ROOT, + R_RESOURCE_RECORD, + R_MACRO_METADATA, + R_ENUM_MEMBER_METADATA, + R_ENUM_METADATA, + R_CONSTANT_METADATA, + R_DECLARATION_RECORD, + R_USAGE_RECORD, + R_GRANT_RECORD, + R_VISIBILITY_RECORD, + R_INACCESSIBLE_RECORD, + R_SCOPE_DIAGNOSTIC, + R_SCOPE_COMPLETENESS, + R_SCOPE_INDEX, + R_DECLARATION_VISIBILITY_INDEX, + R_SCOPE_LOOKUP, + R_SCOPE_TARGET_QUERY, + R_VISIBILITY_RESOLUTION, + R_DECLARATION_VISIBILITY, + R_SCOPE_REPORT_FILTERS, + R_SCOPE_SOURCE_LOCATION, + R_VISIBILITY_PROVENANCE, + R_DECLARATION_REPORT, + R_SCOPE_CHAIN_ENTRY, + R_SCOPE_SECTION, + R_SCOPE_RESOURCE_REPORT, + R_DECLARATION_EXPLANATION, + R_MOVE_PREVIEW, + R_SCOPE_FOLDER, + R_SCOPE_BROWSE_RESULT, + R_SCOPE_LIST_RESULT, + R_SCOPE_REPORT, +]; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/scope_records/mod.rs b/crates/sqlbuild-python/src/bindings/_helpers/scope_records/mod.rs new file mode 100644 index 0000000000..61d449b063 --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/scope_records/mod.rs @@ -0,0 +1 @@ +pub(crate) mod constants; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/scopes/compile_usages.rs b/crates/sqlbuild-python/src/bindings/_helpers/scopes/compile_usages.rs index 40e1cd71cf..1100ddc023 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/scopes/compile_usages.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/scopes/compile_usages.rs @@ -21,7 +21,7 @@ use sqlbuild_scopes::scope_index::models::{ use crate::bindings::_helpers::compiled_project::model_loop_named::scope_path_message; const SCOPE_ERRORS_MODULE: &str = "sqlbuild._native"; -const SCOPE_MODELS_MODULE: &str = "sqlbuild.compiler.scopes.models"; +const SCOPE_MODELS_MODULE: &str = "sqlbuild._native"; const COLLECTOR_MODULE: &str = "sqlbuild._native"; const COMPILE_MODELS_MODULE: &str = "sqlbuild.compiler.compile.models"; const INACCESSIBLE_DECLARATION_CODE: &str = "S006"; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/scopes/declaration_contexts.rs b/crates/sqlbuild-python/src/bindings/_helpers/scopes/declaration_contexts.rs index c8fe07474e..293558c237 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/scopes/declaration_contexts.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/scopes/declaration_contexts.rs @@ -34,7 +34,6 @@ struct PositionRow(Py, Py, Py, u8, Py, u8, u32); pub(crate) struct ContextClasses { visibility_record: Py, pub(crate) resolution_context: Py, - allocate: Py, reasons: Py, } @@ -293,7 +292,7 @@ impl NativeDeclarationContexts { ) } - /// A `VisibilityRecord` built without running its dataclass initializer. + /// A `VisibilityRecord` of `resource`, `declaration`, `reason` and `through`. pub(crate) fn visibility_record_for<'py>( &self, resource: &Bound<'py, PyAny>, @@ -303,13 +302,7 @@ impl NativeDeclarationContexts { ) -> PyResult> { let py: Python<'py> = resource.py(); let class: &Bound<'py, PyAny> = self.classes.visibility_record.bind(py); - let record: Bound<'py, PyAny> = self.classes.allocate.bind(py).call1((class,))?; - let attributes: Bound<'py, PyDict> = - record.getattr("__dict__")?.downcast_into::()?; - attributes.set_item("resource", resource)?; - attributes.set_item("declaration", declaration)?; - attributes.set_item("reason", reason)?; - attributes.set_item("through", through)?; + let record: Bound<'py, PyAny> = class.call1((resource, declaration, reason, through))?; Ok(record) } } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/scopes/scope_command.rs b/crates/sqlbuild-python/src/bindings/_helpers/scopes/scope_command.rs index b73006f8a1..817073f23a 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/scopes/scope_command.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/scopes/scope_command.rs @@ -23,8 +23,8 @@ use crate::bindings::_helpers::scopes::scope_reports::{ }; const NATIVE_MODULE: &str = "sqlbuild._native"; -const MODELS_MODULE: &str = "sqlbuild.compiler.scopes.models"; -const TYPES_MODULE: &str = "sqlbuild.compiler.scopes.types"; +const MODELS_MODULE: &str = "sqlbuild._native"; +const TYPES_MODULE: &str = "sqlbuild._native"; const ERRORS_MODULE: &str = "sqlbuild._native"; const TARGET_ERROR_CODE: &str = "C960"; const BARE_USAGE: &str = "bare names work for models, sources, seeds and functions; write macros, \ diff --git a/crates/sqlbuild-python/src/bindings/_helpers/scopes/scope_objects.rs b/crates/sqlbuild-python/src/bindings/_helpers/scopes/scope_objects.rs index 7ac27f2348..ffeafab18c 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/scopes/scope_objects.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/scopes/scope_objects.rs @@ -23,8 +23,8 @@ use crate::bindings::_helpers::scopes::placement::{ }; use crate::bindings::_helpers::scopes::scope_lookup::grant_kind; -const MODELS_MODULE: &str = "sqlbuild.compiler.scopes.models"; -const TYPES_MODULE: &str = "sqlbuild.compiler.scopes.types"; +const MODELS_MODULE: &str = "sqlbuild._native"; +const TYPES_MODULE: &str = "sqlbuild._native"; const ENUM_MODULE: &str = "enum"; /// The scope model and enum modules every built object comes from. @@ -670,7 +670,7 @@ pub(crate) fn diagnostic_object<'py>( classes.model("ScopeDiagnostic")?.call1(( classes.member("ScopeDiagnosticCode", &item.code)?, item.message.as_str(), - classes.member("DiagnosticSeverity", &item.severity)?, + classes.member("ScopeDiagnosticSeverity", &item.severity)?, item.path.as_deref(), item.line, item.column, diff --git a/crates/sqlbuild-python/src/bindings/_helpers/scopes/scope_report_objects.rs b/crates/sqlbuild-python/src/bindings/_helpers/scopes/scope_report_objects.rs index 4198c9a939..bbfbf1599f 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/scopes/scope_report_objects.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/scopes/scope_report_objects.rs @@ -224,7 +224,7 @@ fn declaration_object<'py>( report.kind.into_bound_py_any(py)?, report.name.into_bound_py_any(py)?, report.owner.into_bound_py_any(py)?, - classes.model("SourceLocation")?.call1(( + classes.model("ScopeSourceLocation")?.call1(( report.definition.path, report.definition.line, report.definition.column, diff --git a/crates/sqlbuild-python/src/bindings/_helpers/scopes/scope_validation.rs b/crates/sqlbuild-python/src/bindings/_helpers/scopes/scope_validation.rs index c63490bee2..292be2f619 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/scopes/scope_validation.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/scopes/scope_validation.rs @@ -25,7 +25,7 @@ use crate::bindings::_helpers::scopes::scope_objects::{ use crate::bindings::types::CompilerDetach; const DATACLASSES_MODULE: &str = "dataclasses"; -const SCOPE_TYPES_MODULE: &str = "sqlbuild.compiler.scopes.types"; +const SCOPE_TYPES_MODULE: &str = "sqlbuild._native"; const SCOPE_ERRORS_MODULE: &str = "sqlbuild._native"; const RUNTIME_USAGE: usize = 2; /// Compile input groups whose items carry declaration usages, after the model inputs. @@ -219,7 +219,7 @@ fn placement_validated_index<'py>( } let classes: ScopeClasses<'py> = ScopeClasses::new(py)?; let severity: Bound<'py, PyAny> = classes.member( - "DiagnosticSeverity", + "ScopeDiagnosticSeverity", if enforce_placement { "error" } else { @@ -379,7 +379,7 @@ pub(crate) fn validate_scope_index(index: &Bound<'_, PyAny>) -> Result<(), PyErr let py: Python<'_> = index.py(); let error: Bound<'_, PyAny> = py .import(SCOPE_TYPES_MODULE)? - .getattr("DiagnosticSeverity")? + .getattr("ScopeDiagnosticSeverity")? .getattr("ERROR")?; let mut errors: Vec> = Vec::new(); for item in index.getattr("diagnostics")?.try_iter()? { diff --git a/fensu.toml b/fensu.toml index 141966664d..d6e7a493a0 100644 --- a/fensu.toml +++ b/fensu.toml @@ -173,11 +173,6 @@ rule = "FFR309" path = "src/sqlbuild/compiler/refactoring/__init__.py" reason = "Refactorings run natively; the package keeps the compiled-project input, manual-location model and error classes the native command reads and raises." -[[targets.python.rule_exceptions]] -rule = "FFR309" -path = "src/sqlbuild/compiler/scopes/__init__.py" -reason = "Declaration scopes are built, queried and validated natively; the package keeps the index, lookup and report models and the errors the native bindings raise." - [[targets.python.rule_exceptions]] rule = "FFR309" path = "src/sqlbuild/compiler/sql_test_glue/__init__.py" diff --git a/python-boundary.toml b/python-boundary.toml index 8667730cda..f6aaf0a3d0 100644 --- a/python-boundary.toml +++ b/python-boundary.toml @@ -2,7 +2,7 @@ # migration work that moves to Rust. `remaining_lines` must equal the current count of that work: # a PR that moves code lowers it, and adding movable Python fails the check. # Native failures must surface as errors; `handled_native_errors` lists the reviewed exceptions. -remaining_lines = 5847 +remaining_lines = 5280 [[stays]] path = "executor" diff --git a/scripts/compiler_differential/_helpers/corpus/emitted_codes.py b/scripts/compiler_differential/_helpers/corpus/emitted_codes.py index 55b090329d..b02f3a7bbe 100644 --- a/scripts/compiler_differential/_helpers/corpus/emitted_codes.py +++ b/scripts/compiler_differential/_helpers/corpus/emitted_codes.py @@ -39,10 +39,9 @@ FailureCase, ProjectRun, ) -from sqlbuild._native import DiscoveryError, ResourceIdentityError +from sqlbuild._native import DiscoveryError, ResourceIdentityError, ScopeDiagnosticCode from sqlbuild.compiler.compile import constants as compile_constants from sqlbuild.compiler.compile.exceptions import CompileInputError -from sqlbuild.compiler.scopes.types import ScopeDiagnosticCode _SOURCE_DIRECTORY: str = "source" _CODE_KEYWORD: str = "code" diff --git a/src/sqlbuild/_native.pyi b/src/sqlbuild/_native.pyi index 4a5eeb5dc1..e7bead5092 100644 --- a/src/sqlbuild/_native.pyi +++ b/src/sqlbuild/_native.pyi @@ -182,29 +182,6 @@ from sqlbuild.compiler.references.types import ( HardCodedRelationOwnerKind, SqlReferenceKind, ) -from sqlbuild.compiler.scopes.models import ( - DeclarationExplanation, - DeclarationIdentity, - DeclarationRecord, - DeclarationVisibility, - MovePreview, - ResourceIdentity, - ScopeBrowseResult, - ScopeDiagnostic, - ScopeIndex, - ScopeListResult, - ScopeLookup, - ScopeReport, - ScopeReportFilters, - ScopeTargetQuery, - UsageRecord, - VisibilityRecord, - VisibilityResolution, -) -from sqlbuild.compiler.scopes.types import ( - DeclarationKind, - ScopeKind, -) from sqlbuild.compiler.source_freshness.models import ( SourceFreshnessRecord, ) @@ -13362,3 +13339,718 @@ class CustomRulesOutcome(NativeRecord): custom_ms: int, custom_cpu_ms: int = ..., ) -> None: ... + +class ResourceKind(StrEnum): + """Kinds of authored resources represented in a scope index.""" + + MODEL = "model" + TEST = "test" + SCENARIO = "scenario" + FUNCTION = "function" + SOURCE = "source" + SEED = "seed" + +class DeclarationKind(StrEnum): + """Independent public declaration namespaces.""" + + MACRO = "macro" + ENUM = "enum" + CONSTANT = "constant" + AUDIT = "audit" + SINGULAR_AUDIT = "singular_audit" + SCHEMA = "schema" + SQL_HOOK = "sql_hook" + PYTHON_HOOK = "python_hook" + +class ScopeKind(StrEnum): + """Compiler-enforced declaration visibility tiers.""" + + GLOBAL = "global" + INHERITED = "inherited" + LOCAL = "local" + PRIVATE = "private" + +class OwnershipRootKind(StrEnum): + """How an ownership root participates in authored resource scope.""" + + RESOURCE = "resource" + GLOBAL = "global" + +class UsageKind(StrEnum): + """Why one indexed identity consumes a declaration.""" + + RUNTIME = "runtime" + GENERATED = "generated" + DECLARATION_DEPENDENCY = "declaration_dependency" + +class GrantKind(StrEnum): + """Compiler relationships that grant additional public vocabulary.""" + + EXPECTED_MODEL = "expected_model" + TESTED_MACRO = "tested_macro" + +class VisibilityReason(StrEnum): + """Positive explanations emitted by the canonical resolver.""" + + GLOBAL = "global" + INHERITED_ANCESTOR = "inherited_ancestor" + LOCAL_OWNER = "local_owner" + PRIVATE_OWNER = "private_owner" + EXPECTED_MODEL = "expected_model" + TESTED_MACRO = "tested_macro" + +class InaccessibleReason(StrEnum): + """Stable explanations for declarations outside effective scope.""" + + LOCAL_BOUNDARY = "local_boundary" + SIBLING_SCOPE = "sibling_scope" + DESCENDANT_SCOPE = "descendant_scope" + UNRELATED_SCOPE = "unrelated_scope" + PRIVATE_OWNER = "private_owner" + UNSUPPORTED_RESOURCE_KIND = "unsupported_resource_kind" + +class ScopeDiagnosticSeverity(StrEnum): + """Severity of a stable scope diagnostic.""" + + ERROR = "error" + WARNING = "warning" + +class ScopeDiagnosticCode(StrEnum): + """Stable diagnostic identities shared by future scope consumers.""" + + UNKNOWN_TARGET = "S001" + UNQUALIFIED_TARGET = "S002" + DUPLICATE_DECLARATION = "S003" + INVALID_DECLARATION_NAME = "S004" + RESERVED_DECLARATION_NAME = "S005" + INACCESSIBLE_DECLARATION = "S006" + LOCAL_NEEDED_BY_DESCENDANT = "S007" + OVER_BROAD_INHERITED = "S008" + REQUIRES_GLOBAL_PLACEMENT = "S009" + UNUSED_DECLARATION = "S010" + INVALID_PROSPECTIVE_PATH = "S011" + NESTED_DECLARATION_ROOT = "S012" + INCOMPLETE_USAGE = "S013" + INVALID_CURSOR = "S014" + MACRO_MODULE_IMPORT = "S015" + INVALID_MACRO_DEPENDENCY = "S016" + MACRO_DEPENDENCY_CYCLE = "S017" + DUPLICATE_RESOURCE = "S018" + RESOURCE_PARSE_ERROR = "S019" + DECLARATION_PARSE_ERROR = "S020" + MACRO_PARSE_ERROR = "S021" + RELATIONSHIP_PARSE_ERROR = "S022" + CONFIG_PARSE_ERROR = "S023" + OVER_BROAD_GLOBAL = "S024" + +class ResourceIdentity(NativeRecord): + """Stable kind-qualified identity of one authored resource.""" + + kind: ResourceKind + name: str + def __init__( + self, + kind: ResourceKind, + name: str, + ) -> None: ... + +class DeclarationIdentity(NativeRecord): + """Stable declaration identity, including a private declaration owner.""" + + kind: DeclarationKind + name: str + owner: ResourceIdentity | None + def __init__( + self, + kind: DeclarationKind, + name: str, + owner: ResourceIdentity | None = ..., + ) -> None: ... + +class OwnershipRoot(NativeRecord): + """Normalized project-relative directory that owns resource scope.""" + + path: str + kind: OwnershipRootKind + resource_kind: ResourceKind | None + def __init__( + self, + path: str, + kind: OwnershipRootKind = ..., + resource_kind: ResourceKind | None = ..., + ) -> None: ... + +class ResourceRecord(NativeRecord): + """One resource participating in lexical declaration scope.""" + + identity: ResourceIdentity + path: str + ownership_root: OwnershipRoot + def __init__( + self, + identity: ResourceIdentity, + path: str, + ownership_root: OwnershipRoot, + ) -> None: ... + +class MacroMetadata(NativeRecord): + """Safe macro metadata required by scope and dependency inspection.""" + + parameters: tuple[str, ...] + dependencies: tuple[DeclarationIdentity, ...] + source_digest: str + def __init__( + self, + parameters: tuple[str, ...] = ..., + dependencies: tuple[DeclarationIdentity, ...] = ..., + source_digest: str = ..., + ) -> None: ... + +class EnumMemberMetadata(NativeRecord): + """One value-free authored enum member identity.""" + + name: str + def __init__( + self, + name: str, + ) -> None: ... + +class EnumMetadata(NativeRecord): + """Safe enum metadata containing member names and their shared scalar type.""" + + members: tuple[EnumMemberMetadata, ...] + scalar_type: str + def __init__( + self, + members: tuple[EnumMemberMetadata, ...], + scalar_type: str, + ) -> None: ... + +class ConstantMetadata(NativeRecord): + """Value-free normalized metadata for one typed constant.""" + + logical_type: str + collection_kind: str | None + item_count: int | None + nullable: bool + render_as: str | None + def __init__( + self, + logical_type: str, + collection_kind: str | None = ..., + item_count: int | None = ..., + nullable: bool = ..., + render_as: str | None = ..., + ) -> None: ... + +class DeclarationRecord(NativeRecord): + """One declaration definition and its compiler-owned scope metadata.""" + + identity: DeclarationIdentity + path: str + line: int + column: int + scope: ScopeKind + ownership_root: OwnershipRoot + owning_path: str | None + macro: MacroMetadata | None + enum: EnumMetadata | None + constant: ConstantMetadata | None + def __init__( + self, + identity: DeclarationIdentity, + path: str, + line: int, + column: int, + scope: ScopeKind, + ownership_root: OwnershipRoot, + owning_path: str | None = ..., + macro: MacroMetadata | None = ..., + enum: EnumMetadata | None = ..., + constant: ConstantMetadata | None = ..., + ) -> None: ... + +class UsageRecord(NativeRecord): + """A runtime or declaration dependency edge.""" + + consumer: ResourceIdentity | DeclarationIdentity + declaration: DeclarationIdentity + kind: UsageKind + through: ResourceIdentity | DeclarationIdentity | None + enum_member: str | None + def __init__( + self, + consumer: ResourceIdentity | DeclarationIdentity, + declaration: DeclarationIdentity, + kind: UsageKind = ..., + through: ResourceIdentity | DeclarationIdentity | None = ..., + enum_member: str | None = ..., + ) -> None: ... + +class GrantRecord(NativeRecord): + """A compiler relationship granting a public declaration to a resource.""" + + resource: ResourceIdentity + declaration: DeclarationIdentity + through: ResourceIdentity | DeclarationIdentity + kind: GrantKind + def __init__( + self, + resource: ResourceIdentity, + declaration: DeclarationIdentity, + through: ResourceIdentity | DeclarationIdentity, + kind: GrantKind = ..., + ) -> None: ... + +class VisibilityRecord(NativeRecord): + """A resolved positive visibility fact with stable provenance.""" + + resource: ResourceIdentity + declaration: DeclarationIdentity + reason: VisibilityReason + through: ResourceIdentity | DeclarationIdentity | None + def __init__( + self, + resource: ResourceIdentity, + declaration: DeclarationIdentity, + reason: VisibilityReason, + through: ResourceIdentity | DeclarationIdentity | None = ..., + ) -> None: ... + +class InaccessibleRecord(NativeRecord): + """A resolved negative visibility fact with stable provenance.""" + + resource: ResourceIdentity + declaration: DeclarationIdentity + reason: InaccessibleReason + def __init__( + self, + resource: ResourceIdentity, + declaration: DeclarationIdentity, + reason: InaccessibleReason, + ) -> None: ... + +class ScopeDiagnostic(NativeRecord): + """Stable diagnostic retained even when an index is partial.""" + + code: ScopeDiagnosticCode + message: str + severity: ScopeDiagnosticSeverity + path: str | None + line: int | None + column: int | None + declaration: DeclarationIdentity | None + resource: ResourceIdentity | None + def __init__( + self, + code: ScopeDiagnosticCode, + message: str, + severity: ScopeDiagnosticSeverity = ..., + path: str | None = ..., + line: int | None = ..., + column: int | None = ..., + declaration: DeclarationIdentity | None = ..., + resource: ResourceIdentity | None = ..., + ) -> None: ... + +class ScopeCompleteness(NativeRecord): + """Independent completeness facts for partially invalid projects.""" + + discovery: bool + static_visibility: bool + runtime_usage: bool + relationships: bool + placement: bool + promotion_impact: bool + def __init__( + self, + discovery: bool = ..., + static_visibility: bool = ..., + runtime_usage: bool = ..., + relationships: bool = ..., + placement: bool = ..., + promotion_impact: bool = ..., + ) -> None: ... + @property + def complete(self) -> bool: ... + +class ScopeIndex(NativeRecord): + """Canonical deterministic facts used by every future scope consumer.""" + + ownership_roots: tuple[OwnershipRoot, ...] + resources: tuple[ResourceRecord, ...] + declarations: tuple[DeclarationRecord, ...] + usages: tuple[UsageRecord, ...] + grants: tuple[GrantRecord, ...] + visibility: tuple[VisibilityRecord, ...] + inaccessible: tuple[InaccessibleRecord, ...] + diagnostics: tuple[ScopeDiagnostic, ...] + completeness: ScopeCompleteness + def __init__( + self, + ownership_roots: tuple[OwnershipRoot, ...] = ..., + resources: tuple[ResourceRecord, ...] = ..., + declarations: tuple[DeclarationRecord, ...] = ..., + usages: tuple[UsageRecord, ...] = ..., + grants: tuple[GrantRecord, ...] = ..., + visibility: tuple[VisibilityRecord, ...] = ..., + inaccessible: tuple[InaccessibleRecord, ...] = ..., + diagnostics: tuple[ScopeDiagnostic, ...] = ..., + completeness: ScopeCompleteness = ..., + ) -> None: ... + +class DeclarationVisibilityIndex(NativeRecord): + """Declaration positions grouped by the scope facts that make them visible.""" + + identities: tuple[DeclarationIdentity, ...] + global_positions: tuple[int, ...] + private_positions: Mapping[ResourceIdentity, tuple[int, ...]] + local_positions: Mapping[str, tuple[int, ...]] + inherited_positions: Mapping[str, tuple[int, ...]] + def __init__( + self, + identities: tuple[DeclarationIdentity, ...], + global_positions: tuple[int, ...], + private_positions: Mapping[ResourceIdentity, tuple[int, ...]], + local_positions: Mapping[str, tuple[int, ...]], + inherited_positions: Mapping[str, tuple[int, ...]], + ) -> None: ... + +class ScopeLookup(NativeRecord): + """Immutable indexes over one canonical ``ScopeIndex``.""" + + index: ScopeIndex + resources: Mapping[ResourceIdentity, tuple[ResourceRecord, ...]] + resources_by_path: Mapping[str, tuple[ResourceRecord, ...]] + declarations: Mapping[DeclarationIdentity, tuple[DeclarationRecord, ...]] + usages_by_consumer: Mapping[ResourceIdentity | DeclarationIdentity, tuple[UsageRecord, ...]] + usages_by_declaration: Mapping[DeclarationIdentity, tuple[UsageRecord, ...]] + grants_by_resource: Mapping[ResourceIdentity, tuple[GrantRecord, ...]] + visibility_index: DeclarationVisibilityIndex + native: object | None + def __init__( + self, + index: ScopeIndex, + resources: Mapping[ResourceIdentity, tuple[ResourceRecord, ...]], + resources_by_path: Mapping[str, tuple[ResourceRecord, ...]], + declarations: Mapping[DeclarationIdentity, tuple[DeclarationRecord, ...]], + usages_by_consumer: Mapping[ + ResourceIdentity | DeclarationIdentity, tuple[UsageRecord, ...] + ], + usages_by_declaration: Mapping[DeclarationIdentity, tuple[UsageRecord, ...]], + grants_by_resource: Mapping[ResourceIdentity, tuple[GrantRecord, ...]], + visibility_index: DeclarationVisibilityIndex, + native: object | None = ..., + ) -> None: ... + +class ScopeTargetQuery(NativeRecord): + """Non-throwing qualified-identity or path lookup result.""" + + value: str | ResourceIdentity | DeclarationIdentity + matches: tuple[ResourceRecord, ...] + declaration_matches: tuple[DeclarationRecord, ...] + def __init__( + self, + value: str | ResourceIdentity | DeclarationIdentity, + matches: tuple[ResourceRecord, ...] = ..., + declaration_matches: tuple[DeclarationRecord, ...] = ..., + ) -> None: ... + @property + def unknown(self) -> bool: ... + +class VisibilityResolution(NativeRecord): + """Complete static visibility classification for one queried target.""" + + target: ScopeTargetQuery + visible: tuple[VisibilityRecord, ...] + inaccessible: tuple[InaccessibleRecord, ...] + def __init__( + self, + target: ScopeTargetQuery, + visible: tuple[VisibilityRecord, ...] = ..., + inaccessible: tuple[InaccessibleRecord, ...] = ..., + ) -> None: ... + +class DeclarationVisibility(NativeRecord): + """Visible facts and inaccessible declaration identities for one queried target.""" + + target: ScopeTargetQuery + visible: tuple[VisibilityRecord, ...] + inaccessible: tuple[DeclarationIdentity, ...] + def __init__( + self, + target: ScopeTargetQuery, + visible: tuple[VisibilityRecord, ...] = ..., + inaccessible: tuple[DeclarationIdentity, ...] = ..., + ) -> None: ... + +class ScopeReportFilters(NativeRecord): + """Orthogonal declaration filters applied in their documented fixed order.""" + + sections: tuple[str, ...] + include_nearby: bool + defined_under: str | None + kinds: tuple[DeclarationKind, ...] + glob: str | None + used_only: bool + dependency_depth: int + sort: str + cursor: str | None + page_size: int + nearby_depth: int + globals: str + def __init__( + self, + sections: tuple[str, ...] = ..., + include_nearby: bool = ..., + defined_under: str | None = ..., + kinds: tuple[DeclarationKind, ...] = ..., + glob: str | None = ..., + used_only: bool = ..., + dependency_depth: int = ..., + sort: str = ..., + cursor: str | None = ..., + page_size: int = ..., + nearby_depth: int = ..., + globals: str = ..., + ) -> None: ... + +class ScopeSourceLocation(NativeRecord): + """Safe project-relative source location.""" + + path: str + line: int + column: int + def __init__( + self, + path: str, + line: int, + column: int, + ) -> None: ... + +class VisibilityProvenance(NativeRecord): + """Why a declaration is visible or inaccessible.""" + + reason: str + through: str | None + def __init__( + self, + reason: str, + through: str | None = ..., + ) -> None: ... + +class DeclarationReport(NativeRecord): + """Value-free declaration projection used by every report section.""" + + identity: str + kind: str + name: str + owner: str | None + definition: SourceLocation + scope: str + owning_path: str | None + visibility: VisibilityProvenance | None + inaccessible_reason: str | None + metadata: tuple[tuple[str, object], ...] + consumers: tuple[str, ...] + dependencies: tuple[str, ...] + grants: tuple[str, ...] + required_scope: str | None + required_path: str | None + promotion_impact: tuple[str, ...] + def __init__( + self, + identity: str, + kind: str, + name: str, + owner: str | None, + definition: SourceLocation, + scope: str, + owning_path: str | None, + visibility: VisibilityProvenance | None = ..., + inaccessible_reason: str | None = ..., + metadata: tuple[tuple[str, object], ...] = ..., + consumers: tuple[str, ...] = ..., + dependencies: tuple[str, ...] = ..., + grants: tuple[str, ...] = ..., + required_scope: str | None = ..., + required_path: str | None = ..., + promotion_impact: tuple[str, ...] = ..., + ) -> None: ... + +class ScopeChainEntry(NativeRecord): + """One exact or inherited lexical scope in deterministic path order.""" + + kind: str + path: str + declaration_count: int + def __init__( + self, + kind: str, + path: str, + declaration_count: int, + ) -> None: ... + +class ScopeSection(NativeRecord): + """Pagination and completeness metadata for one report section.""" + + name: str + total: int + returned: int + collapsed: bool + collapsed_count: int + truncated: bool + complete: bool + next_cursor: str | None + cursor: str | None + page_size: int + def __init__( + self, + name: str, + total: int, + returned: int, + collapsed: bool = ..., + collapsed_count: int = ..., + truncated: bool = ..., + complete: bool = ..., + next_cursor: str | None = ..., + cursor: str | None = ..., + page_size: int = ..., + ) -> None: ... + +class ScopeResourceReport(NativeRecord): + """Resolved existing resource or prospective authored path.""" + + target: str + identity: str | None + path: str | None + prospective: bool + directory: bool + duplicate_count: int + def __init__( + self, + target: str, + identity: str | None, + path: str | None, + prospective: bool = ..., + directory: bool = ..., + duplicate_count: int = ..., + ) -> None: ... + +class DeclarationExplanation(NativeRecord): + """Detailed explanation for one qualified declaration.""" + + declaration: DeclarationReport | None + complete: bool + def __init__( + self, + declaration: DeclarationReport | None, + complete: bool, + ) -> None: ... + +class MovePreview(NativeRecord): + """Pure visibility delta for moving one existing resource.""" + + resource: str + destination: str + new_ownership_root: str | None + retained: tuple[DeclarationReport, ...] + gained: tuple[DeclarationReport, ...] + lost: tuple[DeclarationReport, ...] + invalidated_usages: tuple[str, ...] + private_retained: tuple[DeclarationReport, ...] + relationship_retained: tuple[DeclarationReport, ...] + complete: bool + def __init__( + self, + resource: str, + destination: str, + new_ownership_root: str | None, + retained: tuple[DeclarationReport, ...] = ..., + gained: tuple[DeclarationReport, ...] = ..., + lost: tuple[DeclarationReport, ...] = ..., + invalidated_usages: tuple[str, ...] = ..., + private_retained: tuple[DeclarationReport, ...] = ..., + relationship_retained: tuple[DeclarationReport, ...] = ..., + complete: bool = ..., + ) -> None: ... + +class ScopeFolder(NativeRecord): + """One declaration-definition folder with exact recursive counts.""" + + path: str + name: str + descendant_count: int + child_count: int + used_count: int + kind_counts: tuple[tuple[str, int], ...] + def __init__( + self, + path: str, + name: str, + descendant_count: int, + child_count: int, + used_count: int, + kind_counts: tuple[tuple[str, int], ...], + ) -> None: ... + +class ScopeBrowseResult(NativeRecord): + """Folder-first browse result.""" + + folder: str + folders: tuple[ScopeFolder, ...] + diagnostics: tuple[ScopeDiagnostic, ...] + complete: bool + def __init__( + self, + folder: str, + folders: tuple[ScopeFolder, ...], + diagnostics: tuple[ScopeDiagnostic, ...] = ..., + complete: bool = ..., + ) -> None: ... + +class ScopeListResult(NativeRecord): + """Recursive declaration list beneath one definition folder.""" + + folder: str + declarations: tuple[DeclarationReport, ...] + section: ScopeSection + diagnostics: tuple[ScopeDiagnostic, ...] + def __init__( + self, + folder: str, + declarations: tuple[DeclarationReport, ...], + section: ScopeSection, + diagnostics: tuple[ScopeDiagnostic, ...] = ..., + ) -> None: ... + +class ScopeReport(NativeRecord): + """Canonical payload shared by all non-rendering scope consumers.""" + + resource: ScopeResourceReport + scope_chain: tuple[ScopeChainEntry, ...] + available: tuple[DeclarationReport, ...] + used: tuple[DeclarationReport, ...] + relationship_scope: tuple[DeclarationReport, ...] + nearby_unavailable: tuple[DeclarationReport, ...] + filters: ScopeReportFilters + sections: tuple[ScopeSection, ...] + explanation: DeclarationExplanation | None + move_preview: MovePreview | None + diagnostics: tuple[ScopeDiagnostic, ...] + complete: bool + def __init__( + self, + resource: ScopeResourceReport, + scope_chain: tuple[ScopeChainEntry, ...] = ..., + available: tuple[DeclarationReport, ...] = ..., + used: tuple[DeclarationReport, ...] = ..., + relationship_scope: tuple[DeclarationReport, ...] = ..., + nearby_unavailable: tuple[DeclarationReport, ...] = ..., + filters: ScopeReportFilters = ..., + sections: tuple[ScopeSection, ...] = ..., + explanation: DeclarationExplanation | None = ..., + move_preview: MovePreview | None = ..., + diagnostics: tuple[ScopeDiagnostic, ...] = ..., + complete: bool = ..., + ) -> None: ... diff --git a/src/sqlbuild/compiler/compile/_helpers/render/user_macros.py b/src/sqlbuild/compiler/compile/_helpers/render/user_macros.py index a6c0b0fcaa..dcc9979e2e 100644 --- a/src/sqlbuild/compiler/compile/_helpers/render/user_macros.py +++ b/src/sqlbuild/compiler/compile/_helpers/render/user_macros.py @@ -14,7 +14,12 @@ from typing import cast import sqlbuild._native as _native -from sqlbuild._native import validate_resource_identity +from sqlbuild._native import ( + DeclarationIdentity, + DeclarationKind, + DeclarationRecord, + validate_resource_identity, +) from sqlbuild.compiler.compile.exceptions import ( CompileInputError, MacroDeclarationLookupError, @@ -32,11 +37,6 @@ DiscoveredMacroFile, EnumDeclaration, ) -from sqlbuild.compiler.scopes.models import ( - DeclarationIdentity, - DeclarationRecord, -) -from sqlbuild.compiler.scopes.types import DeclarationKind from sqlbuild.sql_values.models import SqlValue from sqlbuild.sql_values.types import SqlValueKind diff --git a/src/sqlbuild/compiler/compile/models.py b/src/sqlbuild/compiler/compile/models.py index 453c264aba..fd68f9c4aa 100644 --- a/src/sqlbuild/compiler/compile/models.py +++ b/src/sqlbuild/compiler/compile/models.py @@ -22,6 +22,8 @@ CompileModelConfig, CompileSqlScenarioCte, DeclarationExpansionContext, + DeclarationIdentity, + DeclarationRecord, DeclarationScopeResolver, DiagnosticPhase, DiagnosticSeverity, @@ -34,6 +36,10 @@ LiteralSqlRelation, ModelSchemaDeclaration, RelatedLocation, + ResourceIdentity, + ScopeIndex, + UsageRecord, + VisibilityRecord, ) from sqlbuild.compiler.auditing.models import MeasurementContract, MeasurementThresholds from sqlbuild.compiler.auditing.types import AuditEvaluationMode, AuditSeverity @@ -64,14 +70,6 @@ ) from sqlbuild.compiler.lineage.types import InferredNullability from sqlbuild.compiler.references.types import ExternalSqlReferenceResolver, SqlReferenceKind -from sqlbuild.compiler.scopes.models import ( - DeclarationIdentity, - DeclarationRecord, - ResourceIdentity, - ScopeIndex, - UsageRecord, - VisibilityRecord, -) from sqlbuild.compiler.sql_analysis.models import SqlLexicalSyntax from sqlbuild.spec.contracts.models import ( ScenarioConfig, diff --git a/src/sqlbuild/compiler/compile/types.py b/src/sqlbuild/compiler/compile/types.py index 23f8069960..927c6a14fd 100644 --- a/src/sqlbuild/compiler/compile/types.py +++ b/src/sqlbuild/compiler/compile/types.py @@ -27,6 +27,8 @@ FunctionArgument, FunctionReturnColumn, LiteralSqlRelation, + ScopeIndex, + UsageRecord, ) from sqlbuild.compiler.auditing.models import MeasurementContract, MeasurementThresholds from sqlbuild.compiler.auditing.types import AuditEvaluationMode, AuditSeverity @@ -49,7 +51,6 @@ EnumDeclaration, ) from sqlbuild.compiler.references.types import ExternalSqlReferenceResolver - from sqlbuild.compiler.scopes.models import ScopeIndex, UsageRecord from sqlbuild.compiler.sql_analysis.models import SqlLexicalSyntax from sqlbuild.spec.contracts.models import ( ScenarioConfig, diff --git a/src/sqlbuild/compiler/discovery/models.py b/src/sqlbuild/compiler/discovery/models.py index 9958fad246..9570fb3258 100644 --- a/src/sqlbuild/compiler/discovery/models.py +++ b/src/sqlbuild/compiler/discovery/models.py @@ -22,6 +22,7 @@ DiscoveredSqlTestBlock, EnumMember, NativeDiscoverySession, + ScopeKind, ) from sqlbuild.compiler.discovery.constants import ( SQL_SCENARIOS_OWNERSHIP_ROOT, @@ -31,7 +32,6 @@ LoaderConnectionMode, ProjectProvider, ) -from sqlbuild.compiler.scopes.types import ScopeKind from sqlbuild.python_nodes.models import AuditCase, ColumnLineageRef, RetryPolicy, SqlResourceRef from sqlbuild.python_nodes.types import PythonCheckSeverity from sqlbuild.runtime.event_exporting.constants import EVENT_EXPORT_KINDS diff --git a/src/sqlbuild/compiler/scopes/__init__.py b/src/sqlbuild/compiler/scopes/__init__.py deleted file mode 100644 index f49da181bf..0000000000 --- a/src/sqlbuild/compiler/scopes/__init__.py +++ /dev/null @@ -1 +0,0 @@ -"""Canonical compiler-owned declaration scope domain.""" diff --git a/src/sqlbuild/compiler/scopes/models.py b/src/sqlbuild/compiler/scopes/models.py deleted file mode 100644 index ca713943b6..0000000000 --- a/src/sqlbuild/compiler/scopes/models.py +++ /dev/null @@ -1,435 +0,0 @@ -"""Immutable models for the canonical compiler scope index.""" - -from __future__ import annotations - -from collections.abc import Mapping -from dataclasses import dataclass, field -from types import MappingProxyType - -from sqlbuild.compiler.scopes.types import ( - CompletenessSection, - DeclarationKind, - DiagnosticSeverity, - GrantKind, - InaccessibleReason, - OwnershipRootKind, - ResourceKind, - ScopeDiagnosticCode, - ScopeKind, - UsageKind, - VisibilityReason, -) - - -@dataclass(frozen=True, order=True) -class ResourceIdentity: - """Stable kind-qualified identity of one authored resource.""" - - kind: ResourceKind - name: str - - -@dataclass(frozen=True, order=True) -class DeclarationIdentity: - """Stable declaration identity, including a private declaration owner.""" - - kind: DeclarationKind - name: str - owner: ResourceIdentity | None = None - - -@dataclass(frozen=True, order=True) -class OwnershipRoot: - """Normalized project-relative directory that owns resource scope.""" - - path: str - kind: OwnershipRootKind = OwnershipRootKind.RESOURCE - resource_kind: ResourceKind | None = None - - -@dataclass(frozen=True) -class ResourceRecord: - """One resource participating in lexical declaration scope.""" - - identity: ResourceIdentity - path: str - ownership_root: OwnershipRoot - - -@dataclass(frozen=True) -class MacroMetadata: - """Safe macro metadata required by scope and dependency inspection.""" - - parameters: tuple[str, ...] = field(default_factory=tuple) - dependencies: tuple[DeclarationIdentity, ...] = field(default_factory=tuple) - source_digest: str = "" - - -@dataclass(frozen=True) -class EnumMemberMetadata: - """One value-free authored enum member identity.""" - - name: str - - -@dataclass(frozen=True) -class EnumMetadata: - """Safe enum metadata containing member names and their shared scalar type.""" - - members: tuple[EnumMemberMetadata, ...] - scalar_type: str - - -@dataclass(frozen=True) -class ConstantMetadata: - """Value-free normalized metadata for one typed constant.""" - - logical_type: str - collection_kind: str | None = None - item_count: int | None = None - nullable: bool = False - render_as: str | None = None - - -@dataclass(frozen=True) -class DeclarationRecord: - """One declaration definition and its compiler-owned scope metadata.""" - - identity: DeclarationIdentity - path: str - line: int - column: int - scope: ScopeKind - ownership_root: OwnershipRoot - owning_path: str | None = None - macro: MacroMetadata | None = None - enum: EnumMetadata | None = None - constant: ConstantMetadata | None = None - - -@dataclass(frozen=True) -class UsageRecord: - """A runtime or declaration dependency edge.""" - - consumer: ResourceIdentity | DeclarationIdentity - declaration: DeclarationIdentity - kind: UsageKind = UsageKind.RUNTIME - through: ResourceIdentity | DeclarationIdentity | None = None - enum_member: str | None = None - - -@dataclass(frozen=True) -class GrantRecord: - """A compiler relationship granting a public declaration to a resource.""" - - resource: ResourceIdentity - declaration: DeclarationIdentity - through: ResourceIdentity | DeclarationIdentity - kind: GrantKind = GrantKind.EXPECTED_MODEL - - -@dataclass(frozen=True) -class VisibilityRecord: - """A resolved positive visibility fact with stable provenance.""" - - resource: ResourceIdentity - declaration: DeclarationIdentity - reason: VisibilityReason - through: ResourceIdentity | DeclarationIdentity | None = None - - -@dataclass(frozen=True) -class InaccessibleRecord: - """A resolved negative visibility fact with stable provenance.""" - - resource: ResourceIdentity - declaration: DeclarationIdentity - reason: InaccessibleReason - - -@dataclass(frozen=True) -class ScopeDiagnostic: - """Stable diagnostic retained even when an index is partial.""" - - code: ScopeDiagnosticCode - message: str - severity: DiagnosticSeverity = DiagnosticSeverity.ERROR - path: str | None = None - line: int | None = None - column: int | None = None - declaration: DeclarationIdentity | None = None - resource: ResourceIdentity | None = None - - -@dataclass(frozen=True) -class ScopeCompleteness: - """Independent completeness facts for partially invalid projects.""" - - discovery: bool = True - static_visibility: bool = True - runtime_usage: bool = True - relationships: bool = True - placement: bool = True - promotion_impact: bool = True - - @property - def complete(self) -> bool: - """Return whether every index section is complete.""" - - return all(self.as_mapping().values()) - - def as_mapping(self) -> Mapping[CompletenessSection, bool]: - """Return immutable section completeness keyed by stable names.""" - - return MappingProxyType( - { - CompletenessSection.DISCOVERY: self.discovery, - CompletenessSection.STATIC_VISIBILITY: self.static_visibility, - CompletenessSection.RUNTIME_USAGE: self.runtime_usage, - CompletenessSection.RELATIONSHIPS: self.relationships, - CompletenessSection.PLACEMENT: self.placement, - CompletenessSection.PROMOTION_IMPACT: self.promotion_impact, - } - ) - - -@dataclass(frozen=True) -class ScopeIndex: - """Canonical deterministic facts used by every future scope consumer.""" - - ownership_roots: tuple[OwnershipRoot, ...] = field(default_factory=tuple) - resources: tuple[ResourceRecord, ...] = field(default_factory=tuple) - declarations: tuple[DeclarationRecord, ...] = field(default_factory=tuple) - usages: tuple[UsageRecord, ...] = field(default_factory=tuple) - grants: tuple[GrantRecord, ...] = field(default_factory=tuple) - visibility: tuple[VisibilityRecord, ...] = field(default_factory=tuple) - inaccessible: tuple[InaccessibleRecord, ...] = field(default_factory=tuple) - diagnostics: tuple[ScopeDiagnostic, ...] = field(default_factory=tuple) - completeness: ScopeCompleteness = field(default_factory=ScopeCompleteness) - - -@dataclass(frozen=True) -class DeclarationVisibilityIndex: - """Declaration positions grouped by the scope facts that make them visible.""" - - identities: tuple[DeclarationIdentity, ...] - global_positions: tuple[int, ...] - private_positions: Mapping[ResourceIdentity, tuple[int, ...]] - local_positions: Mapping[str, tuple[int, ...]] - inherited_positions: Mapping[str, tuple[int, ...]] - - -@dataclass(frozen=True) -class ScopeLookup: - """Immutable indexes over one canonical ``ScopeIndex``.""" - - index: ScopeIndex - resources: Mapping[ResourceIdentity, tuple[ResourceRecord, ...]] - resources_by_path: Mapping[str, tuple[ResourceRecord, ...]] - declarations: Mapping[DeclarationIdentity, tuple[DeclarationRecord, ...]] - usages_by_consumer: Mapping[ResourceIdentity | DeclarationIdentity, tuple[UsageRecord, ...]] - usages_by_declaration: Mapping[DeclarationIdentity, tuple[UsageRecord, ...]] - grants_by_resource: Mapping[ResourceIdentity, tuple[GrantRecord, ...]] - visibility_index: DeclarationVisibilityIndex - native: object | None = field(default=None, compare=False, repr=False) - - -@dataclass(frozen=True) -class ScopeTargetQuery: - """Non-throwing qualified-identity or path lookup result.""" - - value: str | ResourceIdentity | DeclarationIdentity - matches: tuple[ResourceRecord, ...] = field(default_factory=tuple) - declaration_matches: tuple[DeclarationRecord, ...] = field(default_factory=tuple) - - @property - def unknown(self) -> bool: - """Return whether the target is absent rather than merely inaccessible.""" - - return not self.matches and not self.declaration_matches - - -@dataclass(frozen=True) -class VisibilityResolution: - """Complete static visibility classification for one queried target.""" - - target: ScopeTargetQuery - visible: tuple[VisibilityRecord, ...] = field(default_factory=tuple) - inaccessible: tuple[InaccessibleRecord, ...] = field(default_factory=tuple) - - -@dataclass(frozen=True) -class DeclarationVisibility: - """Visible facts and inaccessible declaration identities for one queried target.""" - - target: ScopeTargetQuery - visible: tuple[VisibilityRecord, ...] = field(default_factory=tuple) - inaccessible: tuple[DeclarationIdentity, ...] = field(default_factory=tuple) - - -@dataclass(frozen=True) -class ScopeReportFilters: - """Orthogonal declaration filters applied in their documented fixed order.""" - - sections: tuple[str, ...] = ("available", "used", "relationship_scope") - include_nearby: bool = False - defined_under: str | None = None - kinds: tuple[DeclarationKind, ...] = field(default_factory=tuple) - glob: str | None = None - used_only: bool = False - dependency_depth: int = 0 - sort: str = "identity" - cursor: str | None = None - page_size: int = 100 - nearby_depth: int = 1 - globals: str = "summary" - - -@dataclass(frozen=True) -class SourceLocation: - """Safe project-relative source location.""" - - path: str - line: int - column: int - - -@dataclass(frozen=True) -class VisibilityProvenance: - """Why a declaration is visible or inaccessible.""" - - reason: str - through: str | None = None - - -@dataclass(frozen=True) -class DeclarationReport: - """Value-free declaration projection used by every report section.""" - - identity: str - kind: str - name: str - owner: str | None - definition: SourceLocation - scope: str - owning_path: str | None - visibility: VisibilityProvenance | None = None - inaccessible_reason: str | None = None - metadata: tuple[tuple[str, object], ...] = field(default_factory=tuple) - consumers: tuple[str, ...] = field(default_factory=tuple) - dependencies: tuple[str, ...] = field(default_factory=tuple) - grants: tuple[str, ...] = field(default_factory=tuple) - required_scope: str | None = None - required_path: str | None = None - promotion_impact: tuple[str, ...] = field(default_factory=tuple) - - -@dataclass(frozen=True) -class ScopeChainEntry: - """One exact or inherited lexical scope in deterministic path order.""" - - kind: str - path: str - declaration_count: int - - -@dataclass(frozen=True) -class ScopeSection: - """Pagination and completeness metadata for one report section.""" - - name: str - total: int - returned: int - collapsed: bool = False - collapsed_count: int = 0 - truncated: bool = False - complete: bool = True - next_cursor: str | None = None - cursor: str | None = None - page_size: int = 100 - - -@dataclass(frozen=True) -class ScopeResourceReport: - """Resolved existing resource or prospective authored path.""" - - target: str - identity: str | None - path: str | None - prospective: bool = False - directory: bool = False - duplicate_count: int = 0 - - -@dataclass(frozen=True) -class DeclarationExplanation: - """Detailed explanation for one qualified declaration.""" - - declaration: DeclarationReport | None - complete: bool - - -@dataclass(frozen=True) -class MovePreview: - """Pure visibility delta for moving one existing resource.""" - - resource: str - destination: str - new_ownership_root: str | None - retained: tuple[DeclarationReport, ...] = field(default_factory=tuple) - gained: tuple[DeclarationReport, ...] = field(default_factory=tuple) - lost: tuple[DeclarationReport, ...] = field(default_factory=tuple) - invalidated_usages: tuple[str, ...] = field(default_factory=tuple) - private_retained: tuple[DeclarationReport, ...] = field(default_factory=tuple) - relationship_retained: tuple[DeclarationReport, ...] = field(default_factory=tuple) - complete: bool = True - - -@dataclass(frozen=True) -class ScopeFolder: - """One declaration-definition folder with exact recursive counts.""" - - path: str - name: str - descendant_count: int - child_count: int - used_count: int - kind_counts: tuple[tuple[str, int], ...] - - -@dataclass(frozen=True) -class ScopeBrowseResult: - """Folder-first browse result.""" - - folder: str - folders: tuple[ScopeFolder, ...] - diagnostics: tuple[ScopeDiagnostic, ...] = field(default_factory=tuple) - complete: bool = True - - -@dataclass(frozen=True) -class ScopeListResult: - """Recursive declaration list beneath one definition folder.""" - - folder: str - declarations: tuple[DeclarationReport, ...] - section: ScopeSection - diagnostics: tuple[ScopeDiagnostic, ...] = field(default_factory=tuple) - - -@dataclass(frozen=True) -class ScopeReport: - """Canonical payload shared by all non-rendering scope consumers.""" - - resource: ScopeResourceReport - scope_chain: tuple[ScopeChainEntry, ...] = field(default_factory=tuple) - available: tuple[DeclarationReport, ...] = field(default_factory=tuple) - used: tuple[DeclarationReport, ...] = field(default_factory=tuple) - relationship_scope: tuple[DeclarationReport, ...] = field(default_factory=tuple) - nearby_unavailable: tuple[DeclarationReport, ...] = field(default_factory=tuple) - filters: ScopeReportFilters = field(default_factory=ScopeReportFilters) - sections: tuple[ScopeSection, ...] = field(default_factory=tuple) - explanation: DeclarationExplanation | None = None - move_preview: MovePreview | None = None - diagnostics: tuple[ScopeDiagnostic, ...] = field(default_factory=tuple) - complete: bool = True diff --git a/src/sqlbuild/compiler/scopes/types.py b/src/sqlbuild/compiler/scopes/types.py deleted file mode 100644 index c9dfe6bcd3..0000000000 --- a/src/sqlbuild/compiler/scopes/types.py +++ /dev/null @@ -1,129 +0,0 @@ -"""Enums used by the compiler-owned scope domain.""" - -from __future__ import annotations - -from enum import StrEnum - - -class ResourceKind(StrEnum): - """Kinds of authored resources represented in a scope index.""" - - MODEL = "model" - TEST = "test" - SCENARIO = "scenario" - FUNCTION = "function" - SOURCE = "source" - SEED = "seed" - - -class DeclarationKind(StrEnum): - """Independent public declaration namespaces.""" - - MACRO = "macro" - ENUM = "enum" - CONSTANT = "constant" - AUDIT = "audit" - SINGULAR_AUDIT = "singular_audit" - SCHEMA = "schema" - SQL_HOOK = "sql_hook" - PYTHON_HOOK = "python_hook" - - -class ScopeKind(StrEnum): - """Compiler-enforced declaration visibility tiers.""" - - GLOBAL = "global" - INHERITED = "inherited" - LOCAL = "local" - PRIVATE = "private" - - -class OwnershipRootKind(StrEnum): - """How an ownership root participates in authored resource scope.""" - - RESOURCE = "resource" - GLOBAL = "global" - - -class UsageKind(StrEnum): - """Why one indexed identity consumes a declaration.""" - - RUNTIME = "runtime" - GENERATED = "generated" - DECLARATION_DEPENDENCY = "declaration_dependency" - - -class GrantKind(StrEnum): - """Compiler relationships that grant additional public vocabulary.""" - - EXPECTED_MODEL = "expected_model" - TESTED_MACRO = "tested_macro" - - -class VisibilityReason(StrEnum): - """Positive explanations emitted by the canonical resolver.""" - - GLOBAL = "global" - INHERITED_ANCESTOR = "inherited_ancestor" - LOCAL_OWNER = "local_owner" - PRIVATE_OWNER = "private_owner" - EXPECTED_MODEL = "expected_model" - TESTED_MACRO = "tested_macro" - - -class InaccessibleReason(StrEnum): - """Stable explanations for declarations outside effective scope.""" - - LOCAL_BOUNDARY = "local_boundary" - SIBLING_SCOPE = "sibling_scope" - DESCENDANT_SCOPE = "descendant_scope" - UNRELATED_SCOPE = "unrelated_scope" - PRIVATE_OWNER = "private_owner" - UNSUPPORTED_RESOURCE_KIND = "unsupported_resource_kind" - - -class DiagnosticSeverity(StrEnum): - """Severity of a stable scope diagnostic.""" - - ERROR = "error" - WARNING = "warning" - - -class ScopeDiagnosticCode(StrEnum): - """Stable diagnostic identities shared by future scope consumers.""" - - UNKNOWN_TARGET = "S001" - UNQUALIFIED_TARGET = "S002" - DUPLICATE_DECLARATION = "S003" - INVALID_DECLARATION_NAME = "S004" - RESERVED_DECLARATION_NAME = "S005" - INACCESSIBLE_DECLARATION = "S006" - LOCAL_NEEDED_BY_DESCENDANT = "S007" - OVER_BROAD_INHERITED = "S008" - REQUIRES_GLOBAL_PLACEMENT = "S009" - UNUSED_DECLARATION = "S010" - INVALID_PROSPECTIVE_PATH = "S011" - NESTED_DECLARATION_ROOT = "S012" - INCOMPLETE_USAGE = "S013" - INVALID_CURSOR = "S014" - MACRO_MODULE_IMPORT = "S015" - INVALID_MACRO_DEPENDENCY = "S016" - MACRO_DEPENDENCY_CYCLE = "S017" - DUPLICATE_RESOURCE = "S018" - RESOURCE_PARSE_ERROR = "S019" - DECLARATION_PARSE_ERROR = "S020" - MACRO_PARSE_ERROR = "S021" - RELATIONSHIP_PARSE_ERROR = "S022" - CONFIG_PARSE_ERROR = "S023" - OVER_BROAD_GLOBAL = "S024" - - -class CompletenessSection(StrEnum): - """Independently tracked scope-index analysis sections.""" - - DISCOVERY = "discovery" - STATIC_VISIBILITY = "static_visibility" - RUNTIME_USAGE = "runtime_usage" - RELATIONSHIPS = "relationships" - PLACEMENT = "placement" - PROMOTION_IMPACT = "promotion_impact" diff --git a/tests/integration/src/sqlbuild/compiler/compile/helpers.py b/tests/integration/src/sqlbuild/compiler/compile/helpers.py index f732d103b2..a083d118c8 100644 --- a/tests/integration/src/sqlbuild/compiler/compile/helpers.py +++ b/tests/integration/src/sqlbuild/compiler/compile/helpers.py @@ -18,6 +18,8 @@ CompileAdapterContext, CompilerStage, MacroExpansionResult, + ResourceIdentity, + ResourceKind, build_compile_inputs, compile_frontier, discover_project_inputs, @@ -32,8 +34,6 @@ from sqlbuild.compiler.discovery.models import DiscoveredProjectInputs from sqlbuild.compiler.macro_bridge.classes.macro_bridge import MacroBridge from sqlbuild.compiler.macro_bridge.constants import ACTIVE_MACRO_BRIDGE -from sqlbuild.compiler.scopes.models import ResourceIdentity -from sqlbuild.compiler.scopes.types import ResourceKind if TYPE_CHECKING: from sqlbuild._native import CompileInputsView diff --git a/tests/integration/src/sqlbuild/compiler/pipeline/test_macro_load_count.py b/tests/integration/src/sqlbuild/compiler/pipeline/test_macro_load_count.py index e26bf73da4..19136be0d5 100644 --- a/tests/integration/src/sqlbuild/compiler/pipeline/test_macro_load_count.py +++ b/tests/integration/src/sqlbuild/compiler/pipeline/test_macro_load_count.py @@ -6,9 +6,9 @@ import pytest +from sqlbuild._native import DeclarationRecord from sqlbuild.adapters.duckdb.classes.duckdb_adapter import DuckDbAdapter from sqlbuild.compiler.pipeline.models import CompilePipelineResult -from sqlbuild.compiler.scopes.models import DeclarationRecord from tests.integration.src.sqlbuild.compiler.pipeline._test_types import ( MacroCompositionIntegrationTestCase, MacroLoadCountIntegrationTestCase, diff --git a/tests/integration/src/sqlbuild/compiler/scopes/helpers.py b/tests/integration/src/sqlbuild/compiler/scopes/helpers.py index 097ddd2d38..42a458f532 100644 --- a/tests/integration/src/sqlbuild/compiler/scopes/helpers.py +++ b/tests/integration/src/sqlbuild/compiler/scopes/helpers.py @@ -14,9 +14,20 @@ import sqlbuild._native as _native from sqlbuild._native import ( + DeclarationIdentity, + DeclarationKind, + DeclarationRecord, DeclarationScopeBuild, DeclarationScopeResolver, + DeclarationVisibility, DiscoveryError, + ResourceIdentity, + ResourceKind, + ScopeIndex, + ScopeKind, + ScopeLookup, + VisibilityReason, + VisibilityRecord, build_declaration_scope, discover_project_inputs, resolve_declaration_context, @@ -32,21 +43,6 @@ DiscoveredProjectInputs, EnumDeclaration, ) -from sqlbuild.compiler.scopes.models import ( - DeclarationIdentity, - DeclarationRecord, - DeclarationVisibility, - ResourceIdentity, - ScopeIndex, - ScopeLookup, - VisibilityRecord, -) -from sqlbuild.compiler.scopes.types import ( - DeclarationKind, - ResourceKind, - ScopeKind, - VisibilityReason, -) from sqlbuild.compiler.sql_analysis.models import SqlLexicalSyntax from tests.integration.src.sqlbuild.compiler.helpers import GoldenEntry, golden_entry diff --git a/tests/integration/src/sqlbuild/compiler/scopes/test_native_scope_parity.py b/tests/integration/src/sqlbuild/compiler/scopes/test_native_scope_parity.py index c0bdd44c72..f97b7170c4 100644 --- a/tests/integration/src/sqlbuild/compiler/scopes/test_native_scope_parity.py +++ b/tests/integration/src/sqlbuild/compiler/scopes/test_native_scope_parity.py @@ -7,7 +7,7 @@ import pytest -from sqlbuild.compiler.scopes.models import ScopeIndex +from sqlbuild._native import ScopeIndex from tests.integration.src.sqlbuild.compiler.scopes._test_types import ( GeneratedScopeParityTestCase, ScopeCommandParityTestCase, diff --git a/tests/unit/src/sqlbuild/cli/commands/_helpers/entry/test_parsing.py b/tests/unit/src/sqlbuild/cli/commands/_helpers/entry/test_parsing.py index 9566ba298c..0c6990f34e 100644 --- a/tests/unit/src/sqlbuild/cli/commands/_helpers/entry/test_parsing.py +++ b/tests/unit/src/sqlbuild/cli/commands/_helpers/entry/test_parsing.py @@ -10,11 +10,11 @@ from sqlbuild._native import ( ColumnLineageMode, CompileLineageMode, + DeclarationKind, ParsedCliInvocation, PlaygroundTemplate, parse_cli_invocation, ) -from sqlbuild.compiler.scopes.types import DeclarationKind from tests.unit.src.sqlbuild.cli.commands._helpers.entry._test_types import ( AuditConcurrencyParsingTestCase, ChoiceListTestCase, diff --git a/tests/unit/src/sqlbuild/cli/commands/_helpers/scope/helpers.py b/tests/unit/src/sqlbuild/cli/commands/_helpers/scope/helpers.py index fbc50afd1e..6d9d48d82c 100644 --- a/tests/unit/src/sqlbuild/cli/commands/_helpers/scope/helpers.py +++ b/tests/unit/src/sqlbuild/cli/commands/_helpers/scope/helpers.py @@ -4,17 +4,19 @@ from dataclasses import replace -from sqlbuild._native import build_scope_lookup, preview_scope_move, query_scope_report -from sqlbuild.compiler.scopes.models import ( +from sqlbuild._native import ( OwnershipRoot, ResourceIdentity, + ResourceKind, ResourceRecord, ScopeIndex, ScopeLookup, ScopeReport, ScopeReportFilters, + build_scope_lookup, + preview_scope_move, + query_scope_report, ) -from sqlbuild.compiler.scopes.types import ResourceKind from tests.unit.src.sqlbuild.compiler.scopes.helpers import report_scope_lookup diff --git a/tests/unit/src/sqlbuild/cli/commands/main/scope/test_scope.py b/tests/unit/src/sqlbuild/cli/commands/main/scope/test_scope.py index 2af8533318..4f82827a5a 100644 --- a/tests/unit/src/sqlbuild/cli/commands/main/scope/test_scope.py +++ b/tests/unit/src/sqlbuild/cli/commands/main/scope/test_scope.py @@ -8,9 +8,8 @@ import pytest from sqlbuild import _native as native -from sqlbuild._native import ScopeCommandRequest, run_scope_command +from sqlbuild._native import ScopeCommandRequest, ScopeCompleteness, ScopeIndex, run_scope_command from sqlbuild.adapter.discovery.main import resolve_adapter as adapter_registry -from sqlbuild.compiler.scopes.models import ScopeCompleteness, ScopeIndex from tests.unit.src.sqlbuild.cli.commands.main.scope._test_types import ScopeCommandCase from tests.unit.src.sqlbuild.compiler.scopes.helpers import report_scope_lookup from tests.unit.src.sqlbuild.presentation.helpers import use_color_terminal diff --git a/tests/unit/src/sqlbuild/compiler/compile/_helpers/_test_types.py b/tests/unit/src/sqlbuild/compiler/compile/_helpers/_test_types.py index 459cc67b16..2c3f460f5c 100644 --- a/tests/unit/src/sqlbuild/compiler/compile/_helpers/_test_types.py +++ b/tests/unit/src/sqlbuild/compiler/compile/_helpers/_test_types.py @@ -5,7 +5,7 @@ from pathlib import Path from typing import TYPE_CHECKING -from sqlbuild._native import CompiledLineageColumnFact, ExpansionSpan +from sqlbuild._native import CompiledLineageColumnFact, ExpansionSpan, UsageRecord from sqlbuild.adapter.contract.models import ExpressionInferenceProfile from sqlbuild.compiler.compile.models import CompiledObjectKey, CompileSqlReference, InferredColumn from sqlbuild.compiler.compile.types import AttachedAuditTargetKind @@ -13,7 +13,6 @@ InferredNullability, ) from sqlbuild.compiler.references.types import SqlReferenceKind -from sqlbuild.compiler.scopes.models import UsageRecord from sqlbuild.compiler.sql_analysis.models import SqlLexicalSyntax from sqlbuild.spec.contracts.models import ResolvedTimeTravelRetention, SourceLocation from sqlbuild.sql_values.models import SqlValue diff --git a/tests/unit/src/sqlbuild/compiler/compile/_helpers/helpers.py b/tests/unit/src/sqlbuild/compiler/compile/_helpers/helpers.py index fc68224946..467fba143c 100644 --- a/tests/unit/src/sqlbuild/compiler/compile/_helpers/helpers.py +++ b/tests/unit/src/sqlbuild/compiler/compile/_helpers/helpers.py @@ -16,10 +16,21 @@ CompiledLineageColumnFact, CompiledModelSqlTestPayload, DeclarationExpansionContext, + DeclarationIdentity, + DeclarationKind, + DeclarationRecord, DeclarationRuntimeProjection, DeclarationScopeResolver, + MacroMetadata, NativeColumnQuery, NativeQueryColumns, + OwnershipRoot, + ResourceIdentity, + ResourceKind, + ResourceRecord, + ScopeIndex, + ScopeKind, + ScopeLookup, assemble_project, build_compile_inputs, build_execution_upstream_deps, @@ -62,17 +73,6 @@ DiscoveredProjectInputs, ) from sqlbuild.compiler.lineage.types import InferredNullability -from sqlbuild.compiler.scopes.models import ( - DeclarationIdentity, - DeclarationRecord, - MacroMetadata, - OwnershipRoot, - ResourceIdentity, - ResourceRecord, - ScopeIndex, - ScopeLookup, -) -from sqlbuild.compiler.scopes.types import DeclarationKind, ResourceKind, ScopeKind if TYPE_CHECKING: from sqlbuild._native import CompileInputsView, ModelInput diff --git a/tests/unit/src/sqlbuild/compiler/compile/_helpers/test_declaration_scopes.py b/tests/unit/src/sqlbuild/compiler/compile/_helpers/test_declaration_scopes.py index 10d1a9b0e6..86b583e1d2 100644 --- a/tests/unit/src/sqlbuild/compiler/compile/_helpers/test_declaration_scopes.py +++ b/tests/unit/src/sqlbuild/compiler/compile/_helpers/test_declaration_scopes.py @@ -10,6 +10,12 @@ import pytest from sqlbuild._native import ( + DeclarationIdentity, + DeclarationKind, + ResourceIdentity, + ResourceKind, + UsageKind, + UsageRecord, assemble_project, build_compile_inputs, discover_project_inputs, @@ -22,16 +28,6 @@ DiscoveredSourceFile, SqlHookEntry, ) -from sqlbuild.compiler.scopes.models import ( - DeclarationIdentity, - ResourceIdentity, - UsageRecord, -) -from sqlbuild.compiler.scopes.types import ( - DeclarationKind, - ResourceKind, - UsageKind, -) from sqlbuild.spec.contracts.models import SourceEntry from tests.unit.src.sqlbuild.compiler.compile._helpers._test_types import ( ExpectedBooleanTestCase, diff --git a/tests/unit/src/sqlbuild/compiler/compile/_helpers/test_macros.py b/tests/unit/src/sqlbuild/compiler/compile/_helpers/test_macros.py index 17e5d1a51b..0ce5692eec 100644 --- a/tests/unit/src/sqlbuild/compiler/compile/_helpers/test_macros.py +++ b/tests/unit/src/sqlbuild/compiler/compile/_helpers/test_macros.py @@ -5,7 +5,7 @@ import pytest -from sqlbuild._native import expand_sql_macros, find_macro_call_names +from sqlbuild._native import ScopeKind, expand_sql_macros, find_macro_call_names from sqlbuild.compiler.compile._helpers.render import user_macros as macros_module from sqlbuild.compiler.compile._helpers.render.user_macros import load_project_macros from sqlbuild.compiler.compile.models import ( @@ -13,7 +13,6 @@ MacroContext, ) from sqlbuild.compiler.discovery.models import DiscoveredMacroFile -from sqlbuild.compiler.scopes.types import ScopeKind from tests.unit.src.sqlbuild.compiler.compile._helpers._test_types import ( ExpandSqlMacrosErrorTestCase, ExpandSqlMacrosTestCase, diff --git a/tests/unit/src/sqlbuild/compiler/compile/_helpers/test_rebind_declaration_scope.py b/tests/unit/src/sqlbuild/compiler/compile/_helpers/test_rebind_declaration_scope.py index 28a9954852..69457b3fb4 100644 --- a/tests/unit/src/sqlbuild/compiler/compile/_helpers/test_rebind_declaration_scope.py +++ b/tests/unit/src/sqlbuild/compiler/compile/_helpers/test_rebind_declaration_scope.py @@ -8,6 +8,8 @@ import pytest from sqlbuild._native import ( + DeclarationIdentity, + DeclarationKind, DeclarationScopeBuild, build_declaration_scope, discover_project_inputs, @@ -16,8 +18,6 @@ from sqlbuild.compiler.compile._helpers.render.user_macros import load_project_macros from sqlbuild.compiler.compile.models import LoadedMacro from sqlbuild.compiler.discovery.models import DiscoveredProjectInputs -from sqlbuild.compiler.scopes.models import DeclarationIdentity -from sqlbuild.compiler.scopes.types import DeclarationKind from sqlbuild.compiler.sql_analysis.models import SqlLexicalSyntax from tests.unit.src.sqlbuild.compiler.compile._helpers._test_types import ( RebindDeclarationScopeTestCase, diff --git a/tests/unit/src/sqlbuild/compiler/compile/_helpers/test_scoped_named_declarations.py b/tests/unit/src/sqlbuild/compiler/compile/_helpers/test_scoped_named_declarations.py index 0d074171cf..0f5f16a290 100644 --- a/tests/unit/src/sqlbuild/compiler/compile/_helpers/test_scoped_named_declarations.py +++ b/tests/unit/src/sqlbuild/compiler/compile/_helpers/test_scoped_named_declarations.py @@ -7,8 +7,8 @@ import pytest +from sqlbuild._native import ScopeDiagnosticSeverity as DiagnosticSeverity from sqlbuild.compiler.compile.types import CompiledProjectView -from sqlbuild.compiler.scopes.types import DiagnosticSeverity from tests.unit.src.sqlbuild.compiler.compile._helpers._test_types import ( NamedDeclarationAcceptedTestCase, NamedDeclarationErrorTestCase, diff --git a/tests/unit/src/sqlbuild/compiler/manifest/test_main.py b/tests/unit/src/sqlbuild/compiler/manifest/test_main.py index 022f77edd1..4e76c3eb44 100644 --- a/tests/unit/src/sqlbuild/compiler/manifest/test_main.py +++ b/tests/unit/src/sqlbuild/compiler/manifest/test_main.py @@ -9,7 +9,15 @@ import sqlbuild._native as _native from sqlbuild._native import ( + DeclarationIdentity, + DeclarationKind, + DeclarationRecord, DiscoveredSqlHookFile, + MacroMetadata, + OwnershipRoot, + ResourceKind, + ScopeIndex, + ScopeKind, build_manifest, project_with, ) @@ -25,14 +33,6 @@ ) from sqlbuild.compiler.discovery.models import SqlHookEntry from sqlbuild.compiler.planner.models import AuditPlanEntry, ChainStep, PlanOutput, SqlTestPlanEntry -from sqlbuild.compiler.scopes.models import ( - DeclarationIdentity, - DeclarationRecord, - MacroMetadata, - OwnershipRoot, - ScopeIndex, -) -from sqlbuild.compiler.scopes.types import DeclarationKind, ResourceKind, ScopeKind from sqlbuild.spec.contracts.models import SchemaColumn, SourceColumnEntry from tests.unit.src.sqlbuild.compiler.manifest._test_types import ( FqnTestCase, diff --git a/tests/unit/src/sqlbuild/compiler/scopes/_test_helpers.py b/tests/unit/src/sqlbuild/compiler/scopes/_test_helpers.py index 96ae930421..d58860beb8 100644 --- a/tests/unit/src/sqlbuild/compiler/scopes/_test_helpers.py +++ b/tests/unit/src/sqlbuild/compiler/scopes/_test_helpers.py @@ -9,34 +9,30 @@ from pathlib import PurePath from sqlbuild._native import ( - build_scope_lookup, - normalize_scope_path, - query_scope_target, - resolve_scope_declaration_visibility, - resolve_scope_visibility, -) -from sqlbuild.compiler.scopes.models import ( DeclarationIdentity, + DeclarationKind, DeclarationRecord, DeclarationVisibility, + GrantKind, GrantRecord, + InaccessibleReason, InaccessibleRecord, OwnershipRoot, ResourceIdentity, + ResourceKind, ResourceRecord, ScopeIndex, + ScopeKind, ScopeLookup, ScopeTargetQuery, + VisibilityReason, VisibilityRecord, VisibilityResolution, -) -from sqlbuild.compiler.scopes.types import ( - DeclarationKind, - GrantKind, - InaccessibleReason, - ResourceKind, - ScopeKind, - VisibilityReason, + build_scope_lookup, + normalize_scope_path, + query_scope_target, + resolve_scope_declaration_visibility, + resolve_scope_visibility, ) _FOLDERS: tuple[str, ...] = ("sales", "sale", "finance", "daily", "orders") diff --git a/tests/unit/src/sqlbuild/compiler/scopes/_test_types.py b/tests/unit/src/sqlbuild/compiler/scopes/_test_types.py index 458ff145d2..5737c253f0 100644 --- a/tests/unit/src/sqlbuild/compiler/scopes/_test_types.py +++ b/tests/unit/src/sqlbuild/compiler/scopes/_test_types.py @@ -5,16 +5,17 @@ from collections.abc import Callable from dataclasses import dataclass -from sqlbuild.compiler.scopes.models import DeclarationIdentity, ResourceIdentity -from sqlbuild.compiler.scopes.types import ( +from sqlbuild._native import ( + DeclarationIdentity, DeclarationKind, - DiagnosticSeverity, InaccessibleReason, + ResourceIdentity, ResourceKind, ScopeDiagnosticCode, ScopeKind, VisibilityReason, ) +from sqlbuild._native import ScopeDiagnosticSeverity as DiagnosticSeverity @dataclass(frozen=True) diff --git a/tests/unit/src/sqlbuild/compiler/scopes/helpers.py b/tests/unit/src/sqlbuild/compiler/scopes/helpers.py index 36c6d58db8..26426537ce 100644 --- a/tests/unit/src/sqlbuild/compiler/scopes/helpers.py +++ b/tests/unit/src/sqlbuild/compiler/scopes/helpers.py @@ -8,35 +8,31 @@ from typing import cast from sqlbuild._native import ( - build_scope_lookup, - get_placement_validated_scope_index, -) -from sqlbuild.compiler.discovery.models import ( - DiscoveredProjectInputs, - DiscoveredSqlModelFile, - EnumDeclaration, -) -from sqlbuild.compiler.scopes.models import ( ConstantMetadata, DeclarationIdentity, + DeclarationKind, DeclarationRecord, EnumMemberMetadata, EnumMetadata, GrantRecord, MacroMetadata, OwnershipRoot, + OwnershipRootKind, ResourceIdentity, + ResourceKind, ResourceRecord, ScopeCompleteness, ScopeIndex, + ScopeKind, ScopeLookup, UsageRecord, + build_scope_lookup, + get_placement_validated_scope_index, ) -from sqlbuild.compiler.scopes.types import ( - DeclarationKind, - OwnershipRootKind, - ResourceKind, - ScopeKind, +from sqlbuild.compiler.discovery.models import ( + DiscoveredProjectInputs, + DiscoveredSqlModelFile, + EnumDeclaration, ) from sqlbuild.spec.contracts.models import LocalConfig, ProjectConfig diff --git a/tests/unit/src/sqlbuild/compiler/scopes/test_builder_and_visibility.py b/tests/unit/src/sqlbuild/compiler/scopes/test_builder_and_visibility.py index 9fbe192b5e..715dba24c5 100644 --- a/tests/unit/src/sqlbuild/compiler/scopes/test_builder_and_visibility.py +++ b/tests/unit/src/sqlbuild/compiler/scopes/test_builder_and_visibility.py @@ -7,6 +7,9 @@ import pytest from sqlbuild._native import ( + DeclarationIdentity, + DeclarationKind, + DeclarationRecord, DiscoveredAuditBlock, DiscoveredAuditFile, DiscoveredConstantFile, @@ -15,7 +18,19 @@ DiscoveredSqlHookFile, DiscoveredSqlTestBlock, EnumMember, + InaccessibleReason, + OwnershipRoot, + ResourceIdentity, + ResourceKind, + ResourceRecord, + ScopeDiagnosticCode, + ScopeIndex, + ScopeKind, + ScopeLookup, + ScopeTargetQuery, ScopeValidationError, + VisibilityReason, + VisibilityResolution, build_scope_index, build_scope_lookup, normalize_sql_value, @@ -32,25 +47,6 @@ DiscoveredSqlTestFile, EnumDeclaration, ) -from sqlbuild.compiler.scopes.models import ( - DeclarationIdentity, - DeclarationRecord, - OwnershipRoot, - ResourceIdentity, - ResourceRecord, - ScopeIndex, - ScopeLookup, - ScopeTargetQuery, - VisibilityResolution, -) -from sqlbuild.compiler.scopes.types import ( - DeclarationKind, - InaccessibleReason, - ResourceKind, - ScopeDiagnosticCode, - ScopeKind, - VisibilityReason, -) from sqlbuild.spec.contracts.models import SourceEntry from tests.unit.src.sqlbuild.compiler.scopes._test_types import ( ExpectedBooleanCase, diff --git a/tests/unit/src/sqlbuild/compiler/scopes/test_identities.py b/tests/unit/src/sqlbuild/compiler/scopes/test_identities.py index 696d092b9c..915a558a8e 100644 --- a/tests/unit/src/sqlbuild/compiler/scopes/test_identities.py +++ b/tests/unit/src/sqlbuild/compiler/scopes/test_identities.py @@ -5,19 +5,17 @@ import pytest from sqlbuild._native import ( - InvalidQualifiedIdentityError, - build_scope_lookup, - explain_scope_declaration, - query_scope_report, -) -from sqlbuild.compiler.scopes.models import ( DeclarationExplanation, DeclarationIdentity, + DeclarationKind, + InvalidQualifiedIdentityError, ScopeDiagnostic, ScopeLookup, ScopeReport, + build_scope_lookup, + explain_scope_declaration, + query_scope_report, ) -from sqlbuild.compiler.scopes.types import DeclarationKind from tests.unit.src.sqlbuild.compiler.scopes._test_types import ( IdentityReportCase, InvalidIdentityCase, diff --git a/tests/unit/src/sqlbuild/compiler/scopes/test_lookup_and_metadata.py b/tests/unit/src/sqlbuild/compiler/scopes/test_lookup_and_metadata.py index 700a8bf189..af0ae86914 100644 --- a/tests/unit/src/sqlbuild/compiler/scopes/test_lookup_and_metadata.py +++ b/tests/unit/src/sqlbuild/compiler/scopes/test_lookup_and_metadata.py @@ -8,20 +8,19 @@ import pytest -from sqlbuild._native import build_scope_lookup, scope_metadata_projection -from sqlbuild.compiler.scopes.models import ( +from sqlbuild._native import ( DeclarationIdentity, + DeclarationKind, DeclarationRecord, OwnershipRoot, ResourceIdentity, + ResourceKind, ResourceRecord, ScopeIndex, - ScopeLookup, -) -from sqlbuild.compiler.scopes.types import ( - DeclarationKind, - ResourceKind, ScopeKind, + ScopeLookup, + build_scope_lookup, + scope_metadata_projection, ) from tests.unit.src.sqlbuild.compiler.scopes._test_types import ( DeclarationContainerProjectionCase, @@ -133,7 +132,7 @@ def test_given_frozen_scope_record_when_mutating_field_then_record_is_immutable( record: DeclarationRecord = scope_index().declarations[0] with pytest.raises(test_case.expected_error): - record.path = "changed.sql" # ty: ignore[invalid-assignment] + record.path = "changed.sql" @pytest.mark.parametrize( diff --git a/tests/unit/src/sqlbuild/compiler/scopes/test_offline_cache.py b/tests/unit/src/sqlbuild/compiler/scopes/test_offline_cache.py index dcf794f89a..3931dadda7 100644 --- a/tests/unit/src/sqlbuild/compiler/scopes/test_offline_cache.py +++ b/tests/unit/src/sqlbuild/compiler/scopes/test_offline_cache.py @@ -10,18 +10,17 @@ import pytest from sqlbuild._native import ( - compiler_cache_directory, - load_or_build_scope_index, - scope_index_fingerprint, -) -from sqlbuild.compiler.scopes.models import DeclarationIdentity, ScopeIndex -from sqlbuild.compiler.scopes.types import ( + DeclarationIdentity, DeclarationKind, - DiagnosticSeverity, GrantKind, ResourceKind, ScopeDiagnosticCode, + ScopeIndex, + compiler_cache_directory, + load_or_build_scope_index, + scope_index_fingerprint, ) +from sqlbuild._native import ScopeDiagnosticSeverity as DiagnosticSeverity from tests.unit.src.sqlbuild.compiler.scopes._test_types import ( CacheFaultCase, FingerprintMutationCase, diff --git a/tests/unit/src/sqlbuild/compiler/scopes/test_placement.py b/tests/unit/src/sqlbuild/compiler/scopes/test_placement.py index 3d64124671..fb19bc09fb 100644 --- a/tests/unit/src/sqlbuild/compiler/scopes/test_placement.py +++ b/tests/unit/src/sqlbuild/compiler/scopes/test_placement.py @@ -5,28 +5,24 @@ import pytest from sqlbuild._native import ( - ScopeValidationError, - get_placement_validated_scope_index, - validate_scope_index, -) -from sqlbuild.compiler.scopes.models import ( DeclarationIdentity, + DeclarationKind, DeclarationRecord, OwnershipRoot, ResourceIdentity, + ResourceKind, ResourceRecord, ScopeCompleteness, ScopeDiagnostic, - ScopeIndex, - UsageRecord, -) -from sqlbuild.compiler.scopes.types import ( - DeclarationKind, - DiagnosticSeverity, - ResourceKind, ScopeDiagnosticCode, + ScopeIndex, ScopeKind, + ScopeValidationError, + UsageRecord, + get_placement_validated_scope_index, + validate_scope_index, ) +from sqlbuild._native import ScopeDiagnosticSeverity as DiagnosticSeverity from tests.unit.src.sqlbuild.compiler.scopes._test_types import ( DeclarationChainPlacementCase, DiamondLadderPlacementCase, diff --git a/tests/unit/src/sqlbuild/compiler/scopes/test_relocation.py b/tests/unit/src/sqlbuild/compiler/scopes/test_relocation.py index 0a508e3933..7fc3ac05bd 100644 --- a/tests/unit/src/sqlbuild/compiler/scopes/test_relocation.py +++ b/tests/unit/src/sqlbuild/compiler/scopes/test_relocation.py @@ -5,13 +5,11 @@ import pytest from sqlbuild._native import ( - relocate_declarations_for_move, -) -from sqlbuild.compiler.scopes.models import ( DeclarationRecord, ResourceIdentity, + ResourceKind, + relocate_declarations_for_move, ) -from sqlbuild.compiler.scopes.types import ResourceKind from tests.unit.src.sqlbuild.compiler.scopes._test_types import RelocationCase from tests.unit.src.sqlbuild.compiler.scopes.helpers import relocation_index diff --git a/tests/unit/src/sqlbuild/compiler/scopes/test_reports.py b/tests/unit/src/sqlbuild/compiler/scopes/test_reports.py index 79ea620d89..8bad2dd96b 100644 --- a/tests/unit/src/sqlbuild/compiler/scopes/test_reports.py +++ b/tests/unit/src/sqlbuild/compiler/scopes/test_reports.py @@ -10,28 +10,24 @@ import pytest from sqlbuild._native import ( - browse_scope_folders, - build_scope_lookup, - explain_scope_declaration, - list_scope_declarations, - preview_scope_move, - query_scope_report, - serialize_scope_report, -) -from sqlbuild.compiler.scopes.models import ( + DeclarationKind, OwnershipRoot, + ResourceKind, ResourceRecord, ScopeBrowseResult, + ScopeDiagnosticCode, ScopeIndex, ScopeListResult, ScopeLookup, ScopeReport, ScopeReportFilters, -) -from sqlbuild.compiler.scopes.types import ( - DeclarationKind, - ResourceKind, - ScopeDiagnosticCode, + browse_scope_folders, + build_scope_lookup, + explain_scope_declaration, + list_scope_declarations, + preview_scope_move, + query_scope_report, + serialize_scope_report, ) from tests.unit.src.sqlbuild.compiler.scopes._test_types import ( ExpectedBooleanCase, diff --git a/tests/unit/src/sqlbuild/rule_engine/_helpers/engine/helpers.py b/tests/unit/src/sqlbuild/rule_engine/_helpers/engine/helpers.py index 2359be6808..a97386c269 100644 --- a/tests/unit/src/sqlbuild/rule_engine/_helpers/engine/helpers.py +++ b/tests/unit/src/sqlbuild/rule_engine/_helpers/engine/helpers.py @@ -22,6 +22,7 @@ RuleExemption, RuleIgnore, RulesCacheConfig, + ScopeIndex, build_rule_catalogue, install_rules_skills, project_with, @@ -31,7 +32,6 @@ from sqlbuild.compiler.compile.models import CompiledObjectKey, CompiledSqlTest from sqlbuild.compiler.compile.types import CompiledProjectView, CompiledResourceType, SqlTestMode from sqlbuild.compiler.discovery.models import DiscoveredSqlTestFile -from sqlbuild.compiler.scopes.models import ScopeIndex from sqlbuild.rule_engine.constants import MIN_CUSTOM_RULE_TEST_CASES from sqlbuild.rule_engine.models import Finding, Rule, RulesConfig, RulesResult from tests.unit.src.sqlbuild.rule_engine._helpers.engine._test_types import CustomRuleTestCase diff --git a/tests/unit/src/sqlbuild/rule_engine/_helpers/engine/test_scope_facts.py b/tests/unit/src/sqlbuild/rule_engine/_helpers/engine/test_scope_facts.py index c35037745b..50a7f3336b 100644 --- a/tests/unit/src/sqlbuild/rule_engine/_helpers/engine/test_scope_facts.py +++ b/tests/unit/src/sqlbuild/rule_engine/_helpers/engine/test_scope_facts.py @@ -5,8 +5,12 @@ import pytest -from sqlbuild._native import scope_metadata_projection -from sqlbuild.compiler.scopes.models import DeclarationRecord, ScopeCompleteness, ScopeIndex +from sqlbuild._native import ( + DeclarationRecord, + ScopeCompleteness, + ScopeIndex, + scope_metadata_projection, +) from tests.unit.src.sqlbuild.compiler.scopes.helpers import scope_index from tests.unit.src.sqlbuild.rule_engine._helpers.engine._test_types import ScopePayloadTestCase from tests.unit.src.sqlbuild.rule_engine._helpers.engine.helpers import ( From 3f408b40e6d75c0a9ef035c9cf9facafd9ae475b Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Thu, 15 Oct 2026 15:33:08 +0100 Subject: [PATCH 02/48] refactor: host configuration, measurement, SQL value, cursor value and lexical syntax shapes natively --- .../src/stage_capture/constants.rs | 354 +++- crates/sqlbuild-config/src/project/models.rs | 2 +- .../header_metadata/_helpers/thresholds.rs | 2 +- .../src/header_metadata/constants.rs | 4 +- .../main/check_measurement_contract.rs | 21 + .../main/check_threshold_bound.rs | 52 + .../main/check_threshold_pair.rs | 12 + .../src/header_metadata/main/mod.rs | 3 + .../src/header_metadata/models.rs | 8 + .../src/header_metadata/tests.rs | 2 + .../tests/test_measurement_checks.rs | 130 ++ .../src/header_metadata/tests/test_types.rs | 16 + .../_helpers/analysis/explicit_references.rs | 2 +- .../bindings/_helpers/auditing/identity.rs | 2 +- .../_helpers/build_command/defer_clone.rs | 2 +- .../_helpers/build_command/invocation.rs | 2 +- .../src/bindings/_helpers/cli/entry.rs | 2 +- .../_helpers/clone/clone_execution.rs | 2 +- .../command_runtime/warehouse_scope.rs | 2 +- .../_helpers/command_runtime/warehouses.rs | 15 +- .../compile_attachment/effective_config.rs | 2 +- .../compile_attachment/model_config.rs | 2 +- .../compile_attachment/model_inputs.rs | 2 +- .../compile_attachment/storage_policies.rs | 4 +- .../_helpers/compile_records/constants.rs | 9 +- .../_helpers/compiled_project/attachment.rs | 2 +- .../_helpers/compiled_project/loop_classes.rs | 4 +- .../_helpers/compiler_enums/constants.rs | 154 ++ .../_helpers/compiler_records/constants.rs | 191 +++ .../_helpers/contract_records/constants.rs | 779 +++++++++ .../bindings/_helpers/contract_records/mod.rs | 1 + .../contracts/contract_diagnostics.rs | 2 +- .../_helpers/contracts/project_config.rs | 23 +- .../_helpers/contracts/spec_resolution.rs | 2 +- .../bindings/_helpers/contracts/sql_values.rs | 4 +- .../bindings/_helpers/cost_command/bounds.rs | 2 +- .../bindings/_helpers/cursor/conversion.rs | 2 +- .../src/bindings/_helpers/cursor/durations.rs | 4 +- .../src/bindings/_helpers/cursor/intervals.rs | 2 +- .../_helpers/dbt_interop/pipeline_phases.rs | 7 +- .../_helpers/dbt_interop/profile_load.rs | 2 +- .../_helpers/debug_command/run_debug.rs | 2 +- .../bindings/_helpers/diff_report/query.rs | 2 +- .../_helpers/discovery/declaration_records.rs | 4 +- .../_helpers/discovery/description_inputs.rs | 2 +- .../_helpers/discovery/header_values.rs | 2 +- .../_helpers/discovery/model_file_record.rs | 2 +- .../_helpers/discovery/scope_snapshot.rs | 2 +- .../_helpers/discovery/sql_test_blocks.rs | 2 +- .../_helpers/fingerprints/latest_rows.rs | 2 +- .../_helpers/fingerprints/storage_io.rs | 2 +- .../_helpers/freshness/observation.rs | 2 +- .../bindings/_helpers/freshness/support.rs | 6 +- .../_helpers/graph/python_lifecycle.rs | 2 +- .../_helpers/graph/python_node_graph.rs | 2 +- .../_helpers/hand_built/compiled_model.rs | 2 +- .../bindings/_helpers/load/load_selection.rs | 2 +- .../src/bindings/_helpers/mod.rs | 1 + .../_helpers/model_config/contract_classes.rs | 10 +- .../_helpers/model_config/header_metadata.rs | 15 +- .../_helpers/plan_output/cursor_details.rs | 2 +- .../bindings/_helpers/plan_output/display.rs | 2 +- .../planner_migrations/model_planning.rs | 2 +- .../planner_migrations/old_name_entries.rs | 4 +- .../planner_migrations/old_name_objects.rs | 4 +- .../_helpers/planner_models/records.rs | 2 +- .../_helpers/planner_phases/constants.rs | 2 +- .../_helpers/planner_records/constants.rs | 94 ++ .../planner_resolution/cursor_policies.rs | 4 +- .../planner_resolution/cursor_ranges.rs | 2 +- .../planner_resolution/cursor_safety.rs | 21 +- .../planner_resolution/cursor_windows.rs | 2 +- .../planner_resolution/future_safety.rs | 2 +- .../planner_resolution/maximum_safety.rs | 2 +- .../bindings/_helpers/planner_scope/values.rs | 2 +- .../_helpers/planner_strategy/microbatches.rs | 4 +- .../_helpers/planner_warehouse/constants.rs | 2 +- .../_helpers/record_behaviours/auditing.rs | 106 ++ .../_helpers/record_behaviours/config.rs | 8 + .../_helpers/record_behaviours/constants.rs | 65 +- .../_helpers/record_behaviours/durations.rs | 147 ++ .../_helpers/record_behaviours/freshness.rs | 40 + .../_helpers/record_behaviours/lexical.rs | 103 ++ .../_helpers/record_behaviours/mod.rs | 7 + .../record_behaviours/python_nodes.rs | 50 + .../_helpers/record_behaviours/values.rs | 45 + .../bindings/_helpers/records/constants.rs | 4 +- .../src/bindings/_helpers/records/fields.rs | 2 + .../src/bindings/_helpers/records/models.rs | 4 + .../_helpers/render/macro_expansion_result.rs | 4 +- .../bindings/_helpers/scopes/scope_loading.rs | 2 +- .../_helpers/semantic_checks/resource_sql.rs | 2 +- .../_helpers/semantic_checks/stage_objects.rs | 2 +- .../_helpers/sql_test_glue/compiled_tests.rs | 2 +- .../_helpers/sql_test_glue/plan_errors.rs | 2 +- .../bindings/_helpers/state_rows/constants.rs | 2 +- .../src/sql_scan/constants.rs | 8 + .../src/sql_scan/main/lexical_syntax_key.rs | 30 + .../sqlbuild-sqltext/src/sql_scan/main/mod.rs | 3 + .../main/reads_differently_from_generic.rs | 26 + .../main/unsupported_line_comment_prefixes.rs | 16 + crates/sqlbuild-sqltext/src/sql_scan/tests.rs | 2 + .../src/sql_scan/tests/helpers.rs | 12 + .../src/sql_scan/tests/test_lexical_rules.rs | 82 + .../src/sql_scan/tests/test_types.rs | 16 +- python-boundary.toml | 2 +- src/sqlbuild/_native.pyi | 1483 ++++++++++++++++- .../adapter/contract/_helpers/seed_csv.py | 2 +- .../adapter/contract/classes/base_adapter.py | 12 +- .../contract/classes/duckdb_backed_adapter.py | 5 +- .../contract/classes/materialization.py | 2 +- .../contract/classes/strict_adapter.py | 3 +- .../contract/main/normalize_seed_csv_value.py | 2 +- .../state_sql/_helpers/source_freshness.py | 3 +- .../adapter/state_sql/_helpers/table_ddl.py | 2 +- ...der_insert_source_freshness_records_sql.py | 2 +- .../main/render_state_table_create_sql.py | 2 +- .../bigquery/classes/bigquery_adapter.py | 12 +- .../databricks/classes/databricks_adapter.py | 5 +- .../postgres/classes/postgres_adapter.py | 5 +- .../snowflake/classes/snowflake_adapter.py | 14 +- .../sqlserver/classes/sqlserver_adapter.py | 12 +- src/sqlbuild/assets.py | 2 +- src/sqlbuild/assets.pyi | 2 +- src/sqlbuild/audits.py | 8 +- .../_helpers/freshness/source_freshness.py | 2 +- src/sqlbuild/compiler/auditing/main/above.py | 2 +- src/sqlbuild/compiler/auditing/main/below.py | 2 +- .../compiler/auditing/main/outside.py | 2 +- src/sqlbuild/compiler/auditing/models.py | 122 -- .../_helpers/diagnostics/python_resources.py | 3 +- .../_helpers/macro_bridge/call_store.py | 2 +- .../compile/_helpers/render/user_macros.py | 4 +- src/sqlbuild/compiler/compile/exceptions.py | 2 +- src/sqlbuild/compiler/compile/models.py | 19 +- src/sqlbuild/compiler/compile/types.py | 21 +- src/sqlbuild/compiler/discovery/models.py | 18 +- src/sqlbuild/compiler/fingerprints/models.py | 41 - src/sqlbuild/compiler/planner/models.py | 40 +- .../python_nodes/_helpers/identity.py | 7 +- .../python_nodes/main/_hook_identities.py | 2 +- .../compiler/python_nodes/main/identity.py | 2 +- src/sqlbuild/compiler/python_nodes/models.py | 108 -- .../compiler/source_freshness/constants.py | 2 +- .../compiler/source_freshness/models.py | 43 - .../compiler/sql_analysis/constants.py | 6 - src/sqlbuild/compiler/sql_analysis/models.py | 84 - src/sqlbuild/cost/constants.py | 3 - src/sqlbuild/cursor_algebra/constants.py | 7 - src/sqlbuild/cursor_algebra/models.py | 143 -- src/sqlbuild/cursor_algebra/types.py | 2 +- .../executor/audit_results/constants.py | 2 +- .../executor/auditing/_helpers/execution.py | 2 +- .../auditing/_helpers/result_projection.py | 3 +- .../executor/auditing/main/_execute.py | 2 +- .../main/_execute_reporting_errors.py | 2 +- src/sqlbuild/executor/auditing/models.py | 3 +- .../executor/build/_helpers/end_audits.py | 3 +- .../build/_helpers/external_source_loads.py | 2 +- .../executor/build/_helpers/retention.py | 2 +- .../executor/build/_helpers/scheduler.py | 2 +- .../executor/build/_helpers/source_node.py | 3 +- .../executor/build/classes/build_scheduler.py | 3 +- src/sqlbuild/executor/build/models.py | 2 +- .../executor/clone/_helpers/fingerprinting.py | 8 +- .../executor/clone/_helpers/retention.py | 2 +- .../_build_destination_transient_models.py | 2 +- .../executor/diff/_helpers/query_artifacts.py | 9 +- .../executor/functions/main/_execute.py | 2 +- .../executor/janitor_events/constants.py | 2 +- .../executor/load/_helpers/cursors.py | 2 +- .../executor/load/_helpers/dag_runtime.py | 2 +- .../executor/load/_helpers/execution.py | 2 +- .../executor/load/_helpers/external.py | 2 +- .../load/_helpers/loader_invocation.py | 3 +- .../executor/load/_helpers/relation_refs.py | 2 +- src/sqlbuild/executor/load/_helpers/rows.py | 2 +- .../executor/load/_helpers/scheduling.py | 2 +- src/sqlbuild/executor/load/_helpers/schema.py | 2 +- .../executor/load/_helpers/staging.py | 3 +- .../load/main/_build_execution_indexes.py | 2 +- src/sqlbuild/executor/load/main/_execute.py | 4 +- .../executor/load/main/_resource_kind.py | 2 +- .../executor/load/main/_skipped_result.py | 2 +- src/sqlbuild/executor/load/main/run.py | 2 +- src/sqlbuild/executor/load/models.py | 2 +- src/sqlbuild/executor/load/types.py | 2 +- .../migrations/_helpers/old_name_views.py | 1 - .../executor/node_results/constants.py | 2 +- .../executor/pipeline/_helpers/scenario.py | 2 +- .../pipeline/_helpers/schema_preparation.py | 2 +- .../executor/pipeline/_helpers/settings.py | 2 +- src/sqlbuild/executor/pipeline/main/run.py | 3 +- .../python_nodes/_helpers/fingerprinting.py | 4 +- .../_helpers/ingress_execution.py | 14 +- .../python_nodes/_helpers/lifecycle_nodes.py | 6 +- .../python_nodes/_helpers/python_checks.py | 7 +- .../read_side_python_execution_tracker.py | 3 +- .../executor/python_nodes/main/_checks.py | 2 +- .../python_nodes/main/_fingerprinting.py | 2 +- .../executor/python_nodes/main/_ingress.py | 3 +- .../executor/python_nodes/main/read_side.py | 2 +- src/sqlbuild/executor/python_nodes/types.py | 2 +- .../run/_helpers/execution/final_audits.py | 3 +- .../executor/run/_helpers/execution/hooks.py | 4 +- .../run/_helpers/execution/staging.py | 2 +- .../run/_helpers/materializations/custom.py | 2 +- .../_helpers/materializations/incremental.py | 4 +- .../_helpers/materializations/microbatch.py | 8 +- .../run/_helpers/materializations/snapshot.py | 3 +- .../run/_helpers/materializations/view.py | 2 +- .../_helpers/reuse/fingerprint_metadata.py | 3 +- .../run/_helpers/reuse/fingerprinting.py | 3 +- .../run/_helpers/validation/contracts.py | 3 +- .../run/_helpers/validation/cursor_bounds.py | 12 +- src/sqlbuild/executor/run/main/_execute.py | 3 +- src/sqlbuild/executor/run/models.py | 12 +- .../scenario/_helpers/local/execution.py | 3 +- src/sqlbuild/executor/scenario/models.py | 2 +- .../executor/seed/_helpers/fingerprinting.py | 3 +- src/sqlbuild/executor/testing/models.py | 2 +- .../dbt/_helpers/profile/render.py | 2 +- src/sqlbuild/integrations/dbt/models.py | 33 - src/sqlbuild/integrations/dbt/types.py | 12 - .../integrations/ingestr/_helpers/command.py | 3 +- .../integrations/ingestr/main/_loaders.py | 1 - src/sqlbuild/loaders.pyi | 2 +- src/sqlbuild/microbatches/_helpers/sql.py | 3 +- src/sqlbuild/microbatches/constants.py | 2 +- .../_helpers/column_normalization.py | 2 +- .../python_nodes/_helpers/loader_authoring.py | 4 +- .../_helpers/loader_normalization.py | 2 +- .../python_nodes/main/_apply_loader.py | 2 +- src/sqlbuild/python_nodes/main/apply_asset.py | 2 +- src/sqlbuild/python_nodes/models.py | 4 +- .../runtime/event_exporting/constants.py | 1 + .../main/lifecycle_event_sink.py | 2 +- src/sqlbuild/spec/contracts/constants.py | 2 +- src/sqlbuild/spec/contracts/models.py | 634 ------- src/sqlbuild/spec/contracts/types.py | 111 -- src/sqlbuild/sql_values/models.py | 37 - src/sqlbuild/sql_values/types.py | 36 - .../cli/commands/main/janitor/helpers.py | 4 +- .../sqlbuild/adapters/bigquery/test_client.py | 5 +- .../adapters/databricks/test_client.py | 5 +- .../sqlbuild/adapters/duckdb/_test_types.py | 2 +- .../sqlbuild/adapters/duckdb/test_client.py | 2 +- .../snapshot_sql_rendering/helpers.py | 2 +- .../adapters/snowflake/_test_types.py | 2 +- .../adapters/snowflake/test_client.py | 2 +- .../snowflake/test_table_type_conversion.py | 3 +- .../cli/commands/main/test_load_command.py | 2 +- .../sqlbuild/cli/commands/main/test_rules.py | 2 +- .../test_measurement_integration_envelope.py | 5 +- .../compiler/analysis_session/helpers.py | 2 +- .../sqlbuild/compiler/attachments/helpers.py | 8 +- .../compiler/auditing/main/_test_types.py | 2 +- .../compiler/auditing/main/test_rendering.py | 3 +- .../_helpers/refs/test_native_references.py | 3 +- .../compile/native_model_config/helpers.py | 2 +- .../compiler/compiled_project/helpers.py | 9 +- .../sqlbuild/compiler/contracts/helpers.py | 12 +- .../compiler/fingerprints/main/_test_types.py | 2 +- .../compiler/fingerprints/main/helpers.py | 3 +- .../fingerprints/main/test_read_write.py | 3 +- .../sqlbuild/compiler/model_config/helpers.py | 17 +- .../pipeline/test_python_source_reads.py | 2 +- .../compiler/planner/_helpers/_test_types.py | 4 +- .../compiler/planner/_helpers/helpers.py | 10 +- .../planner/_helpers/resolve/_test_types.py | 2 +- .../planner/_helpers/resolve/helpers.py | 2 +- .../resolve/test_adapter_overrides.py | 3 +- .../planner/_helpers/resolve/test_resolve.py | 3 +- .../planner/_helpers/test_audit_entry.py | 3 +- .../_helpers/test_warehouse_snapshot.py | 7 +- .../sqlbuild/compiler/planner/main/helpers.py | 10 +- .../planner/main/test_build_execution_plan.py | 18 +- .../src/sqlbuild/compiler/scopes/helpers.py | 2 +- .../source_freshness/main/_test_types.py | 3 +- .../source_freshness/main/test_read_write.py | 2 +- .../sqlbuild/executor/build/_test_types.py | 2 +- .../src/sqlbuild/executor/build/helpers.py | 8 +- .../test_capped_microbatch_dependencies.py | 2 +- .../executor/run/incremental/_test_types.py | 2 +- .../incremental/test_incremental_execution.py | 3 +- .../executor/run/microbatch/_test_types.py | 3 +- .../microbatch/test_microbatch_execution.py | 3 +- .../test_microbatch_observability.py | 2 +- .../test_microbatch_reconciliation.py | 4 +- .../_helpers/coverage/test_discovery.py | 3 +- .../_helpers/coverage/test_render.py | 10 +- .../adapter/contract/_helpers/_test_types.py | 2 +- .../contract/_helpers/test_seed_csv.py | 2 +- .../contract/typed_sql_rendering/helpers.py | 3 +- .../test_typed_sql_rendering.py | 2 +- .../state_sql/test_source_freshness.py | 2 +- .../_helpers/build_planning/_test_types.py | 2 +- .../build_planning/test_execution_limits.py | 3 +- .../build_planning/test_full_refresh.py | 8 +- .../build_planning/test_plan_policies.py | 3 +- .../build_planning/test_retention_decrease.py | 8 +- .../build_planning/test_table_type.py | 8 +- .../commands/_helpers/check/test_planning.py | 9 +- .../cli/commands/_helpers/cost/helpers.py | 3 +- .../_helpers/freshness/_test_types.py | 3 +- .../commands/_helpers/freshness/helpers.py | 17 +- .../sqlbuild/cli/commands/_helpers/helpers.py | 2 +- .../_helpers/load_selection/helpers.py | 7 +- .../_helpers/load_selection/test_selection.py | 5 +- .../commands/_helpers/runtime/_test_types.py | 6 +- .../cli/commands/_helpers/runtime/helpers.py | 7 +- .../_helpers/runtime/test_connection.py | 10 +- .../_helpers/runtime/test_warehouses.py | 5 +- .../cli/commands/main/compile/helpers.py | 2 +- .../commands/main/compile/test_adapters.py | 9 +- .../cli/commands/main/execution/helpers.py | 2 +- .../test_execution_result_document.py | 16 +- .../test_integration_result_measurements.py | 4 +- .../classes/test_terminal_event_index.py | 3 +- .../test_execution_protocol_v1.py | 3 +- .../sqlbuild/cli/output/main/plan/helpers.py | 5 +- .../cli/output/main/plan/test_plan_text.py | 2 +- .../output/main/plan_json/test_plan_json.py | 3 +- .../src/sqlbuild/compiler/auditing/helpers.py | 2 +- .../compiler/auditing/main/_test_types.py | 2 +- .../compiler/auditing/main/helpers.py | 3 +- .../main/test_evaluate_measurement.py | 6 +- .../compiler/auditing/main/test_identity.py | 4 +- .../main/test_parse_audit_instance.py | 2 +- .../main/test_parse_audit_instances.py | 3 +- .../compiler/auditing/main/test_rendering.py | 3 +- .../sqlbuild/compiler/auditing/test_models.py | 6 +- .../compiler/compile/_helpers/_test_types.py | 13 +- .../_helpers/analysis/test_dynamic_pivot.py | 2 +- .../compile/_helpers/config/_test_types.py | 3 +- .../compile/_helpers/config/helpers.py | 18 +- .../_helpers/config/test_table_type.py | 9 +- .../compile/_helpers/test_audit_factories.py | 8 +- .../_helpers/test_declaration_scopes.py | 2 +- .../_helpers/test_microbatch_redesign.py | 3 +- .../_helpers/test_model_config_validation.py | 9 +- .../compile/_helpers/test_model_schemas.py | 4 +- .../_helpers/test_parameterized_sql_tests.py | 10 +- .../_helpers/test_rebind_declaration_scope.py | 2 +- .../compile/_helpers/test_sql_references.py | 3 +- .../sqlbuild/compiler/compile/_test_types.py | 4 +- .../compile/test_effective_runtime.py | 7 +- .../sqlbuild/compiler/compile/test_main.py | 26 +- .../_helpers/test_repository_edits.py | 8 +- .../sqlbuild/compiler/contracts/helpers.py | 21 +- .../compiler/contracts/test_validate.py | 3 +- .../src/sqlbuild/compiler/dag/main/helpers.py | 12 +- .../discovery/_helpers/_test_types.py | 10 +- .../compiler/discovery/_helpers/helpers.py | 3 +- .../_helpers/test_filesystem_python_nodes.py | 1 - .../_helpers/test_sql_declarations.py | 2 +- .../discovery/_helpers/test_sql_models.py | 2 +- .../discovery/_helpers/test_sql_tests.py | 2 +- .../_helpers/test_target_warehouses.py | 3 +- .../discovery/_helpers/test_yml_project.py | 10 +- .../discovery/_helpers/test_yml_schema.py | 7 +- .../discovery/_helpers/test_yml_sources.py | 4 +- .../sqlbuild/compiler/discovery/helpers.py | 6 +- .../compiler/fingerprints/main/helpers.py | 3 +- .../main/shared/_helpers/test_sql.py | 8 +- .../compiler/fingerprints/main/test_read.py | 9 +- .../compiler/fingerprints/main/test_write.py | 3 +- .../src/sqlbuild/compiler/lineage/helpers.py | 4 +- .../compiler/macro_bridge/_helpers/helpers.py | 3 +- .../src/sqlbuild/compiler/manifest/helpers.py | 12 +- .../sqlbuild/compiler/manifest/test_main.py | 3 +- .../compiler/pipeline/_helpers/helpers.py | 3 +- .../_helpers/target_validation/_test_types.py | 2 +- .../_helpers/target_validation/test_main.py | 12 +- .../compiler/pipeline/_helpers/test_clone.py | 4 +- .../_helpers/test_deferred_locations.py | 3 +- .../_helpers/test_python_plan_entries.py | 6 +- .../compiler/pipeline/main/helpers.py | 3 +- .../pipeline/main/test_relation_targets.py | 9 +- .../compiler/planner/_helpers/_test_types.py | 6 +- .../planner/_helpers/changes/_test_helpers.py | 5 +- .../planner/_helpers/changes/_test_types.py | 2 +- .../planner/_helpers/changes/test_detect.py | 2 +- .../changes/test_reference_renames.py | 2 +- .../planner/_helpers/changes/test_schema.py | 2 +- .../compiler/planner/_helpers/helpers.py | 24 +- .../test_column_migration_candidates.py | 2 +- .../planner/_helpers/planning/_test_types.py | 3 +- .../planner/_helpers/planning/helpers.py | 18 +- .../_helpers/planning/test_retention.py | 16 +- .../planner/_helpers/resolve/_test_types.py | 2 +- .../planner/_helpers/resolve/test_cursor.py | 4 +- .../planner/_helpers/resolve/test_hooks.py | 2 +- .../planner/_helpers/resolve/test_sources.py | 3 +- .../_helpers/sql_test_assembly/helpers.py | 2 +- .../planner/_helpers/test_audit_entry.py | 3 +- .../_helpers/test_fixture_planning_work.py | 2 +- .../planner/_helpers/test_microbatch_limit.py | 2 +- .../planner/_helpers/test_scenario_cli.py | 2 +- .../_helpers/test_scenario_relations.py | 2 +- .../planner/_helpers/test_seed_identity.py | 3 +- .../_helpers/test_selection_staleness.py | 2 +- .../planner/_helpers/test_source_columns.py | 2 +- .../planner/_helpers/test_source_freshness.py | 6 +- .../_helpers/test_source_load_nodes.py | 4 +- .../planner/_helpers/warehouse/_test_types.py | 2 +- .../planner/_helpers/warehouse/helpers.py | 2 +- .../warehouse/test_semantic_sources.py | 2 +- .../_helpers/warehouse/test_source_tables.py | 2 +- .../sqlbuild/compiler/planner/_test_types.py | 2 +- .../compiler/planner/main/_test_types.py | 2 +- .../sqlbuild/compiler/planner/main/helpers.py | 5 +- .../compiler/planner/main/identity/helpers.py | 3 +- .../planner/main/test_future_cursor_safety.py | 4 +- .../planner/main/test_maximum_start_safety.py | 9 +- .../main/test_static_command_planning.py | 2 +- .../sqlbuild/compiler/planner/test_models.py | 2 +- .../compiler/python_nodes/_helpers/helpers.py | 12 +- .../python_nodes/_helpers/test_identity.py | 2 +- .../python_nodes/_helpers/test_inventory.py | 16 +- .../_helpers/test_run_lifecycle.py | 8 +- .../_helpers/test_run_selectors.py | 3 +- .../python_nodes/_helpers/test_selectors.py | 2 +- .../_helpers/test_unified_selectors.py | 3 +- .../src/sqlbuild/compiler/scopes/helpers.py | 3 +- .../scopes/test_builder_and_visibility.py | 2 +- .../_helpers/test_age_policy.py | 3 +- .../source_freshness/main/_test_types.py | 3 +- .../configured_observation/_test_types.py | 2 +- .../test_observation.py | 6 +- .../compiler/source_freshness/main/helpers.py | 8 +- .../main/test_adapter_observation.py | 4 +- .../source_freshness/main/test_age_policy.py | 7 +- .../source_freshness/main/test_models.py | 9 +- .../source_freshness/main/test_observation.py | 6 +- .../source_freshness/main/test_planning.py | 13 +- .../source_freshness/main/test_read.py | 2 +- .../source_freshness/main/test_state.py | 5 +- .../sql_analysis/_helpers/_test_types.py | 2 +- .../compiler/sql_analysis/_helpers/helpers.py | 2 +- .../sql_analysis/_helpers/test_scanning.py | 2 +- .../main/test_cursor_algebra.py | 2 +- .../sqlbuild/cursor_algebra/test_duration.py | 2 +- .../executor/auditing/main/helpers.py | 2 +- .../executor/build/_helpers/helpers.py | 3 +- .../_helpers/test_external_source_loads.py | 2 +- .../executor/build/_helpers/test_scheduler.py | 4 +- .../clone/_helpers/fingerprinting/helpers.py | 3 +- .../fingerprinting/test_fingerprinting.py | 2 +- .../executor/clone/_helpers/helpers.py | 8 +- .../sqlbuild/executor/janitor/main/helpers.py | 10 +- .../src/sqlbuild/executor/load/_test_types.py | 2 +- .../sqlbuild/executor/load/test_cursors.py | 2 +- .../executor/load/test_dag_runtime.py | 2 +- .../sqlbuild/executor/load/test_execute.py | 4 +- .../src/sqlbuild/executor/load/test_rows.py | 2 +- .../src/sqlbuild/executor/load/test_run.py | 3 +- .../executor/pipeline/_helpers/helpers.py | 4 +- .../pipeline/_helpers/test_scenario.py | 2 +- .../executor/pipeline/main/helpers.py | 2 +- .../executor/pipeline/main/test_identity.py | 3 +- .../executor/python_nodes/_helpers/helpers.py | 12 +- .../_helpers/test_fingerprinting.py | 2 +- .../_helpers/test_ingress_execution.py | 3 +- .../_helpers/test_lifecycle_nodes.py | 2 +- .../_helpers/test_python_checks.py | 2 +- .../_helpers/test_read_side_execution.py | 2 +- .../executor/run/_helpers/_test_types.py | 2 +- .../sqlbuild/executor/run/_helpers/helpers.py | 2 +- .../executor/run/_helpers/test_contracts.py | 2 +- .../run/_helpers/test_cursor_bounds.py | 5 +- .../run/_helpers/test_fingerprint_metadata.py | 2 +- .../run/_helpers/test_snapshot_lifecycle.py | 2 +- .../executor/scenario/main/helpers.py | 2 +- .../sqlbuild/executor/seed/main/helpers.py | 2 +- .../integrations/dagster/test_resource.py | 8 +- .../sqlbuild/integrations/dbt/_test_types.py | 9 +- .../src/sqlbuild/integrations/dbt/helpers.py | 5 +- .../integrations/dbt/profile/helpers.py | 2 +- .../dbt/profile/test_normalize.py | 7 +- .../integrations/dbt/profile/test_render.py | 2 +- .../sqlbuild/integrations/dbt/test_config.py | 3 +- .../integrations/ingestr/test_command.py | 2 +- tests/unit/src/sqlbuild/lint/_test_types.py | 2 +- .../lint/test_adapter_lexical_syntax.py | 3 +- .../classes/shared_fact_encoder/helpers.py | 2 +- .../classes/test_dispatcher.py | 1 - .../runtime/event_exporting/test_sinks.py | 2 +- .../spec/contracts/main/_test_types.py | 6 +- .../spec/contracts/main/test_constants.py | 8 +- .../main/test_loader_destination_parts.py | 3 +- .../spec/contracts/main/test_targets.py | 15 +- .../sqlbuild/sql_values/test_normalization.py | 2 +- .../sqlbuild/sql_values/test_state_literal.py | 3 +- 494 files changed, 5031 insertions(+), 2396 deletions(-) create mode 100644 crates/sqlbuild-model-config/src/header_metadata/main/check_measurement_contract.rs create mode 100644 crates/sqlbuild-model-config/src/header_metadata/main/check_threshold_bound.rs create mode 100644 crates/sqlbuild-model-config/src/header_metadata/main/check_threshold_pair.rs create mode 100644 crates/sqlbuild-model-config/src/header_metadata/tests/test_measurement_checks.rs create mode 100644 crates/sqlbuild-python/src/bindings/_helpers/contract_records/constants.rs create mode 100644 crates/sqlbuild-python/src/bindings/_helpers/contract_records/mod.rs create mode 100644 crates/sqlbuild-python/src/bindings/_helpers/record_behaviours/auditing.rs create mode 100644 crates/sqlbuild-python/src/bindings/_helpers/record_behaviours/config.rs create mode 100644 crates/sqlbuild-python/src/bindings/_helpers/record_behaviours/durations.rs create mode 100644 crates/sqlbuild-python/src/bindings/_helpers/record_behaviours/freshness.rs create mode 100644 crates/sqlbuild-python/src/bindings/_helpers/record_behaviours/lexical.rs create mode 100644 crates/sqlbuild-python/src/bindings/_helpers/record_behaviours/python_nodes.rs create mode 100644 crates/sqlbuild-python/src/bindings/_helpers/record_behaviours/values.rs create mode 100644 crates/sqlbuild-sqltext/src/sql_scan/main/lexical_syntax_key.rs create mode 100644 crates/sqlbuild-sqltext/src/sql_scan/main/reads_differently_from_generic.rs create mode 100644 crates/sqlbuild-sqltext/src/sql_scan/main/unsupported_line_comment_prefixes.rs create mode 100644 crates/sqlbuild-sqltext/src/sql_scan/tests/test_lexical_rules.rs delete mode 100644 src/sqlbuild/compiler/auditing/models.py delete mode 100644 src/sqlbuild/compiler/fingerprints/models.py delete mode 100644 src/sqlbuild/compiler/python_nodes/models.py delete mode 100644 src/sqlbuild/compiler/source_freshness/models.py delete mode 100644 src/sqlbuild/compiler/sql_analysis/models.py delete mode 100644 src/sqlbuild/cursor_algebra/models.py delete mode 100644 src/sqlbuild/integrations/dbt/models.py delete mode 100644 src/sqlbuild/integrations/dbt/types.py delete mode 100644 src/sqlbuild/spec/contracts/models.py delete mode 100644 src/sqlbuild/spec/contracts/types.py delete mode 100644 src/sqlbuild/sql_values/models.py delete mode 100644 src/sqlbuild/sql_values/types.py diff --git a/crates/sqlbuild-cache/src/stage_capture/constants.rs b/crates/sqlbuild-cache/src/stage_capture/constants.rs index 85a6625b34..bb0c1a1f70 100644 --- a/crates/sqlbuild-cache/src/stage_capture/constants.rs +++ b/crates/sqlbuild-cache/src/stage_capture/constants.rs @@ -383,7 +383,11 @@ pub const CANONICAL_TYPES: [(&str, &str); 11] = [ /// Sequence types captured as their items. pub const DECODED_SEQUENCES: [&str; 1] = ["sqlbuild.compiler.compile.models:CompactLineageFacts"]; /// The Python model type a natively hosted compiler shape is captured as. -pub const HOSTED_SHAPE_TYPES: [(&str, &str); 257] = [ +pub const HOSTED_SHAPE_TYPES: [(&str, &str); 345] = [ + ( + "sqlbuild._native:AlignedInterval", + "sqlbuild.cursor_algebra.models:AlignedInterval", + ), ( "sqlbuild._native:AnalysisCacheContext", "sqlbuild.compiler.compile.models:AnalysisCacheContext", @@ -392,6 +396,18 @@ pub const HOSTED_SHAPE_TYPES: [(&str, &str); 257] = [ "sqlbuild._native:AttachedAuditGateEdge", "sqlbuild.compiler.graph.models:AttachedAuditGateEdge", ), + ( + "sqlbuild._native:AuditGateIdentity", + "sqlbuild.compiler.auditing.models:AuditGateIdentity", + ), + ( + "sqlbuild._native:AuditIdentity", + "sqlbuild.compiler.auditing.models:AuditIdentity", + ), + ( + "sqlbuild._native:AuthoredTimeTravelRetention", + "sqlbuild.spec.contracts.models:AuthoredTimeTravelRetention", + ), ( "sqlbuild._native:ClonePipelineConnection", "sqlbuild.compiler.pipeline.models:ClonePipelineConnection", @@ -404,10 +420,18 @@ pub const HOSTED_SHAPE_TYPES: [(&str, &str); 257] = [ "sqlbuild._native:ClonePipelineResult", "sqlbuild.compiler.pipeline.models:ClonePipelineResult", ), + ( + "sqlbuild._native:ClonePolicy", + "sqlbuild.spec.contracts.models:ClonePolicy", + ), ( "sqlbuild._native:CollectionRendering", "sqlbuild.sql_values.types:CollectionRendering", ), + ( + "sqlbuild._native:ColumnContractMode", + "sqlbuild.spec.contracts.types:ColumnContractMode", + ), ( "sqlbuild._native:ColumnLineage", "sqlbuild.compiler.lineage.models:ColumnLineage", @@ -528,6 +552,10 @@ pub const HOSTED_SHAPE_TYPES: [(&str, &str); 257] = [ "sqlbuild._native:ConstantMetadata", "sqlbuild.compiler.scopes.models:ConstantMetadata", ), + ( + "sqlbuild._native:ConstantsConfig", + "sqlbuild.spec.contracts.models:ConstantsConfig", + ), ( "sqlbuild._native:ContractAdoptionError", "sqlbuild.compiler.contract_adoption.exceptions:ContractAdoptionError", @@ -552,14 +580,26 @@ pub const HOSTED_SHAPE_TYPES: [(&str, &str); 257] = [ "sqlbuild._native:ContractValidationResult", "sqlbuild.compiler.contracts.models:ContractValidationResult", ), + ( + "sqlbuild._native:CostConfig", + "sqlbuild.spec.contracts.models:CostConfig", + ), ( "sqlbuild._native:CursorInputRoles", "sqlbuild.compiler.compile.models:CursorInputRoles", ), + ( + "sqlbuild._native:CursorsConfig", + "sqlbuild.spec.contracts.models:CursorsConfig", + ), ( "sqlbuild._native:CustomRulesOutcome", "sqlbuild.rule_engine.models:CustomRulesOutcome", ), + ( + "sqlbuild._native:DateValue", + "sqlbuild.cursor_algebra.models:DateValue", + ), ( "sqlbuild._native:DbtCliOptions", "sqlbuild.integrations.dbt.models:DbtCliOptions", @@ -568,6 +608,10 @@ pub const HOSTED_SHAPE_TYPES: [(&str, &str); 257] = [ "sqlbuild._native:DbtCommandResult", "sqlbuild.integrations.dbt.models:DbtCommandResult", ), + ( + "sqlbuild._native:DbtConfig", + "sqlbuild.spec.contracts.models:DbtConfig", + ), ( "sqlbuild._native:DbtInitProgressCallbacks", "sqlbuild.integrations.dbt.models:DbtInitProgressCallbacks", @@ -652,6 +696,10 @@ pub const HOSTED_SHAPE_TYPES: [(&str, &str); 257] = [ "sqlbuild._native:DeclarationVisibilityIndex", "sqlbuild.compiler.scopes.models:DeclarationVisibilityIndex", ), + ( + "sqlbuild._native:DefaultsConfig", + "sqlbuild.spec.contracts.models:DefaultsConfig", + ), ( "sqlbuild._native:DeferralInputs", "sqlbuild.compiler.planner.models:DeferralInputs", @@ -664,6 +712,10 @@ pub const HOSTED_SHAPE_TYPES: [(&str, &str); 257] = [ "sqlbuild._native:DiagnosticSeverity", "sqlbuild.compiler.compile.types:DiagnosticSeverity", ), + ( + "sqlbuild._native:DiffConfig", + "sqlbuild.spec.contracts.models:DiffConfig", + ), ( "sqlbuild._native:DirectSemanticColumnUse", "sqlbuild.compiler.lineage.models:DirectSemanticColumnUse", @@ -712,6 +764,10 @@ pub const HOSTED_SHAPE_TYPES: [(&str, &str); 257] = [ "sqlbuild._native:DiscoveredPythonLoaderMetadata", "sqlbuild.compiler.python_nodes.models:DiscoveredPythonLoaderMetadata", ), + ( + "sqlbuild._native:DiscoveredPythonNode", + "sqlbuild.compiler.python_nodes.models:DiscoveredPythonNode", + ), ( "sqlbuild._native:DiscoveredPythonTaskMetadata", "sqlbuild.compiler.python_nodes.models:DiscoveredPythonTaskMetadata", @@ -756,6 +812,10 @@ pub const HOSTED_SHAPE_TYPES: [(&str, &str); 257] = [ "sqlbuild._native:DisplayOptions", "sqlbuild.presentation.models:DisplayOptions", ), + ( + "sqlbuild._native:Duration", + "sqlbuild.cursor_algebra.models:Duration", + ), ( "sqlbuild._native:DynamicColumnContractProof", "sqlbuild.compiler.compile.models:DynamicColumnContractProof", @@ -776,10 +836,18 @@ pub const HOSTED_SHAPE_TYPES: [(&str, &str); 257] = [ "sqlbuild._native:EnumMetadata", "sqlbuild.compiler.scopes.models:EnumMetadata", ), + ( + "sqlbuild._native:EventExportSeverity", + "sqlbuild.spec.contracts.types:EventExportSeverity", + ), ( "sqlbuild._native:EventExporterDiscoveryError", "sqlbuild.compiler.discovery.exceptions:EventExporterDiscoveryError", ), + ( + "sqlbuild._native:ExecutionLimitsConfig", + "sqlbuild.spec.contracts.models:ExecutionLimitsConfig", + ), ( "sqlbuild._native:ExpandedLintFile", "sqlbuild.lint.models:ExpandedLintFile", @@ -788,10 +856,18 @@ pub const HOSTED_SHAPE_TYPES: [(&str, &str); 257] = [ "sqlbuild._native:ExpansionSpan", "sqlbuild.compiler.compile.models:ExpansionSpan", ), + ( + "sqlbuild._native:Fingerprint", + "sqlbuild.compiler.fingerprints.models:Fingerprint", + ), ( "sqlbuild._native:FingerprintInputError", "sqlbuild.compiler.fingerprints.exceptions:FingerprintInputError", ), + ( + "sqlbuild._native:FingerprintSet", + "sqlbuild.compiler.fingerprints.models:FingerprintSet", + ), ( "sqlbuild._native:FixVerdict", "sqlbuild.lint.models:FixVerdict", @@ -808,6 +884,14 @@ pub const HOSTED_SHAPE_TYPES: [(&str, &str); 257] = [ "sqlbuild._native:FunctionReturnColumn", "sqlbuild.compiler.compile.models:FunctionReturnColumn", ), + ( + "sqlbuild._native:FutureCursorAction", + "sqlbuild.spec.contracts.types:FutureCursorAction", + ), + ( + "sqlbuild._native:FutureCursorsConfig", + "sqlbuild.spec.contracts.models:FutureCursorsConfig", + ), ( "sqlbuild._native:GrantKind", "sqlbuild.compiler.scopes.types:GrantKind", @@ -844,6 +928,14 @@ pub const HOSTED_SHAPE_TYPES: [(&str, &str); 257] = [ "sqlbuild._native:InaccessibleRecord", "sqlbuild.compiler.scopes.models:InaccessibleRecord", ), + ( + "sqlbuild._native:IntegerValue", + "sqlbuild.cursor_algebra.models:IntegerValue", + ), + ( + "sqlbuild._native:IntegrationLoaderConfig", + "sqlbuild.spec.contracts.models:IntegrationLoaderConfig", + ), ( "sqlbuild._native:InterpolationSite", "sqlbuild.lint.models:InterpolationSite", @@ -860,6 +952,18 @@ pub const HOSTED_SHAPE_TYPES: [(&str, &str); 257] = [ "sqlbuild._native:InvalidSqlReferenceCall", "sqlbuild.compiler.compile.models:InvalidSqlReferenceCall", ), + ( + "sqlbuild._native:JanitorConfig", + "sqlbuild.spec.contracts.models:JanitorConfig", + ), + ( + "sqlbuild._native:LifecycleEventSinkFilterConfig", + "sqlbuild.spec.contracts.models:LifecycleEventSinkFilterConfig", + ), + ( + "sqlbuild._native:LifecycleEventSinksConfig", + "sqlbuild.spec.contracts.models:LifecycleEventSinksConfig", + ), ( "sqlbuild._native:LineageGraphViews", "sqlbuild.compiler.graph.models:LineageGraphViews", @@ -898,6 +1002,26 @@ pub const HOSTED_SHAPE_TYPES: [(&str, &str); 257] = [ "sqlbuild._native:LiteralSqlRelation", "sqlbuild.compiler.references.models:LiteralSqlRelation", ), + ( + "sqlbuild._native:LoaderDestinationParts", + "sqlbuild.spec.contracts.models:LoaderDestinationParts", + ), + ( + "sqlbuild._native:LocalClonePolicy", + "sqlbuild.spec.contracts.models:LocalClonePolicy", + ), + ( + "sqlbuild._native:LocalConfig", + "sqlbuild.spec.contracts.models:LocalConfig", + ), + ( + "sqlbuild._native:LocalDbtConfig", + "sqlbuild.spec.contracts.models:LocalDbtConfig", + ), + ( + "sqlbuild._native:LocalTargetConfig", + "sqlbuild.spec.contracts.models:LocalTargetConfig", + ), ( "sqlbuild._native:MacroExpansionResult", "sqlbuild.compiler.compile.models:MacroExpansionResult", @@ -914,10 +1038,42 @@ pub const HOSTED_SHAPE_TYPES: [(&str, &str); 257] = [ "sqlbuild._native:MappedOffset", "sqlbuild.compiler.compile.models:MappedOffset", ), + ( + "sqlbuild._native:MaterializationDefaultsConfig", + "sqlbuild.spec.contracts.models:MaterializationDefaultsConfig", + ), + ( + "sqlbuild._native:MaterializationRetentionDefaults", + "sqlbuild.spec.contracts.models:MaterializationRetentionDefaults", + ), ( "sqlbuild._native:MeasurementAuditError", "sqlbuild.compiler.auditing.exceptions:MeasurementAuditError", ), + ( + "sqlbuild._native:MeasurementContract", + "sqlbuild.compiler.auditing.models:MeasurementContract", + ), + ( + "sqlbuild._native:MeasurementThresholdBound", + "sqlbuild.compiler.auditing.models:MeasurementThresholdBound", + ), + ( + "sqlbuild._native:MeasurementThresholds", + "sqlbuild.compiler.auditing.models:MeasurementThresholds", + ), + ( + "sqlbuild._native:MicrobatchLimitAction", + "sqlbuild.spec.contracts.types:MicrobatchLimitAction", + ), + ( + "sqlbuild._native:MicrobatchLimitsConfig", + "sqlbuild.spec.contracts.models:MicrobatchLimitsConfig", + ), + ( + "sqlbuild._native:MicrobatchesConfig", + "sqlbuild.spec.contracts.models:MicrobatchesConfig", + ), ( "sqlbuild._native:MigrationCompatibility", "sqlbuild.compiler.migrations.types:MigrationCompatibility", @@ -926,6 +1082,14 @@ pub const HOSTED_SHAPE_TYPES: [(&str, &str); 257] = [ "sqlbuild._native:MigrationDiscovery", "sqlbuild.compiler.migrations.types:MigrationDiscovery", ), + ( + "sqlbuild._native:MigrationsConfig", + "sqlbuild.spec.contracts.models:MigrationsConfig", + ), + ( + "sqlbuild._native:MissingMigrationOriginPolicy", + "sqlbuild.spec.contracts.types:MissingMigrationOriginPolicy", + ), ( "sqlbuild._native:ModelColumnLineage", "sqlbuild.compiler.lineage.models:ModelColumnLineage", @@ -1082,6 +1246,10 @@ pub const HOSTED_SHAPE_TYPES: [(&str, &str); 257] = [ "sqlbuild._native:ProjectCompileError", "sqlbuild.lint.exceptions:ProjectCompileError", ), + ( + "sqlbuild._native:ProjectConfig", + "sqlbuild.spec.contracts.models:ProjectConfig", + ), ( "sqlbuild._native:ProjectConfigError", "sqlbuild.compiler.discovery.exceptions:ProjectConfigError", @@ -1098,6 +1266,10 @@ pub const HOSTED_SHAPE_TYPES: [(&str, &str); 257] = [ "sqlbuild._native:ProviderDiscoveryError", "sqlbuild.compiler.discovery.exceptions:ProviderDiscoveryError", ), + ( + "sqlbuild._native:PythonIdentityDependency", + "sqlbuild.compiler.python_nodes.models:PythonIdentityDependency", + ), ( "sqlbuild._native:PythonIdentityStatus", "sqlbuild.compiler.python_nodes.types:PythonIdentityStatus", @@ -1110,6 +1282,14 @@ pub const HOSTED_SHAPE_TYPES: [(&str, &str); 257] = [ "sqlbuild._native:PythonNodeDiscoveryError", "sqlbuild.compiler.discovery.exceptions:PythonNodeDiscoveryError", ), + ( + "sqlbuild._native:PythonNodeGraph", + "sqlbuild.compiler.python_nodes.models:PythonNodeGraph", + ), + ( + "sqlbuild._native:PythonNodeIdentity", + "sqlbuild.compiler.python_nodes.models:PythonNodeIdentity", + ), ( "sqlbuild._native:PythonPlanEntry", "sqlbuild.compiler.pipeline.models:PythonPlanEntry", @@ -1126,6 +1306,10 @@ pub const HOSTED_SHAPE_TYPES: [(&str, &str); 257] = [ "sqlbuild._native:PythonSqlReferenceReport", "sqlbuild.compiler.compile.models:PythonSqlReferenceReport", ), + ( + "sqlbuild._native:PythonSqlRunLifecyclePlan", + "sqlbuild.compiler.python_nodes.models:PythonSqlRunLifecyclePlan", + ), ( "sqlbuild._native:PythonSqlRunSelection", "sqlbuild.compiler.python_nodes.models:PythonSqlRunSelection", @@ -1150,10 +1334,26 @@ pub const HOSTED_SHAPE_TYPES: [(&str, &str); 257] = [ "sqlbuild._native:RefactorValueError", "sqlbuild.compiler.refactoring.exceptions:RefactorValueError", ), + ( + "sqlbuild._native:ReferencesConfig", + "sqlbuild.spec.contracts.models:ReferencesConfig", + ), ( "sqlbuild._native:RelatedLocation", "sqlbuild.compiler.compile.models:RelatedLocation", ), + ( + "sqlbuild._native:ResolvedDbtProfileOutput", + "sqlbuild.integrations.dbt.models:ResolvedDbtProfileOutput", + ), + ( + "sqlbuild._native:ResolvedTableType", + "sqlbuild.spec.contracts.models:ResolvedTableType", + ), + ( + "sqlbuild._native:ResolvedTimeTravelRetention", + "sqlbuild.spec.contracts.models:ResolvedTimeTravelRetention", + ), ( "sqlbuild._native:ResourceIdentity", "sqlbuild.compiler.scopes.models:ResourceIdentity", @@ -1170,6 +1370,10 @@ pub const HOSTED_SHAPE_TYPES: [(&str, &str); 257] = [ "sqlbuild._native:ResourceRecord", "sqlbuild.compiler.scopes.models:ResourceRecord", ), + ( + "sqlbuild._native:RetentionDecreasePolicy", + "sqlbuild.spec.contracts.types:RetentionDecreasePolicy", + ), ( "sqlbuild._native:RuleExemption", "sqlbuild.rule_engine.models:RuleExemption", @@ -1198,10 +1402,38 @@ pub const HOSTED_SHAPE_TYPES: [(&str, &str); 257] = [ "sqlbuild._native:RulesRunResult", "sqlbuild.rule_engine.models:RulesRunResult", ), + ( + "sqlbuild._native:ScenarioConfig", + "sqlbuild.spec.contracts.models:ScenarioConfig", + ), + ( + "sqlbuild._native:ScenarioSnapshotLimitsConfig", + "sqlbuild.spec.contracts.models:ScenarioSnapshotLimitsConfig", + ), + ( + "sqlbuild._native:SchemaAuditInstance", + "sqlbuild.spec.contracts.models:SchemaAuditInstance", + ), + ( + "sqlbuild._native:SchemaColumn", + "sqlbuild.spec.contracts.models:SchemaColumn", + ), + ( + "sqlbuild._native:SchemaDynamicColumnFamily", + "sqlbuild.spec.contracts.models:SchemaDynamicColumnFamily", + ), + ( + "sqlbuild._native:SchemaModelEntry", + "sqlbuild.spec.contracts.models:SchemaModelEntry", + ), ( "sqlbuild._native:SchemaParseError", "sqlbuild.compiler.discovery.exceptions:SchemaParseError", ), + ( + "sqlbuild._native:SchemaSeedEntry", + "sqlbuild.spec.contracts.models:SchemaSeedEntry", + ), ( "sqlbuild._native:ScopeBrowseResult", "sqlbuild.compiler.scopes.models:ScopeBrowseResult", @@ -1278,6 +1510,14 @@ pub const HOSTED_SHAPE_TYPES: [(&str, &str); 257] = [ "sqlbuild._native:ScopeValidationError", "sqlbuild.compiler.scopes.exceptions:ScopeValidationError", ), + ( + "sqlbuild._native:ScopesConfig", + "sqlbuild.spec.contracts.models:ScopesConfig", + ), + ( + "sqlbuild._native:SeedCsvSettings", + "sqlbuild.spec.contracts.models:SeedCsvSettings", + ), ( "sqlbuild._native:SeedDiscoveryError", "sqlbuild.compiler.discovery.exceptions:SeedDiscoveryError", @@ -1286,14 +1526,66 @@ pub const HOSTED_SHAPE_TYPES: [(&str, &str); 257] = [ "sqlbuild._native:SelectStarAllow", "sqlbuild.rule_engine.models:SelectStarAllow", ), + ( + "sqlbuild._native:SelectedDbtProfileOutput", + "sqlbuild.integrations.dbt.models:SelectedDbtProfileOutput", + ), ( "sqlbuild._native:SelectorKind", "sqlbuild.compiler.planner.types:SelectorKind", ), + ( + "sqlbuild._native:SettingsConfig", + "sqlbuild.spec.contracts.models:SettingsConfig", + ), + ( + "sqlbuild._native:SinksConfig", + "sqlbuild.spec.contracts.models:SinksConfig", + ), + ( + "sqlbuild._native:SnapshotsConfig", + "sqlbuild.spec.contracts.models:SnapshotsConfig", + ), + ( + "sqlbuild._native:SourceColumnEntry", + "sqlbuild.spec.contracts.models:SourceColumnEntry", + ), + ( + "sqlbuild._native:SourceEntry", + "sqlbuild.spec.contracts.models:SourceEntry", + ), + ( + "sqlbuild._native:SourceFreshnessAgePolicy", + "sqlbuild.spec.contracts.models:SourceFreshnessAgePolicy", + ), + ( + "sqlbuild._native:SourceFreshnessConfig", + "sqlbuild.spec.contracts.models:SourceFreshnessConfig", + ), + ( + "sqlbuild._native:SourceFreshnessRecord", + "sqlbuild.compiler.source_freshness.models:SourceFreshnessRecord", + ), + ( + "sqlbuild._native:SourceFreshnessStrategy", + "sqlbuild.spec.contracts.types:SourceFreshnessStrategy", + ), + ( + "sqlbuild._native:SourceFreshnessValueKind", + "sqlbuild.spec.contracts.types:SourceFreshnessValueKind", + ), + ( + "sqlbuild._native:SourceLocation", + "sqlbuild.spec.contracts.models:SourceLocation", + ), ( "sqlbuild._native:SourceParseError", "sqlbuild.compiler.discovery.exceptions:SourceParseError", ), + ( + "sqlbuild._native:SourceWriteStrategy", + "sqlbuild.spec.contracts.types:SourceWriteStrategy", + ), ( "sqlbuild._native:SpecConfigError", "sqlbuild.spec.contracts.exceptions:SpecConfigError", @@ -1322,6 +1614,14 @@ pub const HOSTED_SHAPE_TYPES: [(&str, &str); 257] = [ "sqlbuild._native:SqlHookParseError", "sqlbuild.compiler.discovery.exceptions:SqlHookParseError", ), + ( + "sqlbuild._native:SqlLexicalSyntax", + "sqlbuild.compiler.sql_analysis.models:SqlLexicalSyntax", + ), + ( + "sqlbuild._native:SqlLogicalType", + "sqlbuild.sql_values.models:SqlLogicalType", + ), ( "sqlbuild._native:SqlReferenceScan", "sqlbuild.compiler.compile.models:SqlReferenceScan", @@ -1354,14 +1654,62 @@ pub const HOSTED_SHAPE_TYPES: [(&str, &str); 257] = [ "sqlbuild._native:SqlTestSelection", "sqlbuild.compiler.planner.models:SqlTestSelection", ), + ( + "sqlbuild._native:SqlValue", + "sqlbuild.sql_values.models:SqlValue", + ), + ( + "sqlbuild._native:SqlValueKind", + "sqlbuild.sql_values.types:SqlValueKind", + ), + ( + "sqlbuild._native:StartCursorsConfig", + "sqlbuild.spec.contracts.models:StartCursorsConfig", + ), + ( + "sqlbuild._native:StateSqlValueType", + "sqlbuild.sql_values.types:StateSqlValueType", + ), ( "sqlbuild._native:StaticCommandContext", "sqlbuild.compiler.pipeline.models:StaticCommandContext", ), + ( + "sqlbuild._native:TableType", + "sqlbuild.spec.contracts.types:TableType", + ), + ( + "sqlbuild._native:TableTypeDowngradePolicy", + "sqlbuild.spec.contracts.types:TableTypeDowngradePolicy", + ), + ( + "sqlbuild._native:TableTypeSource", + "sqlbuild.spec.contracts.types:TableTypeSource", + ), + ( + "sqlbuild._native:TargetConfig", + "sqlbuild.spec.contracts.models:TargetConfig", + ), + ( + "sqlbuild._native:TargetDiffConfig", + "sqlbuild.spec.contracts.models:TargetDiffConfig", + ), + ( + "sqlbuild._native:TargetWarehousesConfig", + "sqlbuild.spec.contracts.models:TargetWarehousesConfig", + ), ( "sqlbuild._native:ThresholdOverride", "sqlbuild.rule_engine.models:ThresholdOverride", ), + ( + "sqlbuild._native:TimeTravelRetentionSource", + "sqlbuild.spec.contracts.types:TimeTravelRetentionSource", + ), + ( + "sqlbuild._native:TimestampValue", + "sqlbuild.cursor_algebra.models:TimestampValue", + ), ( "sqlbuild._native:TolerantScopeDiscovery", "sqlbuild.compiler.discovery.models:TolerantScopeDiscovery", @@ -1406,4 +1754,8 @@ pub const HOSTED_SHAPE_TYPES: [(&str, &str); 257] = [ "sqlbuild._native:VisibleModelDeclarations", "sqlbuild.compiler.compile.models:VisibleModelDeclarations", ), + ( + "sqlbuild._native:WarehouseGroup", + "sqlbuild.spec.contracts.types:WarehouseGroup", + ), ]; diff --git a/crates/sqlbuild-config/src/project/models.rs b/crates/sqlbuild-config/src/project/models.rs index ff2a018aa6..d583900e86 100644 --- a/crates/sqlbuild-config/src/project/models.rs +++ b/crates/sqlbuild-config/src/project/models.rs @@ -25,7 +25,7 @@ pub struct ConfigPaths { pub enum Built { /// A plain value, including `None`, booleans, integers and raw authored mappings. Value(ConfigValue), - /// A dataclass from `sqlbuild.spec.contracts.models`; omitted fields keep their defaults. + /// A native project configuration record; omitted fields keep their defaults. Object(&'static str, Vec<(&'static str, Built)>), /// A member of a named `StrEnum`, by value. Enum(&'static str, String), diff --git a/crates/sqlbuild-model-config/src/header_metadata/_helpers/thresholds.rs b/crates/sqlbuild-model-config/src/header_metadata/_helpers/thresholds.rs index a2730f2ab2..1e100c626e 100644 --- a/crates/sqlbuild-model-config/src/header_metadata/_helpers/thresholds.rs +++ b/crates/sqlbuild-model-config/src/header_metadata/_helpers/thresholds.rs @@ -100,7 +100,7 @@ fn finite_limit( } /// Check `MeasurementThresholds.__post_init__` for the parsed bounds. -fn check_pair( +pub(crate) fn check_pair( warn: Option, error: Option, ) -> Result<(), &'static str> { diff --git a/crates/sqlbuild-model-config/src/header_metadata/constants.rs b/crates/sqlbuild-model-config/src/header_metadata/constants.rs index 387bd87f93..949cb97071 100644 --- a/crates/sqlbuild-model-config/src/header_metadata/constants.rs +++ b/crates/sqlbuild-model-config/src/header_metadata/constants.rs @@ -20,7 +20,9 @@ pub const AUDIT_OPTION_KEYS: [&str; 8] = [ pub const THRESHOLD_WARN_KEY: &str = "warn"; pub const THRESHOLD_ERROR_KEY: &str = "error"; /// Threshold operators, as `ThresholdOperator` values. -pub const THRESHOLD_OPERATORS: [&str; 3] = ["below", "above", "outside"]; +pub const THRESHOLD_OPERATORS: [&str; 3] = [BELOW_OPERATOR, ABOVE_OPERATOR, "outside"]; +pub const BELOW_OPERATOR: &str = "below"; +pub const ABOVE_OPERATOR: &str = "above"; /// Audit severities a header may name. pub const AUDIT_SEVERITIES: [&str; 2] = ["warn", "error"]; /// The audit that contradicts `nullable = true`. diff --git a/crates/sqlbuild-model-config/src/header_metadata/main/check_measurement_contract.rs b/crates/sqlbuild-model-config/src/header_metadata/main/check_measurement_contract.rs new file mode 100644 index 0000000000..84faae9b98 --- /dev/null +++ b/crates/sqlbuild-model-config/src/header_metadata/main/check_measurement_contract.rs @@ -0,0 +1,21 @@ +//! `MeasurementContract.__post_init__`: the measurement query's named columns and unit. + +use sqlbuild_core::text::main::python_strip::python_strip; + +/// Whether every named contract field is non-blank, or the measurement audit error text. +pub fn check_measurement_contract( + value_column: &str, + sample_count_column: Option<&str>, + sample_unit: Option<&str>, +) -> Result<(), &'static str> { + if python_strip(value_column).is_empty() { + return Err("measurement contract value_column must be non-empty"); + } + if sample_count_column.is_some_and(|column| python_strip(column).is_empty()) { + return Err("measurement contract sample_count_column must be non-empty"); + } + if sample_unit.is_some_and(|unit| python_strip(unit).is_empty()) { + return Err("measurement contract sample_unit must be non-empty"); + } + Ok(()) +} diff --git a/crates/sqlbuild-model-config/src/header_metadata/main/check_threshold_bound.rs b/crates/sqlbuild-model-config/src/header_metadata/main/check_threshold_bound.rs new file mode 100644 index 0000000000..55af2103e8 --- /dev/null +++ b/crates/sqlbuild-model-config/src/header_metadata/main/check_threshold_bound.rs @@ -0,0 +1,52 @@ +//! `MeasurementThresholdBound.__post_init__`: one directional bound and its limits. + +use crate::header_metadata::constants::{ABOVE_OPERATOR, BELOW_OPERATOR}; +use crate::header_metadata::models::{ThresholdBound, ThresholdInput}; + +/// The bound `operator` declares over `(limit, lower, upper)`, or the measurement audit error text. +pub fn check_threshold_bound( + operator: &str, + (limit, lower, upper): (ThresholdInput, ThresholdInput, ThresholdInput), +) -> Result { + if operator == BELOW_OPERATOR || operator == ABOVE_OPERATOR { + let limit: f64 = match (limit, lower, upper) { + (ThresholdInput::Number(limit), ThresholdInput::Absent, ThresholdInput::Absent) => { + limit + } + (ThresholdInput::Boolean, ThresholdInput::Absent, ThresholdInput::Absent) => { + return Err("measurement threshold limit must not be boolean".to_owned()); + } + _ => return Err(format!("{operator} threshold requires limit only")), + }; + if !limit.is_finite() { + return Err("measurement threshold limit must be finite".to_owned()); + } + return Ok(if operator == BELOW_OPERATOR { + ThresholdBound::Below(limit) + } else { + ThresholdBound::Above(limit) + }); + } + if limit != ThresholdInput::Absent + || lower == ThresholdInput::Absent + || upper == ThresholdInput::Absent + { + return Err("outside threshold requires lower and upper only".to_owned()); + } + let ThresholdInput::Number(lower) = lower else { + return Err("measurement threshold lower must not be boolean".to_owned()); + }; + let ThresholdInput::Number(upper) = upper else { + return Err("measurement threshold upper must not be boolean".to_owned()); + }; + if !lower.is_finite() { + return Err("measurement threshold lower must be finite".to_owned()); + } + if !upper.is_finite() { + return Err("measurement threshold upper must be finite".to_owned()); + } + if lower > upper { + return Err("outside threshold lower must be less than or equal to upper".to_owned()); + } + Ok(ThresholdBound::Outside(lower, upper)) +} diff --git a/crates/sqlbuild-model-config/src/header_metadata/main/check_threshold_pair.rs b/crates/sqlbuild-model-config/src/header_metadata/main/check_threshold_pair.rs new file mode 100644 index 0000000000..002a96426d --- /dev/null +++ b/crates/sqlbuild-model-config/src/header_metadata/main/check_threshold_pair.rs @@ -0,0 +1,12 @@ +//! `MeasurementThresholds.__post_init__`: the warn and error bounds of one audit. + +use crate::header_metadata::_helpers::thresholds::check_pair; +use crate::header_metadata::models::ThresholdBound; + +/// Whether `warn` and `error` form a valid threshold policy, or the measurement audit error text. +pub fn check_threshold_pair( + warn: Option, + error: Option, +) -> Result<(), &'static str> { + check_pair(warn, error) +} diff --git a/crates/sqlbuild-model-config/src/header_metadata/main/mod.rs b/crates/sqlbuild-model-config/src/header_metadata/main/mod.rs index c7b2438638..d195d8f0d8 100644 --- a/crates/sqlbuild-model-config/src/header_metadata/main/mod.rs +++ b/crates/sqlbuild-model-config/src/header_metadata/main/mod.rs @@ -1,4 +1,7 @@ pub mod audit_factory_references; +pub mod check_measurement_contract; +pub mod check_threshold_bound; +pub mod check_threshold_pair; pub mod merged_audit_identities; pub mod parse_audit_list; pub mod parse_dynamic_columns; diff --git a/crates/sqlbuild-model-config/src/header_metadata/models.rs b/crates/sqlbuild-model-config/src/header_metadata/models.rs index 165b823b9b..9a1ea2e896 100644 --- a/crates/sqlbuild-model-config/src/header_metadata/models.rs +++ b/crates/sqlbuild-model-config/src/header_metadata/models.rs @@ -28,6 +28,14 @@ pub enum ThresholdBound { Outside(f64, f64), } +/// One limit passed to a measurement threshold bound, as Python sees its type. +#[derive(Clone, Copy, Debug, PartialEq)] +pub enum ThresholdInput { + Absent, + Boolean, + Number(f64), +} + /// An audit's warn and error thresholds; at least one is set. #[derive(Clone, Copy, Debug, PartialEq)] pub struct ParsedThresholds { diff --git a/crates/sqlbuild-model-config/src/header_metadata/tests.rs b/crates/sqlbuild-model-config/src/header_metadata/tests.rs index fed845e0be..bedc46c47f 100644 --- a/crates/sqlbuild-model-config/src/header_metadata/tests.rs +++ b/crates/sqlbuild-model-config/src/header_metadata/tests.rs @@ -2,6 +2,8 @@ mod header_extensions; #[path = "tests/helpers.rs"] mod helpers; +#[path = "tests/test_measurement_checks.rs"] +mod measurement_checks; #[path = "tests/test_parse_header_metadata.rs"] mod parse_header_metadata; #[path = "tests/test_parse_schema_columns.rs"] diff --git a/crates/sqlbuild-model-config/src/header_metadata/tests/test_measurement_checks.rs b/crates/sqlbuild-model-config/src/header_metadata/tests/test_measurement_checks.rs new file mode 100644 index 0000000000..acdad37ca4 --- /dev/null +++ b/crates/sqlbuild-model-config/src/header_metadata/tests/test_measurement_checks.rs @@ -0,0 +1,130 @@ +use crate::header_metadata::main::check_measurement_contract::check_measurement_contract; +use crate::header_metadata::main::check_threshold_bound::check_threshold_bound; +use crate::header_metadata::models::{ThresholdBound, ThresholdInput}; +use crate::header_metadata::tests::test_types::{ + MeasurementContractTestCase, ThresholdBoundTestCase, +}; + +#[test] +fn given_threshold_limits_when_checking_bound_then_matches_python_validation() { + let test_cases = [ + ThresholdBoundTestCase { + description: "a below limit becomes the bound", + operator: "below", + inputs: ( + ThresholdInput::Number(2.0), + ThresholdInput::Absent, + ThresholdInput::Absent, + ), + expected_bound: Ok(ThresholdBound::Below(2.0)), + }, + ThresholdBoundTestCase { + description: "a boolean limit is rejected after the shape check", + operator: "above", + inputs: ( + ThresholdInput::Boolean, + ThresholdInput::Absent, + ThresholdInput::Absent, + ), + expected_bound: Err("measurement threshold limit must not be boolean"), + }, + ThresholdBoundTestCase { + description: "a directional bound with a lower limit is rejected", + operator: "above", + inputs: ( + ThresholdInput::Number(1.0), + ThresholdInput::Number(0.0), + ThresholdInput::Absent, + ), + expected_bound: Err("above threshold requires limit only"), + }, + ThresholdBoundTestCase { + description: "an infinite limit is rejected", + operator: "below", + inputs: ( + ThresholdInput::Number(f64::INFINITY), + ThresholdInput::Absent, + ThresholdInput::Absent, + ), + expected_bound: Err("measurement threshold limit must be finite"), + }, + ThresholdBoundTestCase { + description: "an outside bound without an upper limit fails the shape check first", + operator: "outside", + inputs: ( + ThresholdInput::Absent, + ThresholdInput::Boolean, + ThresholdInput::Absent, + ), + expected_bound: Err("outside threshold requires lower and upper only"), + }, + ThresholdBoundTestCase { + description: "a boolean upper limit is rejected", + operator: "outside", + inputs: ( + ThresholdInput::Absent, + ThresholdInput::Number(1.0), + ThresholdInput::Boolean, + ), + expected_bound: Err("measurement threshold upper must not be boolean"), + }, + ThresholdBoundTestCase { + description: "a reversed outside range is rejected", + operator: "outside", + inputs: ( + ThresholdInput::Absent, + ThresholdInput::Number(3.0), + ThresholdInput::Number(1.0), + ), + expected_bound: Err("outside threshold lower must be less than or equal to upper"), + }, + ThresholdBoundTestCase { + description: "an outside range becomes the bound", + operator: "outside", + inputs: ( + ThresholdInput::Absent, + ThresholdInput::Number(1.0), + ThresholdInput::Number(3.0), + ), + expected_bound: Ok(ThresholdBound::Outside(1.0, 3.0)), + }, + ]; + for test_case in test_cases { + assert_eq!( + check_threshold_bound(test_case.operator, test_case.inputs), + test_case.expected_bound.map_err(str::to_owned), + "{}", + test_case.description + ); + } +} + +#[test] +fn given_contract_fields_when_checking_then_blank_names_are_rejected() { + let test_cases = [ + MeasurementContractTestCase { + description: "named columns are accepted", + fields: ("value", Some("samples"), None), + expected_result: Ok(()), + }, + MeasurementContractTestCase { + description: "a blank value column is rejected", + fields: (" ", None, None), + expected_result: Err("measurement contract value_column must be non-empty"), + }, + MeasurementContractTestCase { + description: "a blank sample unit is rejected", + fields: ("value", None, Some("")), + expected_result: Err("measurement contract sample_unit must be non-empty"), + }, + ]; + for test_case in test_cases { + let (value_column, sample_count_column, sample_unit) = test_case.fields; + assert_eq!( + check_measurement_contract(value_column, sample_count_column, sample_unit), + test_case.expected_result, + "{}", + test_case.description + ); + } +} diff --git a/crates/sqlbuild-model-config/src/header_metadata/tests/test_types.rs b/crates/sqlbuild-model-config/src/header_metadata/tests/test_types.rs index 1d6136ac43..0a09443e13 100644 --- a/crates/sqlbuild-model-config/src/header_metadata/tests/test_types.rs +++ b/crates/sqlbuild-model-config/src/header_metadata/tests/test_types.rs @@ -1,3 +1,4 @@ +use crate::header_metadata::models::{ThresholdBound, ThresholdInput}; use crate::tests::test_types::Value; /// One header's `columns` and `audits` values and the parse the native parser must produce. @@ -31,3 +32,18 @@ pub(super) struct SchemaColumnsTestCase { /// The comma-separated column names, or the first error message. pub(super) expected_summary: &'static str, } + +/// One threshold bound's operator and limits and the bound or error it produces. +pub(super) struct ThresholdBoundTestCase { + pub(super) description: &'static str, + pub(super) operator: &'static str, + pub(super) inputs: (ThresholdInput, ThresholdInput, ThresholdInput), + pub(super) expected_bound: Result, +} + +/// One measurement contract's fields and the error it produces, if any. +pub(super) struct MeasurementContractTestCase { + pub(super) description: &'static str, + pub(super) fields: (&'static str, Option<&'static str>, Option<&'static str>), + pub(super) expected_result: Result<(), &'static str>, +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/analysis/explicit_references.rs b/crates/sqlbuild-python/src/bindings/_helpers/analysis/explicit_references.rs index ea8a364d13..c9f5446a7e 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/analysis/explicit_references.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/analysis/explicit_references.rs @@ -34,7 +34,7 @@ const COMPILE_ERRORS_MODULE: &str = "sqlbuild.compiler.compile.exceptions"; const COMPILE_INPUT_ERROR: &str = "CompileInputError"; const COMPILE_MODELS_MODULE: &str = "sqlbuild.compiler.compile.models"; const REFERENCE_MODELS_MODULE: &str = "sqlbuild.compiler.references.models"; -const CONTRACT_MODELS_MODULE: &str = "sqlbuild.spec.contracts.models"; +const CONTRACT_MODELS_MODULE: &str = "sqlbuild._native"; const REFERENCE_TYPES_MODULE: &str = "sqlbuild.compiler.references.types"; const LOADER_DEFINITION_MODULE: &str = "sqlbuild.python_nodes.main.read_loader_definition"; const NATIVE_MODULE: &str = "sqlbuild._native"; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/auditing/identity.rs b/crates/sqlbuild-python/src/bindings/_helpers/auditing/identity.rs index d0b0490370..2bdbaebbfb 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/auditing/identity.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/auditing/identity.rs @@ -8,7 +8,7 @@ use sqlbuild_analysis::auditing::models::{ GateIdentity, MeasurementFacts, PlannedAudit, ThresholdBound, Thresholds, }; -const AUDITING_MODELS_MODULE: &str = "sqlbuild.compiler.auditing.models"; +const AUDITING_MODELS_MODULE: &str = "sqlbuild._native"; const MEASUREMENT_MODE: &str = "measurement"; fn enum_value(value: &Bound<'_, PyAny>) -> PyResult { diff --git a/crates/sqlbuild-python/src/bindings/_helpers/build_command/defer_clone.rs b/crates/sqlbuild-python/src/bindings/_helpers/build_command/defer_clone.rs index 1f81f7cb57..7b183ca425 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/build_command/defer_clone.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/build_command/defer_clone.rs @@ -20,7 +20,7 @@ const NATIVE_RECORDS_MODULE: &str = "sqlbuild._native"; const NATIVE_MODULE: &str = "sqlbuild._native"; const CLONE_MODELS: &str = "sqlbuild.executor.clone.models"; const CLONE_TYPES: &str = "sqlbuild.executor.clone.types"; -const SPEC_TYPES: &str = "sqlbuild.spec.contracts.types"; +const SPEC_TYPES: &str = "sqlbuild._native"; const TRANSIENT_MODELS_MODULE: &str = "sqlbuild.executor.clone.main._build_destination_transient_models"; const RETENTION_REQUESTS_MODULE: &str = "sqlbuild.executor.clone.main._build_retention_requests"; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/build_command/invocation.rs b/crates/sqlbuild-python/src/bindings/_helpers/build_command/invocation.rs index 0845f9e4b9..b7dd57683a 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/build_command/invocation.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/build_command/invocation.rs @@ -18,7 +18,7 @@ const SETTINGS_MODULE: &str = "sqlbuild._native"; const ADAPTER_NAME_MODULE: &str = "sqlbuild._native"; const TARGET_CONFIG_MODULE: &str = "sqlbuild._native"; const TARGET_NAME_MODULE: &str = "sqlbuild._native"; -const CONTRACT_MODELS: &str = "sqlbuild.spec.contracts.models"; +const CONTRACT_MODELS: &str = "sqlbuild._native"; const DEFER_CONFLICT_CODE: &str = "C408"; /// One build's request, resolved invocation and output settings. diff --git a/crates/sqlbuild-python/src/bindings/_helpers/cli/entry.rs b/crates/sqlbuild-python/src/bindings/_helpers/cli/entry.rs index aa61b40fff..c9dffcda47 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/cli/entry.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/cli/entry.rs @@ -37,7 +37,7 @@ const TRANSIENT_LINE_MODULE: &str = "sqlbuild._native"; const SKILLS_MODULE: &str = "sqlbuild._native"; const RULES_ERRORS_MODULE: &str = "sqlbuild.rule_engine.exceptions"; const LINT_ERRORS_MODULE: &str = "sqlbuild._native"; -const CONTRACT_MODELS_MODULE: &str = "sqlbuild.spec.contracts.models"; +const CONTRACT_MODELS_MODULE: &str = "sqlbuild._native"; const NATIVE_MODULE: &str = "sqlbuild._native"; const CLI_LOGGER: &str = "sqlbuild.cli"; const UNKNOWN_COMMAND: &str = "unknown"; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/clone/clone_execution.rs b/crates/sqlbuild-python/src/bindings/_helpers/clone/clone_execution.rs index fa12657b63..1fedbc14e2 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/clone/clone_execution.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/clone/clone_execution.rs @@ -24,7 +24,7 @@ const TRANSIENT_MODELS_MODULE: &str = const FINGERPRINTING_MODULE: &str = "sqlbuild.executor.clone.main._fingerprinting"; const EXECUTE_MODULE: &str = "sqlbuild.executor.clone.main.execute"; const TARGET_CONFIG_MODULE: &str = "sqlbuild._native"; -const CONTRACT_TYPES: &str = "sqlbuild.spec.contracts.types"; +const CONTRACT_TYPES: &str = "sqlbuild._native"; const LIFECYCLE_MODULE: &str = "sqlbuild.runtime.observability.classes.operation_lifecycle"; const DESTINATION_ENTRY_GROUPS: [&str; 4] = [ "destination_source_entries", diff --git a/crates/sqlbuild-python/src/bindings/_helpers/command_runtime/warehouse_scope.rs b/crates/sqlbuild-python/src/bindings/_helpers/command_runtime/warehouse_scope.rs index 208e5184f4..25cd0771dd 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/command_runtime/warehouse_scope.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/command_runtime/warehouse_scope.rs @@ -13,7 +13,7 @@ use crate::bindings::_helpers::compile_driver::python_calls::{call_with, importe const NATIVE_MODULE: &str = "sqlbuild._native"; const ENTRY_MODELS: &str = "sqlbuild._native"; -const WAREHOUSE_GROUP: (&str, &str) = ("sqlbuild.spec.contracts.types", "WarehouseGroup"); +const WAREHOUSE_GROUP: (&str, &str) = ("sqlbuild._native", "WarehouseGroup"); const COMMAND_TYPES: &str = "sqlbuild._native"; const CONTEXT_ATTRIBUTE: &str = "_command_warehouse_scope_context"; const CONTEXT_NAME: &str = "sqlbuild_command_warehouse_scope"; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/command_runtime/warehouses.rs b/crates/sqlbuild-python/src/bindings/_helpers/command_runtime/warehouses.rs index 0b24d368a5..83238fb68c 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/command_runtime/warehouses.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/command_runtime/warehouses.rs @@ -28,6 +28,10 @@ use crate::bindings::_helpers::compile_driver::python_calls::{ use crate::bindings::_helpers::discovery::configuration::discover_project_configuration; use crate::bindings::_helpers::model_config::template_data::expanded_template_data; +/// The `WarehouseGroup` whose warehouse is a target's `warehouses.build`; any other reads `query`. +const BUILD_GROUP: &str = "build"; +const QUERY_GROUP: &str = "query"; + const ENTRY_MODELS: &str = "sqlbuild._native"; const TARGET_NAME_MODULE: &str = "sqlbuild._native"; const ADAPTER_NAME_MODULE: &str = "sqlbuild._native"; @@ -119,11 +123,12 @@ fn group_value(target: &Bound<'_, PyAny>, group: &Bound<'_, PyAny>) -> PyResult< if target.is_none() { return Ok(None); } - optional_text( - &target - .getattr("warehouses")? - .call_method1("for_group", (group,))?, - ) + let attribute: &str = if group.str()?.extract::()? == BUILD_GROUP { + BUILD_GROUP + } else { + QUERY_GROUP + }; + optional_text(&target.getattr("warehouses")?.getattr(attribute)?) } /// The active target's authored, unexpanded warehouse for one command group. diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compile_attachment/effective_config.rs b/crates/sqlbuild-python/src/bindings/_helpers/compile_attachment/effective_config.rs index 96bf6589cd..c390e9eb03 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compile_attachment/effective_config.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compile_attachment/effective_config.rs @@ -16,7 +16,7 @@ use crate::bindings::_helpers::model_config::template_data::{ const NATIVE_MODULE: &str = "sqlbuild._native"; const COMPILE_ERRORS_MODULE: &str = "sqlbuild.compiler.compile.exceptions"; -const CONTRACT_MODELS_MODULE: &str = "sqlbuild.spec.contracts.models"; +const CONTRACT_MODELS_MODULE: &str = "sqlbuild._native"; const CONNECTION_LABEL: &str = "effective connection"; /// Merge base project connection with the selected environment overrides. diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compile_attachment/model_config.rs b/crates/sqlbuild-python/src/bindings/_helpers/compile_attachment/model_config.rs index 49da3e115f..a53bf2781d 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compile_attachment/model_config.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compile_attachment/model_config.rs @@ -14,7 +14,7 @@ const NATIVE_RECORDS_MODULE: &str = "sqlbuild._native"; const NATIVE_MODULE: &str = "sqlbuild._native"; const DISCOVERY_MODELS_MODULE: &str = "sqlbuild.compiler.discovery.models"; -const CONTRACT_MODELS_MODULE: &str = "sqlbuild.spec.contracts.models"; +const CONTRACT_MODELS_MODULE: &str = "sqlbuild._native"; /// Capture the project inputs every model's native config build and validation read. #[pyfunction] diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compile_attachment/model_inputs.rs b/crates/sqlbuild-python/src/bindings/_helpers/compile_attachment/model_inputs.rs index 12c8cd732b..b5b0a06f01 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compile_attachment/model_inputs.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compile_attachment/model_inputs.rs @@ -28,7 +28,7 @@ const SCOPE_TYPES_MODULE: &str = "sqlbuild._native"; const COLLECTOR_MODULE: &str = "sqlbuild._native"; const PROJECT_VARS_MODULE: &str = "sqlbuild._native"; const MACRO_BRIDGE_MODULE: &str = "sqlbuild.compiler.macro_bridge.main.active_macro_bridge"; -const SQL_ANALYSIS_MODELS_MODULE: &str = "sqlbuild.compiler.sql_analysis.models"; +const SQL_ANALYSIS_MODELS_MODULE: &str = "sqlbuild._native"; const PYTHON_NODE_MODELS_MODULE: &str = "sqlbuild.python_nodes.models"; const PYTHON_NODE_TYPES_MODULE: &str = "sqlbuild.python_nodes.types"; /// The explicit reference call kinds and the `SqlResourceRefKind` member each resolves to. diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compile_attachment/storage_policies.rs b/crates/sqlbuild-python/src/bindings/_helpers/compile_attachment/storage_policies.rs index 462c10e10f..9d91bb850e 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compile_attachment/storage_policies.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compile_attachment/storage_policies.rs @@ -5,8 +5,8 @@ use pyo3::types::{PyBool, PyString, PyTuple}; use crate::bindings::_helpers::compile_driver::python_calls::{call_with, imported}; -const CONTRACT_MODELS_MODULE: &str = "sqlbuild.spec.contracts.models"; -const CONTRACT_TYPES_MODULE: &str = "sqlbuild.spec.contracts.types"; +const CONTRACT_MODELS_MODULE: &str = "sqlbuild._native"; +const CONTRACT_TYPES_MODULE: &str = "sqlbuild._native"; /// `value`, unless it is `None`. fn present(value: Bound<'_, PyAny>) -> Option> { diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compile_records/constants.rs b/crates/sqlbuild-python/src/bindings/_helpers/compile_records/constants.rs index b5a8bd0f08..5a0cf88c03 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compile_records/constants.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compile_records/constants.rs @@ -52,14 +52,11 @@ const R_COMPILE_MODEL_CONFIG: RecordSpec = RecordSpec { FieldSpec::new("logical_database", FieldDefault::None), FieldSpec::new( "time_travel_retention", - FieldDefault::Call( - "sqlbuild.spec.contracts.models", - "ResolvedTimeTravelRetention", - ), + FieldDefault::Call("sqlbuild._native", "ResolvedTimeTravelRetention"), ), FieldSpec::new( "table_type", - FieldDefault::Call("sqlbuild.spec.contracts.models", "ResolvedTableType"), + FieldDefault::Call("sqlbuild._native", "ResolvedTableType"), ), ], }; @@ -299,7 +296,7 @@ const R_COMPILE_PROJECT_INPUTS: RecordSpec = RecordSpec { FieldSpec::new("effective_connection", FieldDefault::EmptyDict), FieldSpec::new( "effective_settings", - FieldDefault::Call("sqlbuild.spec.contracts.models", "SettingsConfig"), + FieldDefault::Call("sqlbuild._native", "SettingsConfig"), ), FieldSpec::new("no_sql_validation", FieldDefault::False), FieldSpec::new("effective_vars", FieldDefault::EmptyDict), diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/attachment.rs b/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/attachment.rs index 19b1191942..de2f57c822 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/attachment.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/attachment.rs @@ -34,7 +34,7 @@ const COMPILE_TYPES_MODULE: &str = "sqlbuild.compiler.compile.types"; const SCOPE_TYPES_MODULE: &str = "sqlbuild._native"; const SCOPE_MODELS_MODULE: &str = "sqlbuild._native"; const AUDITING_TYPES_MODULE: &str = "sqlbuild.compiler.auditing.types"; -const CONTRACT_MODELS_MODULE: &str = "sqlbuild.spec.contracts.models"; +const CONTRACT_MODELS_MODULE: &str = "sqlbuild._native"; const SQL_VALUE_ERRORS_MODULE: &str = "sqlbuild.sql_values.exceptions"; const YAML_FILE_SUFFIXES: [&str; 2] = [".yml", ".yaml"]; const PYTHON_RESOURCES_MODULE: &str = diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/loop_classes.rs b/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/loop_classes.rs index 22a7629921..c07b76c9c6 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/loop_classes.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/loop_classes.rs @@ -19,8 +19,8 @@ const REFERENCE_TYPES_MODULE: &str = "sqlbuild.compiler.references.types"; const SCOPE_ERRORS_MODULE: &str = "sqlbuild._native"; const SCOPE_MODELS_MODULE: &str = "sqlbuild._native"; const SCOPE_TYPES_MODULE: &str = "sqlbuild._native"; -const CONTRACT_MODELS_MODULE: &str = "sqlbuild.spec.contracts.models"; -const CONTRACT_TYPES_MODULE: &str = "sqlbuild.spec.contracts.types"; +const CONTRACT_MODELS_MODULE: &str = "sqlbuild._native"; +const CONTRACT_TYPES_MODULE: &str = "sqlbuild._native"; const NATIVE_MODULE: &str = "sqlbuild._native"; /// The materializations a model's storage policies are inherited for, besides none. const TABLE_BACKED_MATERIALIZATIONS: [&str; 3] = ["TABLE", "INCREMENTAL", "SNAPSHOT"]; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compiler_enums/constants.rs b/crates/sqlbuild-python/src/bindings/_helpers/compiler_enums/constants.rs index 59b859ecb9..98b0933030 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compiler_enums/constants.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compiler_enums/constants.rs @@ -304,6 +304,144 @@ const E_SCOPE_DIAGNOSTIC_CODE: EnumSpec = EnumSpec { ("OVER_BROAD_GLOBAL", "S024"), ], }; +const E_EVENT_EXPORT_SEVERITY: EnumSpec = EnumSpec { + name: "EventExportSeverity", + doc: "Severity used to filter exported lifecycle events.", + members: &[ + ("DEBUG", "debug"), + ("INFO", "info"), + ("WARNING", "warning"), + ("ERROR", "error"), + ], +}; +const E_WAREHOUSE_GROUP: EnumSpec = EnumSpec { + name: "WarehouseGroup", + doc: "Command group that selects a target's default warehouse.", + members: &[("BUILD", "build"), ("QUERY", "query")], +}; +const E_SOURCE_WRITE_STRATEGY: EnumSpec = EnumSpec { + name: "SourceWriteStrategy", + doc: "", + members: &[ + ("APPEND", "append"), + ("DELETE_INSERT", "delete_insert"), + ("MERGE", "merge"), + ("TABLE", "table"), + ], +}; +const E_FUTURE_CURSOR_ACTION: EnumSpec = EnumSpec { + name: "FutureCursorAction", + doc: "Response when an effective cursor lies beyond the future horizon.", + members: &[("CAP", "cap"), ("ERROR", "error")], +}; +const E_MICROBATCH_LIMIT_ACTION: EnumSpec = EnumSpec { + name: "MicrobatchLimitAction", + doc: "Response when one model plans more than the configured batch limit.", + members: &[ + ("CAP_FROM_END", "cap_from_end"), + ("CAP_FROM_START", "cap_from_start"), + ("ERROR", "error"), + ("WARN", "warn"), + ], +}; +const E_COLUMN_CONTRACT_MODE: EnumSpec = EnumSpec { + name: "ColumnContractMode", + doc: "Default contract behavior for model column declarations.", + members: &[("IMPLICIT", "implicit"), ("EXPLICIT", "explicit")], +}; +const E_SOURCE_FRESHNESS_STRATEGY: EnumSpec = EnumSpec { + name: "SourceFreshnessStrategy", + doc: "", + members: &[("ADAPTER", "adapter"), ("COLUMN", "column"), ("SQL", "sql")], +}; +const E_SOURCE_FRESHNESS_VALUE_KIND: EnumSpec = EnumSpec { + name: "SourceFreshnessValueKind", + doc: "", + members: &[ + ("TIMESTAMP", "timestamp"), + ("INTEGER", "integer"), + ("STRING", "string"), + ], +}; +const E_TIME_TRAVEL_RETENTION_SOURCE: EnumSpec = EnumSpec { + name: "TimeTravelRetentionSource", + doc: "Authored layer that supplied the effective retention policy.", + members: &[ + ("TARGET", "target"), + ("MATERIALIZATION", "materialization"), + ("MODEL", "model"), + ], +}; +const E_TABLE_TYPE: EnumSpec = EnumSpec { + name: "TableType", + doc: "Snowflake managed table type.", + members: &[("PERMANENT", "permanent"), ("TRANSIENT", "transient")], +}; +const E_TABLE_TYPE_SOURCE: EnumSpec = EnumSpec { + name: "TableTypeSource", + doc: "Authored layer that supplied the effective table type.", + members: &[ + ("DEFAULT", "default"), + ("TARGET", "target"), + ("MATERIALIZATION", "materialization"), + ("MODEL", "model"), + ], +}; +const E_TABLE_TYPE_DOWNGRADE_POLICY: EnumSpec = EnumSpec { + name: "TableTypeDowngradePolicy", + doc: "Permanent-to-transient conversion policy.", + members: &[ + ("DENY", "deny"), + ("REQUIRE_CONFIRMATION", "require_confirmation"), + ("ALLOW", "allow"), + ], +}; +const E_MISSING_MIGRATION_ORIGIN_POLICY: EnumSpec = EnumSpec { + name: "MissingMigrationOriginPolicy", + doc: "Policy for a declared migrate_from whose origin does not exist in a target.", + members: &[ + ("DENY", "deny"), + ("REQUIRE_CONFIRMATION", "require_confirmation"), + ("ALLOW", "allow"), + ], +}; +const E_RETENTION_DECREASE_POLICY: EnumSpec = EnumSpec { + name: "RetentionDecreasePolicy", + doc: "Policy for lowering live time-travel retention.", + members: &[ + ("DENY", "deny"), + ("REQUIRE_CONFIRMATION", "require_confirmation"), + ("ALLOW", "allow"), + ], +}; +const E_SQL_VALUE_KIND: EnumSpec = EnumSpec { + name: "SqlValueKind", + doc: "Closed logical kinds accepted by the typed SQL value boundary.", + members: &[ + ("STRING", "string"), + ("INTEGER", "integer"), + ("BOOLEAN", "boolean"), + ("FLOAT", "float"), + ("DECIMAL", "decimal"), + ("NULL", "null"), + ("LIST", "list"), + ("SET", "set"), + ("OBJECT", "object"), + ], +}; +const E_STATE_SQL_VALUE_TYPE: EnumSpec = EnumSpec { + name: "StateSqlValueType", + doc: "Closed physical value kinds used by warehouse state columns.", + members: &[ + ("STRING", "string"), + ("INTEGER", "integer"), + ("BOOLEAN", "boolean"), + ("TIMESTAMP", "timestamp"), + ("TEXT_TIMESTAMP", "text_timestamp"), + ("DATE", "date"), + ("JSON", "json"), + ], +}; /// Compiler, rule, lint and config string enums. pub(crate) const COMPILER_ENUMS: &[EnumSpec] = &[ @@ -334,4 +472,20 @@ pub(crate) const COMPILER_ENUMS: &[EnumSpec] = &[ E_INACCESSIBLE_REASON, E_SCOPE_DIAGNOSTIC_SEVERITY, E_SCOPE_DIAGNOSTIC_CODE, + E_EVENT_EXPORT_SEVERITY, + E_WAREHOUSE_GROUP, + E_SOURCE_WRITE_STRATEGY, + E_FUTURE_CURSOR_ACTION, + E_MICROBATCH_LIMIT_ACTION, + E_COLUMN_CONTRACT_MODE, + E_SOURCE_FRESHNESS_STRATEGY, + E_SOURCE_FRESHNESS_VALUE_KIND, + E_TIME_TRAVEL_RETENTION_SOURCE, + E_TABLE_TYPE, + E_TABLE_TYPE_SOURCE, + E_TABLE_TYPE_DOWNGRADE_POLICY, + E_MISSING_MIGRATION_ORIGIN_POLICY, + E_RETENTION_DECREASE_POLICY, + E_SQL_VALUE_KIND, + E_STATE_SQL_VALUE_TYPE, ]; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compiler_records/constants.rs b/crates/sqlbuild-python/src/bindings/_helpers/compiler_records/constants.rs index 9d37a9f40f..85baffdc92 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compiler_records/constants.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compiler_records/constants.rs @@ -781,6 +781,182 @@ const R_DBT_LS_RESULT: RecordSpec = RecordSpec { doc: "Parsed result from one dbt ls invocation.", fields: &[FieldSpec::required("nodes"), FieldSpec::required("command")], }; +const R_MEASUREMENT_THRESHOLD_BOUND: RecordSpec = RecordSpec { + name: "MeasurementThresholdBound", + doc: "One directional threshold bound for a measurement audit.", + fields: &[ + FieldSpec::required("operator"), + FieldSpec::new("limit", FieldDefault::None), + FieldSpec::new("lower", FieldDefault::None), + FieldSpec::new("upper", FieldDefault::None), + ], +}; +const R_MEASUREMENT_THRESHOLDS: RecordSpec = RecordSpec { + name: "MeasurementThresholds", + doc: "Warning and error policy for a measurement audit.", + fields: &[ + FieldSpec::new("warn", FieldDefault::None), + FieldSpec::new("error", FieldDefault::None), + ], +}; +const R_MEASUREMENT_CONTRACT: RecordSpec = RecordSpec { + name: "MeasurementContract", + doc: "Columns and sample unit returned by a measurement query.", + fields: &[ + FieldSpec::required("value_column"), + FieldSpec::new("sample_count_column", FieldDefault::None), + FieldSpec::new("sample_unit", FieldDefault::None), + ], +}; +const R_AUDIT_IDENTITY: RecordSpec = RecordSpec { + name: "AuditIdentity", + doc: "Stable identity values for one planned audit binding.", + fields: &[ + FieldSpec::required("binding_key"), + FieldSpec::required("audit_name"), + FieldSpec::required("definition_fingerprint"), + FieldSpec::required("execution_fingerprint"), + FieldSpec::required("severity"), + FieldSpec::required("run_scope_phase"), + FieldSpec::required("attachment_kind"), + FieldSpec::new("attached_target_name", FieldDefault::None), + FieldSpec::new("attached_column_name", FieldDefault::None), + FieldSpec::new("always_run", FieldDefault::False), + ], +}; +const R_AUDIT_GATE_IDENTITY: RecordSpec = RecordSpec { + name: "AuditGateIdentity", + doc: "Aggregate identity for a model's runtime audit gate.", + fields: &[ + FieldSpec::required("binding_set_hash"), + FieldSpec::required("blocking_set_hash"), + FieldSpec::required("audits"), + ], +}; +const R_SQL_LOGICAL_TYPE: RecordSpec = RecordSpec { + name: "SqlLogicalType", + doc: "Logical type inferred for a normalized SQL value.", + fields: &[ + FieldSpec::required("kind"), + FieldSpec::new("element_type", FieldDefault::None), + ], +}; +const R_SQL_VALUE: RecordSpec = RecordSpec { + name: "SqlValue", + doc: "One validated, canonical SQL value.", + fields: &[ + FieldSpec::required("logical_type"), + FieldSpec::required("value"), + ], +}; +const R_SELECTED_DBT_PROFILE_OUTPUT: RecordSpec = RecordSpec { + name: "SelectedDbtProfileOutput", + doc: "Selected dbt profile target output before rendering.", + fields: &[ + FieldSpec::required("profile_name"), + FieldSpec::required("target_name"), + FieldSpec::required("output"), + ], +}; +const R_RESOLVED_DBT_PROFILE_OUTPUT: RecordSpec = RecordSpec { + name: "ResolvedDbtProfileOutput", + doc: "Rendered dbt profile target output.", + fields: &[ + FieldSpec::required("project_dir"), + FieldSpec::required("profiles_dir"), + FieldSpec::required("profile_name"), + FieldSpec::required("target_name"), + FieldSpec::required("output"), + ], +}; +const R_DISCOVERED_PYTHON_NODE: RecordSpec = RecordSpec { + name: "DiscoveredPythonNode", + doc: "Shared internal view of a discovered Python DAG node.", + fields: &[ + FieldSpec::required("kind"), + FieldSpec::required("file_path"), + FieldSpec::required("relative_path"), + FieldSpec::required("name"), + FieldSpec::required("function"), + FieldSpec::new("depends_on", FieldDefault::EmptyTuple), + FieldSpec::new("sql_deps", FieldDefault::EmptyTuple), + FieldSpec::new("tags", FieldDefault::EmptyTuple), + FieldSpec::new("group", FieldDefault::None), + FieldSpec::new("description", FieldDefault::None), + FieldSpec::new("meta", FieldDefault::None), + FieldSpec::new("provider_usages", FieldDefault::EmptyTuple), + FieldSpec::new("loader", FieldDefault::None), + FieldSpec::new("task", FieldDefault::None), + FieldSpec::new("asset", FieldDefault::None), + FieldSpec::new("check", FieldDefault::None), + FieldSpec::new("identity", FieldDefault::None), + ], +}; +const R_PYTHON_NODE_GRAPH: RecordSpec = RecordSpec { + name: "PythonNodeGraph", + doc: "Internal graph inventory for discovered executable Python nodes.", + fields: &[ + FieldSpec::required("nodes"), + FieldSpec::required("dependency_edges"), + FieldSpec::required("upstream_deps"), + FieldSpec::required("downstream_deps"), + FieldSpec::required("tag_index"), + FieldSpec::required("path_index"), + FieldSpec::required("nodes_by_name"), + FieldSpec::required("nodes_by_typed_selector"), + ], +}; +const R_PYTHON_SQL_RUN_LIFECYCLE_PLAN: RecordSpec = RecordSpec { + name: "PythonSqlRunLifecyclePlan", + doc: "Lifecycle-aware run classification for SQL resources and Python nodes.", + fields: &[ + FieldSpec::required("ingress_python_node_names"), + FieldSpec::required("ingress_loader_names"), + FieldSpec::required("read_side_python_node_names"), + ], +}; +const R_PYTHON_IDENTITY_DEPENDENCY: RecordSpec = RecordSpec { + name: "PythonIdentityDependency", + doc: "One first-party Python source object included in a Python node identity.", + fields: &[ + FieldSpec::required("kind"), + FieldSpec::required("module"), + FieldSpec::required("qualname"), + FieldSpec::required("source_path"), + FieldSpec::required("source_hash"), + FieldSpec::required("source_text"), + ], +}; +const R_PYTHON_NODE_IDENTITY: RecordSpec = RecordSpec { + name: "PythonNodeIdentity", + doc: "Read-only identity metadata for a SQLBuild Python node.", + fields: &[ + FieldSpec::required("node_type"), + FieldSpec::required("node_name"), + FieldSpec::required("source_path"), + FieldSpec::required("source_hash"), + FieldSpec::required("definition_hash"), + FieldSpec::required("version_hash"), + FieldSpec::new("dependencies", FieldDefault::EmptyTuple), + FieldSpec::new("definition_json", FieldDefault::Text("{}")), + FieldSpec::new("metadata_json", FieldDefault::Text("{}")), + ], +}; +const R_SQL_LEXICAL_SYNTAX: RecordSpec = RecordSpec { + name: "SqlLexicalSyntax", + doc: "Dialect rules for where SQL quoted text and comments end.", + fields: &[ + FieldSpec::new("backslash_escape_quotes", FieldDefault::EmptyFrozenset), + FieldSpec::new("escape_string_prefix", FieldDefault::False), + FieldSpec::new("raw_string_prefix", FieldDefault::False), + FieldSpec::new("triple_quoted_strings", FieldDefault::False), + FieldSpec::new("nested_block_comments", FieldDefault::False), + FieldSpec::new( + "line_comment_prefixes", + FieldDefault::TextFrozenset(&["--"]), + ), + ], +}; /// The compiler, lint and config value records. pub(crate) const COMPILER_RECORDS: &[RecordSpec] = &[ @@ -849,4 +1025,19 @@ pub(crate) const COMPILER_RECORDS: &[RecordSpec] = &[ R_DBT_COMMAND_RESULT, R_DBT_NODE_EXECUTION_RESULT, R_DBT_LS_RESULT, + R_MEASUREMENT_THRESHOLD_BOUND, + R_MEASUREMENT_THRESHOLDS, + R_MEASUREMENT_CONTRACT, + R_AUDIT_IDENTITY, + R_AUDIT_GATE_IDENTITY, + R_SQL_LOGICAL_TYPE, + R_SQL_VALUE, + R_SELECTED_DBT_PROFILE_OUTPUT, + R_RESOLVED_DBT_PROFILE_OUTPUT, + R_DISCOVERED_PYTHON_NODE, + R_PYTHON_NODE_GRAPH, + R_PYTHON_SQL_RUN_LIFECYCLE_PLAN, + R_PYTHON_IDENTITY_DEPENDENCY, + R_PYTHON_NODE_IDENTITY, + R_SQL_LEXICAL_SYNTAX, ]; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/contract_records/constants.rs b/crates/sqlbuild-python/src/bindings/_helpers/contract_records/constants.rs new file mode 100644 index 0000000000..afd0ce4986 --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/contract_records/constants.rs @@ -0,0 +1,779 @@ +//! Layouts of the project configuration and contract value records hosted natively. + +use crate::bindings::_helpers::records::models::{FieldDefault, FieldSpec, RecordSpec}; + +const R_CLONE_POLICY: RecordSpec = RecordSpec { + name: "ClonePolicy", + doc: "Environment clone policy.", + fields: &[ + FieldSpec::new("allow_as_clone_origin", FieldDefault::False), + FieldSpec::new("allow_as_clone_destination", FieldDefault::False), + ], +}; +const R_LOCAL_CLONE_POLICY: RecordSpec = RecordSpec { + name: "LocalClonePolicy", + doc: "Local environment clone policy overrides.", + fields: &[ + FieldSpec::new("allow_as_clone_origin", FieldDefault::None), + FieldSpec::new("allow_as_clone_destination", FieldDefault::None), + ], +}; +const R_AUTHORED_TIME_TRAVEL_RETENTION: RecordSpec = RecordSpec { + name: "AuthoredTimeTravelRetention", + doc: "One authored time-travel retention policy.", + fields: &[ + FieldSpec::new("desired_days", FieldDefault::None), + FieldSpec::new("unmanaged", FieldDefault::False), + ], +}; +const R_MATERIALIZATION_RETENTION_DEFAULTS: RecordSpec = RecordSpec { + name: "MaterializationRetentionDefaults", + doc: "Retention defaults for one built-in materialization.", + fields: &[ + FieldSpec::new("time_travel_retention", FieldDefault::None), + FieldSpec::new("table_type", FieldDefault::None), + ], +}; +const R_MATERIALIZATION_DEFAULTS_CONFIG: RecordSpec = RecordSpec { + name: "MaterializationDefaultsConfig", + doc: "Project retention defaults by materialization type.", + fields: &[ + FieldSpec::new( + "table", + FieldDefault::Call("sqlbuild._native", "MaterializationRetentionDefaults"), + ), + FieldSpec::new( + "incremental", + FieldDefault::Call("sqlbuild._native", "MaterializationRetentionDefaults"), + ), + FieldSpec::new( + "snapshot", + FieldDefault::Call("sqlbuild._native", "MaterializationRetentionDefaults"), + ), + ], +}; +const R_RESOLVED_TIME_TRAVEL_RETENTION: RecordSpec = RecordSpec { + name: "ResolvedTimeTravelRetention", + doc: "Effective retention policy attached to a compiled model.", + fields: &[ + FieldSpec::new("desired_days", FieldDefault::None), + FieldSpec::new("unmanaged", FieldDefault::True), + FieldSpec::new("source", FieldDefault::None), + ], +}; +const R_RESOLVED_TABLE_TYPE: RecordSpec = RecordSpec { + name: "ResolvedTableType", + doc: "Effective Snowflake table type attached to a compiled model.", + fields: &[ + FieldSpec::new( + "value", + FieldDefault::Member("sqlbuild._native", "TableType", "TRANSIENT"), + ), + FieldSpec::new( + "source", + FieldDefault::Member("sqlbuild._native", "TableTypeSource", "DEFAULT"), + ), + FieldSpec::new("declared", FieldDefault::False), + ], +}; +const R_EXECUTION_LIMITS_CONFIG: RecordSpec = RecordSpec { + name: "ExecutionLimitsConfig", + doc: "Optional target-scoped build execution limits.", + fields: &[ + FieldSpec::new("max_models", FieldDefault::None), + FieldSpec::new("max_duration", FieldDefault::None), + FieldSpec::new("max_duration_seconds", FieldDefault::None), + FieldSpec::new("remediation", FieldDefault::None), + ], +}; +const R_TARGET_DIFF_CONFIG: RecordSpec = RecordSpec { + name: "TargetDiffConfig", + doc: "Optional target-scoped data-diff limits.", + fields: &[FieldSpec::new("max_full_rows", FieldDefault::None)], +}; +const R_TARGET_WAREHOUSES_CONFIG: RecordSpec = RecordSpec { + name: "TargetWarehousesConfig", + doc: "Optional per-command-group default warehouses for one target.", + fields: &[ + FieldSpec::new("build", FieldDefault::None), + FieldSpec::new("query", FieldDefault::None), + ], +}; +const R_TARGET_CONFIG: RecordSpec = RecordSpec { + name: "TargetConfig", + doc: "One named target configuration.", + fields: &[ + FieldSpec::new("connection", FieldDefault::EmptyDict), + FieldSpec::new("connection_name", FieldDefault::None), + FieldSpec::new("vars", FieldDefault::EmptyDict), + FieldSpec::new("database", FieldDefault::None), + FieldSpec::new("schema", FieldDefault::None), + FieldSpec::new("loader_schema", FieldDefault::None), + FieldSpec::new("defer_sources_to", FieldDefault::None), + FieldSpec::new("defer_clone_from", FieldDefault::None), + FieldSpec::new( + "clone", + FieldDefault::Call("sqlbuild._native", "ClonePolicy"), + ), + FieldSpec::new("compile_cache", FieldDefault::None), + FieldSpec::new("time_travel_retention", FieldDefault::None), + FieldSpec::new( + "time_travel_retention_by_materialization", + FieldDefault::EmptyDict, + ), + FieldSpec::new("owns_time_travel_retention_namespace", FieldDefault::False), + FieldSpec::new("default_table_type", FieldDefault::None), + FieldSpec::new( + "table_type_downgrade", + FieldDefault::Member( + "sqlbuild._native", + "TableTypeDowngradePolicy", + "REQUIRE_CONFIRMATION", + ), + ), + FieldSpec::new( + "time_travel_retention_decrease", + FieldDefault::Member("sqlbuild._native", "RetentionDecreasePolicy", "DENY"), + ), + FieldSpec::new( + "missing_migration_origin", + FieldDefault::Member("sqlbuild._native", "MissingMigrationOriginPolicy", "ALLOW"), + ), + FieldSpec::new( + "execution_limits", + FieldDefault::Call("sqlbuild._native", "ExecutionLimitsConfig"), + ), + FieldSpec::new( + "diff", + FieldDefault::Call("sqlbuild._native", "TargetDiffConfig"), + ), + FieldSpec::new( + "warehouses", + FieldDefault::Call("sqlbuild._native", "TargetWarehousesConfig"), + ), + ], +}; +const R_LOCAL_TARGET_CONFIG: RecordSpec = RecordSpec { + name: "LocalTargetConfig", + doc: "Local developer overrides for one named target.", + fields: &[ + FieldSpec::new("connection", FieldDefault::EmptyDict), + FieldSpec::new("connection_name", FieldDefault::None), + FieldSpec::new("vars", FieldDefault::EmptyDict), + FieldSpec::new("database", FieldDefault::None), + FieldSpec::new("schema", FieldDefault::None), + FieldSpec::new("loader_schema", FieldDefault::None), + FieldSpec::new("defer_sources_to", FieldDefault::None), + FieldSpec::new("defer_clone_from", FieldDefault::None), + FieldSpec::new( + "clone", + FieldDefault::Call("sqlbuild._native", "LocalClonePolicy"), + ), + FieldSpec::new("compile_cache", FieldDefault::None), + FieldSpec::new("time_travel_retention", FieldDefault::None), + FieldSpec::new( + "time_travel_retention_by_materialization", + FieldDefault::None, + ), + FieldSpec::new("owns_time_travel_retention_namespace", FieldDefault::None), + FieldSpec::new("default_table_type", FieldDefault::None), + FieldSpec::new("table_type_downgrade", FieldDefault::None), + FieldSpec::new("time_travel_retention_decrease", FieldDefault::None), + FieldSpec::new("missing_migration_origin", FieldDefault::None), + FieldSpec::new( + "execution_limits", + FieldDefault::Call("sqlbuild._native", "ExecutionLimitsConfig"), + ), + FieldSpec::new( + "diff", + FieldDefault::Call("sqlbuild._native", "TargetDiffConfig"), + ), + FieldSpec::new( + "warehouses", + FieldDefault::Call("sqlbuild._native", "TargetWarehousesConfig"), + ), + ], +}; +const R_SETTINGS_CONFIG: RecordSpec = RecordSpec { + name: "SettingsConfig", + doc: "Global feature toggles.", + fields: &[ + FieldSpec::new("sql_analysis", FieldDefault::True), + FieldSpec::new("require_sql_analysis", FieldDefault::False), + FieldSpec::new("query_change_tracking", FieldDefault::True), + FieldSpec::new( + "column_contract_mode", + FieldDefault::Member("sqlbuild._native", "ColumnContractMode", "IMPLICIT"), + ), + FieldSpec::new("concurrency", FieldDefault::Int(1)), + FieldSpec::new("auto_load_sources", FieldDefault::True), + FieldSpec::new("microbatch_concurrency", FieldDefault::False), + FieldSpec::new( + "microbatch_unaccounted_partition_policy", + FieldDefault::Text("synthesize"), + ), + FieldSpec::new("table_promotion_mode", FieldDefault::None), + FieldSpec::new("default_audit_severity", FieldDefault::None), + FieldSpec::new("default_audit_run_scope", FieldDefault::None), + ], +}; +const R_SCOPES_CONFIG: RecordSpec = RecordSpec { + name: "ScopesConfig", + doc: "Declaration scope policy configuration.", + fields: &[FieldSpec::new("enforce_placement", FieldDefault::True)], +}; +const R_REFERENCES_CONFIG: RecordSpec = RecordSpec { + name: "ReferencesConfig", + doc: "Explicit-reference enforcement policy configuration.", + fields: &[FieldSpec::new("enforce_explicit", FieldDefault::True)], +}; +const R_COST_CONFIG: RecordSpec = RecordSpec { + name: "CostConfig", + doc: "Snowflake compute estimate configuration.", + fields: &[ + FieldSpec::new( + "usd_per_credit", + FieldDefault::CallText("decimal", "Decimal", "3.00"), + ), + FieldSpec::new("usd_per_credit_is_default", FieldDefault::True), + ], +}; +const R_CONSTANTS_CONFIG: RecordSpec = RecordSpec { + name: "ConstantsConfig", + doc: "Project-wide constant rendering configuration.", + fields: &[FieldSpec::new( + "collection_rendering", + FieldDefault::Member("sqlbuild._native", "CollectionRendering", "VALUE_LIST"), + )], +}; +const R_FUTURE_CURSORS_CONFIG: RecordSpec = RecordSpec { + name: "FutureCursorsConfig", + doc: "Safety policy for automatically discovered future cursor watermarks.", + fields: &[ + FieldSpec::new("max_distance", FieldDefault::None), + FieldSpec::new( + "action", + FieldDefault::Member("sqlbuild._native", "FutureCursorAction", "ERROR"), + ), + ], +}; +const R_START_CURSORS_CONFIG: RecordSpec = RecordSpec { + name: "StartCursorsConfig", + doc: "Safety policy for automatically discovered incremental starts.", + fields: &[ + FieldSpec::new("max_ahead", FieldDefault::None), + FieldSpec::new( + "action", + FieldDefault::Member("sqlbuild._native", "FutureCursorAction", "ERROR"), + ), + ], +}; +const R_CURSORS_CONFIG: RecordSpec = RecordSpec { + name: "CursorsConfig", + doc: "Project-wide cursor safety configuration.", + fields: &[ + FieldSpec::new( + "future", + FieldDefault::Call("sqlbuild._native", "FutureCursorsConfig"), + ), + FieldSpec::new( + "start", + FieldDefault::Call("sqlbuild._native", "StartCursorsConfig"), + ), + ], +}; +const R_MICROBATCH_LIMITS_CONFIG: RecordSpec = RecordSpec { + name: "MicrobatchLimitsConfig", + doc: "Per-model microbatch planning safety limit.", + fields: &[ + FieldSpec::new("max_batches", FieldDefault::None), + FieldSpec::new( + "action", + FieldDefault::Member("sqlbuild._native", "MicrobatchLimitAction", "ERROR"), + ), + ], +}; +const R_MICROBATCHES_CONFIG: RecordSpec = RecordSpec { + name: "MicrobatchesConfig", + doc: "Project-wide microbatch configuration.", + fields: &[FieldSpec::new( + "limits", + FieldDefault::Call("sqlbuild._native", "MicrobatchLimitsConfig"), + )], +}; +const R_DEFAULTS_CONFIG: RecordSpec = RecordSpec { + name: "DefaultsConfig", + doc: "Project-wide resource defaults.", + fields: &[ + FieldSpec::new("materialized", FieldDefault::None), + FieldSpec::new("database", FieldDefault::None), + FieldSpec::new("schema", FieldDefault::None), + FieldSpec::new("seed_database", FieldDefault::None), + FieldSpec::new("seed_schema", FieldDefault::None), + FieldSpec::new("function_database", FieldDefault::None), + FieldSpec::new("function_schema", FieldDefault::None), + FieldSpec::new("incremental_strategy", FieldDefault::None), + FieldSpec::new("incremental_mode", FieldDefault::None), + FieldSpec::new("microbatch_strategy", FieldDefault::None), + FieldSpec::new("cursor_watermark_mode", FieldDefault::None), + FieldSpec::new("merge_exclude_columns", FieldDefault::EmptyTuple), + FieldSpec::new("full_refresh", FieldDefault::None), + FieldSpec::new("append_cursor_inclusive", FieldDefault::None), + FieldSpec::new("cursor_start", FieldDefault::None), + FieldSpec::new("cursor_end", FieldDefault::None), + FieldSpec::new("cursor_start_max_ahead", FieldDefault::None), + FieldSpec::new("cursor_start_max_action", FieldDefault::None), + FieldSpec::new("cursor_future_max_distance", FieldDefault::None), + FieldSpec::new("cursor_future_action", FieldDefault::None), + FieldSpec::new("lookback", FieldDefault::None), + FieldSpec::new("batch_size", FieldDefault::None), + FieldSpec::new("batch_concurrency", FieldDefault::None), + FieldSpec::new("max_microbatches", FieldDefault::None), + FieldSpec::new("unaccounted_partition_policy", FieldDefault::None), + FieldSpec::new("replay_on_change", FieldDefault::None), + FieldSpec::new("row_diff_exclude_columns", FieldDefault::EmptyTuple), + FieldSpec::new("row_diff_tolerances", FieldDefault::EmptyDict), + FieldSpec::new("row_diff_sample_rows", FieldDefault::None), + FieldSpec::new("row_diff_sample_seed", FieldDefault::None), + FieldSpec::new("tags", FieldDefault::EmptyTuple), + FieldSpec::new("contract", FieldDefault::None), + FieldSpec::new("pre_hooks", FieldDefault::None), + FieldSpec::new("post_hooks", FieldDefault::None), + ], +}; +const R_DIFF_CONFIG: RecordSpec = RecordSpec { + name: "DiffConfig", + doc: "Project-wide data-diff configuration.", + fields: &[FieldSpec::new( + "query_artifact_ttl", + FieldDefault::Text("24h"), + )], +}; +const R_JANITOR_CONFIG: RecordSpec = RecordSpec { + name: "JanitorConfig", + doc: "Janitor command defaults.", + fields: &[ + FieldSpec::new("enabled", FieldDefault::False), + FieldSpec::new("retention_days", FieldDefault::None), + FieldSpec::new("archive_retention_days", FieldDefault::Int(14)), + FieldSpec::new("direct_state_history_versions", FieldDefault::Int(20)), + FieldSpec::new("delete_tracked_only", FieldDefault::True), + FieldSpec::new("exclude_patterns", FieldDefault::EmptyTuple), + ], +}; +const R_MIGRATIONS_CONFIG: RecordSpec = RecordSpec { + name: "MigrationsConfig", + doc: "Model migration defaults.", + fields: &[FieldSpec::new("old_name_views", FieldDefault::Text("30d"))], +}; +const R_SNAPSHOTS_CONFIG: RecordSpec = RecordSpec { + name: "SnapshotsConfig", + doc: "Snapshot materialization safety defaults.", + fields: &[ + FieldSpec::new("current_state_full_refresh", FieldDefault::Text("deny")), + FieldSpec::new( + "historical_full_refresh", + FieldDefault::Text("require_confirmation"), + ), + FieldSpec::new("schema_change", FieldDefault::Text("append_new_columns")), + FieldSpec::new( + "wildcard_check_schema_change", + FieldDefault::Text("require_confirmation"), + ), + ], +}; +const R_SCENARIO_SNAPSHOT_LIMITS_CONFIG: RecordSpec = RecordSpec { + name: "ScenarioSnapshotLimitsConfig", + doc: "Scenario snapshot capture safety limits.", + fields: &[ + FieldSpec::new("max_rows_per_relation", FieldDefault::None), + FieldSpec::new("max_total_rows", FieldDefault::None), + FieldSpec::new("max_bytes_per_relation", FieldDefault::None), + FieldSpec::new("max_total_bytes", FieldDefault::None), + ], +}; +const R_SCENARIO_CONFIG: RecordSpec = RecordSpec { + name: "ScenarioConfig", + doc: "Scenario command configuration.", + fields: &[ + FieldSpec::new("run_namespace", FieldDefault::None), + FieldSpec::new("local_type_overrides", FieldDefault::EmptyDict), + FieldSpec::new( + "snapshot_limits", + FieldDefault::Call("sqlbuild._native", "ScenarioSnapshotLimitsConfig"), + ), + ], +}; +const R_DBT_CONFIG: RecordSpec = RecordSpec { + name: "DbtConfig", + doc: "dbt interop configuration.", + fields: &[ + FieldSpec::new("project_dir", FieldDefault::None), + FieldSpec::new("profiles_dir", FieldDefault::None), + FieldSpec::new("target", FieldDefault::None), + FieldSpec::new("target_path", FieldDefault::None), + FieldSpec::new("vars", FieldDefault::EmptyDict), + ], +}; +const R_LOCAL_DBT_CONFIG: RecordSpec = RecordSpec { + name: "LocalDbtConfig", + doc: "Local dbt interop configuration overrides.", + fields: &[ + FieldSpec::new("target", FieldDefault::None), + FieldSpec::new("vars", FieldDefault::EmptyDict), + ], +}; +const R_LIFECYCLE_EVENT_SINK_FILTER_CONFIG: RecordSpec = RecordSpec { + name: "LifecycleEventSinkFilterConfig", + doc: "Destination-neutral runtime narrowing for lifecycle-event sinks.", + fields: &[ + FieldSpec::new("event_kinds", FieldDefault::None), + FieldSpec::new("min_severity", FieldDefault::None), + ], +}; +const R_LIFECYCLE_EVENT_SINKS_CONFIG: RecordSpec = RecordSpec { + name: "LifecycleEventSinksConfig", + doc: "Global and named lifecycle-event sink runtime filters and delivery settings.", + fields: &[ + FieldSpec::new( + "defaults", + FieldDefault::Call("sqlbuild._native", "LifecycleEventSinkFilterConfig"), + ), + FieldSpec::new("named", FieldDefault::EmptyDict), + FieldSpec::new("shutdown_timeout", FieldDefault::None), + FieldSpec::new("shutdown_timeout_seconds", FieldDefault::None), + ], +}; +const R_SINKS_CONFIG: RecordSpec = RecordSpec { + name: "SinksConfig", + doc: "Typed destination sink configuration.", + fields: &[FieldSpec::new( + "lifecycle", + FieldDefault::Call("sqlbuild._native", "LifecycleEventSinksConfig"), + )], +}; +const R_PROJECT_CONFIG: RecordSpec = RecordSpec { + name: "ProjectConfig", + doc: "Shared project configuration loaded from sqlbuild_project.toml.", + fields: &[ + FieldSpec::required("name"), + FieldSpec::required("adapter"), + FieldSpec::new("default_target", FieldDefault::None), + FieldSpec::new("connection", FieldDefault::EmptyDict), + FieldSpec::new("connections", FieldDefault::EmptyDict), + FieldSpec::new( + "settings", + FieldDefault::Call("sqlbuild._native", "SettingsConfig"), + ), + FieldSpec::new( + "scopes", + FieldDefault::Call("sqlbuild._native", "ScopesConfig"), + ), + FieldSpec::new( + "references", + FieldDefault::Call("sqlbuild._native", "ReferencesConfig"), + ), + FieldSpec::new("cost", FieldDefault::Call("sqlbuild._native", "CostConfig")), + FieldSpec::new( + "constants", + FieldDefault::Call("sqlbuild._native", "ConstantsConfig"), + ), + FieldSpec::new( + "cursors", + FieldDefault::Call("sqlbuild._native", "CursorsConfig"), + ), + FieldSpec::new( + "microbatches", + FieldDefault::Call("sqlbuild._native", "MicrobatchesConfig"), + ), + FieldSpec::new( + "defaults", + FieldDefault::Call("sqlbuild._native", "DefaultsConfig"), + ), + FieldSpec::new("diff", FieldDefault::Call("sqlbuild._native", "DiffConfig")), + FieldSpec::new( + "materialization_defaults", + FieldDefault::Call("sqlbuild._native", "MaterializationDefaultsConfig"), + ), + FieldSpec::new("path_defaults", FieldDefault::EmptyDict), + FieldSpec::new("vars", FieldDefault::EmptyDict), + FieldSpec::new("targets", FieldDefault::EmptyDict), + FieldSpec::new( + "janitor", + FieldDefault::Call("sqlbuild._native", "JanitorConfig"), + ), + FieldSpec::new( + "migrations", + FieldDefault::Call("sqlbuild._native", "MigrationsConfig"), + ), + FieldSpec::new( + "snapshots", + FieldDefault::Call("sqlbuild._native", "SnapshotsConfig"), + ), + FieldSpec::new( + "scenario", + FieldDefault::Call("sqlbuild._native", "ScenarioConfig"), + ), + FieldSpec::new("dbt", FieldDefault::Call("sqlbuild._native", "DbtConfig")), + FieldSpec::new( + "sinks", + FieldDefault::Call("sqlbuild._native", "SinksConfig"), + ), + ], +}; +const R_LOCAL_CONFIG: RecordSpec = RecordSpec { + name: "LocalConfig", + doc: "Local developer overrides from sqlbuild_local.toml.", + fields: &[ + FieldSpec::new("target", FieldDefault::None), + FieldSpec::new("adapter", FieldDefault::None), + FieldSpec::new("connection", FieldDefault::EmptyDict), + FieldSpec::new("connections", FieldDefault::EmptyDict), + FieldSpec::new("targets", FieldDefault::EmptyDict), + FieldSpec::new( + "settings", + FieldDefault::Call("sqlbuild._native", "SettingsConfig"), + ), + FieldSpec::new("setting_overrides", FieldDefault::EmptyFrozenset), + FieldSpec::new("vars", FieldDefault::EmptyDict), + FieldSpec::new( + "dbt", + FieldDefault::Call("sqlbuild._native", "LocalDbtConfig"), + ), + FieldSpec::new( + "scenario", + FieldDefault::Call("sqlbuild._native", "ScenarioConfig"), + ), + ], +}; +const R_SOURCE_LOCATION: RecordSpec = RecordSpec { + name: "SourceLocation", + doc: "An authored source location for compiler diagnostics.", + fields: &[ + FieldSpec::required("path"), + FieldSpec::required("line"), + FieldSpec::required("column"), + FieldSpec::new("end_line", FieldDefault::None), + FieldSpec::new("end_column", FieldDefault::None), + ], +}; +const R_SEED_CSV_SETTINGS: RecordSpec = RecordSpec { + name: "SeedCsvSettings", + doc: "CSV reader settings for a seed file.", + fields: &[ + FieldSpec::new("delimiter", FieldDefault::None), + FieldSpec::new("quotechar", FieldDefault::None), + FieldSpec::new("doublequote", FieldDefault::None), + FieldSpec::new("escapechar", FieldDefault::None), + FieldSpec::new("skipinitialspace", FieldDefault::None), + FieldSpec::new("lineterminator", FieldDefault::None), + FieldSpec::new("encoding", FieldDefault::None), + FieldSpec::new("na_values", FieldDefault::None), + FieldSpec::new("keep_default_na", FieldDefault::None), + ], +}; +const R_SCHEMA_AUDIT_INSTANCE: RecordSpec = RecordSpec { + name: "SchemaAuditInstance", + doc: "One audit instance attached to model, column, or seed metadata.", + fields: &[ + FieldSpec::required("definition_name"), + FieldSpec::new("arguments", FieldDefault::EmptyDict), + FieldSpec::new("name", FieldDefault::None), + FieldSpec::new("description", FieldDefault::None), + FieldSpec::new("severity", FieldDefault::None), + FieldSpec::new("run_scope", FieldDefault::None), + FieldSpec::new("always_run", FieldDefault::False), + FieldSpec::new("thresholds", FieldDefault::None), + FieldSpec::new("minimum_samples", FieldDefault::None), + FieldSpec::new("evidence_limit", FieldDefault::None), + FieldSpec { + name: "location", + default: FieldDefault::None, + compared: false, + }, + ], +}; +const R_SCHEMA_COLUMN: RecordSpec = RecordSpec { + name: "SchemaColumn", + doc: "One declared model, seed, or source column entry.", + fields: &[ + FieldSpec::required("name"), + FieldSpec::new("type", FieldDefault::None), + FieldSpec::new("nullable", FieldDefault::None), + FieldSpec::new("description", FieldDefault::None), + FieldSpec::new("meta", FieldDefault::EmptyDict), + FieldSpec::new("audits", FieldDefault::EmptyTuple), + FieldSpec::new("location", FieldDefault::None), + FieldSpec::new("migrate_from", FieldDefault::None), + ], +}; +const R_SCHEMA_DYNAMIC_COLUMN_FAMILY: RecordSpec = RecordSpec { + name: "SchemaDynamicColumnFamily", + doc: "One declared family of columns generated by a runtime-dynamic pivot.", + fields: &[ + FieldSpec::required("name"), + FieldSpec::required("pivot_column"), + FieldSpec::required("value_column"), + FieldSpec::required("aggregate"), + FieldSpec::required("type"), + FieldSpec::new("name_pattern", FieldDefault::None), + ], +}; +const R_SCHEMA_MODEL_ENTRY: RecordSpec = RecordSpec { + name: "SchemaModelEntry", + doc: "One model metadata entry normalized from MODEL(...).", + fields: &[ + FieldSpec::required("name"), + FieldSpec::new("model_schema", FieldDefault::None), + FieldSpec::new("description", FieldDefault::None), + FieldSpec::new("type_enforcement", FieldDefault::None), + FieldSpec::new("meta", FieldDefault::EmptyDict), + FieldSpec::new("columns", FieldDefault::EmptyTuple), + FieldSpec::new("dynamic_columns", FieldDefault::EmptyTuple), + FieldSpec::new("audits", FieldDefault::EmptyTuple), + FieldSpec::new("tags", FieldDefault::EmptyTuple), + ], +}; +const R_SCHEMA_SEED_ENTRY: RecordSpec = RecordSpec { + name: "SchemaSeedEntry", + doc: "One seed metadata entry from seed YAML.", + fields: &[ + FieldSpec::required("name"), + FieldSpec::new("description", FieldDefault::None), + FieldSpec::new("database", FieldDefault::None), + FieldSpec::new("schema", FieldDefault::None), + FieldSpec::new("meta", FieldDefault::EmptyDict), + FieldSpec::new( + "csv_settings", + FieldDefault::Call("sqlbuild._native", "SeedCsvSettings"), + ), + FieldSpec::new("columns", FieldDefault::EmptyTuple), + FieldSpec::new("tags", FieldDefault::EmptyTuple), + FieldSpec::new("audits", FieldDefault::EmptyTuple), + ], +}; +const R_INTEGRATION_LOADER_CONFIG: RecordSpec = RecordSpec { + name: "IntegrationLoaderConfig", + doc: "Typed configuration for one declarative integration loader.", + fields: &[FieldSpec::required("kind"), FieldSpec::required("config")], +}; +const R_SOURCE_FRESHNESS_AGE_POLICY: RecordSpec = RecordSpec { + name: "SourceFreshnessAgePolicy", + doc: "Age-based source freshness warning/error thresholds.", + fields: &[ + FieldSpec::new("warn_after", FieldDefault::None), + FieldSpec::new("error_after", FieldDefault::None), + ], +}; +const R_SOURCE_FRESHNESS_CONFIG: RecordSpec = RecordSpec { + name: "SourceFreshnessConfig", + doc: "Configured source freshness observation for planning.", + fields: &[ + FieldSpec::required("strategy"), + FieldSpec::new("value_kind", FieldDefault::None), + FieldSpec::new("column", FieldDefault::None), + FieldSpec::new("query", FieldDefault::None), + FieldSpec::new("filter", FieldDefault::None), + FieldSpec::new("lag_tolerance", FieldDefault::None), + FieldSpec::new("age_policy", FieldDefault::None), + ], +}; +const R_SOURCE_COLUMN_ENTRY: RecordSpec = RecordSpec { + name: "SourceColumnEntry", + doc: "One source column entry from sources/*.yml.", + fields: &[ + FieldSpec::required("name"), + FieldSpec::new("type", FieldDefault::None), + FieldSpec::new("nullable", FieldDefault::None), + FieldSpec::new("description", FieldDefault::None), + FieldSpec::new("meta", FieldDefault::EmptyDict), + FieldSpec::new("audits", FieldDefault::EmptyTuple), + ], +}; +const R_SOURCE_ENTRY: RecordSpec = RecordSpec { + name: "SourceEntry", + doc: "One source declaration from sources/*.yml.", + fields: &[ + FieldSpec::required("name"), + FieldSpec::new("database", FieldDefault::None), + FieldSpec::new("schema", FieldDefault::None), + FieldSpec::new("table", FieldDefault::None), + FieldSpec::new("loader", FieldDefault::None), + FieldSpec::new("managed", FieldDefault::False), + FieldSpec::new("integration_loader", FieldDefault::None), + FieldSpec::new("freshness", FieldDefault::None), + FieldSpec::new("write_strategy", FieldDefault::None), + FieldSpec::new("load_batch_size", FieldDefault::None), + FieldSpec::new("cursor_column", FieldDefault::None), + FieldSpec::new("unique_key", FieldDefault::EmptyTuple), + FieldSpec::new("expression", FieldDefault::None), + FieldSpec::new("description", FieldDefault::None), + FieldSpec::new("type_enforcement", FieldDefault::None), + FieldSpec::new("contract", FieldDefault::None), + FieldSpec::new("meta", FieldDefault::EmptyDict), + FieldSpec::new("columns", FieldDefault::EmptyTuple), + FieldSpec::new("audits", FieldDefault::EmptyTuple), + ], +}; +const R_LOADER_DESTINATION_PARTS: RecordSpec = RecordSpec { + name: "LoaderDestinationParts", + doc: "Parsed warehouse relation parts for a loader destination.", + fields: &[ + FieldSpec::required("database"), + FieldSpec::required("schema"), + FieldSpec::required("table"), + ], +}; + +/// The project configuration and contract value records. +pub(crate) const CONTRACT_RECORDS: &[RecordSpec] = &[ + R_CLONE_POLICY, + R_LOCAL_CLONE_POLICY, + R_AUTHORED_TIME_TRAVEL_RETENTION, + R_MATERIALIZATION_RETENTION_DEFAULTS, + R_MATERIALIZATION_DEFAULTS_CONFIG, + R_RESOLVED_TIME_TRAVEL_RETENTION, + R_RESOLVED_TABLE_TYPE, + R_EXECUTION_LIMITS_CONFIG, + R_TARGET_DIFF_CONFIG, + R_TARGET_WAREHOUSES_CONFIG, + R_TARGET_CONFIG, + R_LOCAL_TARGET_CONFIG, + R_SETTINGS_CONFIG, + R_SCOPES_CONFIG, + R_REFERENCES_CONFIG, + R_COST_CONFIG, + R_CONSTANTS_CONFIG, + R_FUTURE_CURSORS_CONFIG, + R_START_CURSORS_CONFIG, + R_CURSORS_CONFIG, + R_MICROBATCH_LIMITS_CONFIG, + R_MICROBATCHES_CONFIG, + R_DEFAULTS_CONFIG, + R_DIFF_CONFIG, + R_JANITOR_CONFIG, + R_MIGRATIONS_CONFIG, + R_SNAPSHOTS_CONFIG, + R_SCENARIO_SNAPSHOT_LIMITS_CONFIG, + R_SCENARIO_CONFIG, + R_DBT_CONFIG, + R_LOCAL_DBT_CONFIG, + R_LIFECYCLE_EVENT_SINK_FILTER_CONFIG, + R_LIFECYCLE_EVENT_SINKS_CONFIG, + R_SINKS_CONFIG, + R_PROJECT_CONFIG, + R_LOCAL_CONFIG, + R_SOURCE_LOCATION, + R_SEED_CSV_SETTINGS, + R_SCHEMA_AUDIT_INSTANCE, + R_SCHEMA_COLUMN, + R_SCHEMA_DYNAMIC_COLUMN_FAMILY, + R_SCHEMA_MODEL_ENTRY, + R_SCHEMA_SEED_ENTRY, + R_INTEGRATION_LOADER_CONFIG, + R_SOURCE_FRESHNESS_AGE_POLICY, + R_SOURCE_FRESHNESS_CONFIG, + R_SOURCE_COLUMN_ENTRY, + R_SOURCE_ENTRY, + R_LOADER_DESTINATION_PARTS, +]; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/contract_records/mod.rs b/crates/sqlbuild-python/src/bindings/_helpers/contract_records/mod.rs new file mode 100644 index 0000000000..61d449b063 --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/contract_records/mod.rs @@ -0,0 +1 @@ +pub(crate) mod constants; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/contracts/contract_diagnostics.rs b/crates/sqlbuild-python/src/bindings/_helpers/contracts/contract_diagnostics.rs index 291232f396..ff294bfbdb 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/contracts/contract_diagnostics.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/contracts/contract_diagnostics.rs @@ -19,7 +19,7 @@ const NATIVE_MODULE: &str = "sqlbuild._native"; const COMPILE_MODELS_MODULE: &str = "sqlbuild.compiler.compile.models"; const COMPILE_TYPES_MODULE: &str = "sqlbuild.compiler.compile.types"; const CONTRACT_MODELS_MODULE: &str = "sqlbuild._native"; -const SPEC_MODELS_MODULE: &str = "sqlbuild.spec.contracts.models"; +const SPEC_MODELS_MODULE: &str = "sqlbuild._native"; const PATHLIB_MODULE: &str = "pathlib"; const IMPLICIT_COLUMN_CONTRACTS: &str = "implicit"; const GENERIC_DIALECT: &str = "generic"; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/contracts/project_config.rs b/crates/sqlbuild-python/src/bindings/_helpers/contracts/project_config.rs index 8f4c9e76a3..c08e048f41 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/contracts/project_config.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/contracts/project_config.rs @@ -17,7 +17,7 @@ use sqlbuild_config::project::models::{Built, ConfigFile, ConfigPaths}; use crate::bindings::_helpers::boundary::config_values::config_value_to_python; -const MODELS_MODULE: &str = "sqlbuild.spec.contracts.models"; +const MODELS_MODULE: &str = "sqlbuild._native"; const ERRORS_MODULE: &str = "sqlbuild._native"; const PROJECT_CONFIG_ERROR: &str = "ProjectConfigError"; const FILE_NAMES: [&str; 4] = [ @@ -28,17 +28,14 @@ const FILE_NAMES: [&str; 4] = [ ]; /// The module of each `StrEnum` a loaded config holds. const ENUM_MODULES: [(&str, &str); 10] = [ - ("ColumnContractMode", "sqlbuild.spec.contracts.types"), - ("FutureCursorAction", "sqlbuild.spec.contracts.types"), - ("MicrobatchLimitAction", "sqlbuild.spec.contracts.types"), - ("TableType", "sqlbuild.spec.contracts.types"), - ("TableTypeDowngradePolicy", "sqlbuild.spec.contracts.types"), - ("RetentionDecreasePolicy", "sqlbuild.spec.contracts.types"), - ( - "MissingMigrationOriginPolicy", - "sqlbuild.spec.contracts.types", - ), - ("EventExportSeverity", "sqlbuild.spec.contracts.types"), + ("ColumnContractMode", "sqlbuild._native"), + ("FutureCursorAction", "sqlbuild._native"), + ("MicrobatchLimitAction", "sqlbuild._native"), + ("TableType", "sqlbuild._native"), + ("TableTypeDowngradePolicy", "sqlbuild._native"), + ("RetentionDecreasePolicy", "sqlbuild._native"), + ("MissingMigrationOriginPolicy", "sqlbuild._native"), + ("EventExportSeverity", "sqlbuild._native"), ("AuditSeverity", "sqlbuild.compiler.auditing.types"), ("CollectionRendering", "sqlbuild._native"), ]; @@ -131,7 +128,7 @@ fn enum_member<'py>(py: Python<'py>, class: &str, value: &str) -> PyResult) -> PyResult> { diff --git a/crates/sqlbuild-python/src/bindings/_helpers/cursor/conversion.rs b/crates/sqlbuild-python/src/bindings/_helpers/cursor/conversion.rs index 80c115906e..4d22920f54 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/cursor/conversion.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/cursor/conversion.rs @@ -19,7 +19,7 @@ use crate::bindings::_helpers::cursor::values::{ timedelta_micros, }; -pub(crate) const MODELS_MODULE: &str = "sqlbuild.cursor_algebra.models"; +pub(crate) const MODELS_MODULE: &str = "sqlbuild._native"; const EXCEPTIONS_MODULE: &str = "sqlbuild.cursor_algebra.exceptions"; const PLANNER_TYPES_MODULE: &str = "sqlbuild.compiler.planner.types"; const GRAIN_NAMES: [(&str, Grain); 6] = [ diff --git a/crates/sqlbuild-python/src/bindings/_helpers/cursor/durations.rs b/crates/sqlbuild-python/src/bindings/_helpers/cursor/durations.rs index 7bd4590a5c..1fbc70322f 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/cursor/durations.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/cursor/durations.rs @@ -19,7 +19,7 @@ const MONTHS_PER_YEAR: i64 = 12; /// `(years, months, days, hours, minutes, seconds)` of a duration like `1y6mo` or `30d`. #[pyfunction] #[pyo3(signature = (value))] -fn parse_cursor_duration<'py>( +pub(crate) fn parse_cursor_duration<'py>( py: Python<'py>, value: &str, ) -> PyResult>> { @@ -86,7 +86,7 @@ pub(crate) fn shifted_moment<'py>( /// `moment` moved forward (`direction` 1) or back (`direction` -1) by `duration`. #[pyfunction] #[pyo3(signature = (*, moment, duration, direction))] -fn shift_by_cursor_duration<'py>( +pub(crate) fn shift_by_cursor_duration<'py>( moment: &Bound<'py, PyAny>, duration: &Bound<'py, PyAny>, direction: i64, diff --git a/crates/sqlbuild-python/src/bindings/_helpers/cursor/intervals.rs b/crates/sqlbuild-python/src/bindings/_helpers/cursor/intervals.rs index 12aa7365ea..f522059e2b 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/cursor/intervals.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/cursor/intervals.rs @@ -46,7 +46,7 @@ fn python_intervals<'py>( /// Raise `CursorAlgebraError` unless the ends form a valid aligned interval. #[pyfunction(name = "validate_cursor_interval")] #[pyo3(signature = (*, start, end, grain))] -fn interval_validation( +pub(crate) fn interval_validation( start: &Bound<'_, PyAny>, end: &Bound<'_, PyAny>, grain: &Bound<'_, PyAny>, diff --git a/crates/sqlbuild-python/src/bindings/_helpers/dbt_interop/pipeline_phases.rs b/crates/sqlbuild-python/src/bindings/_helpers/dbt_interop/pipeline_phases.rs index e2c81bcc2a..7a5d4ff0ac 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/dbt_interop/pipeline_phases.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/dbt_interop/pipeline_phases.rs @@ -232,12 +232,7 @@ fn execution_limits<'py>( ) -> PyResult> { let py = discovered.py(); if target.is_none() { - return imported( - py, - "sqlbuild.spec.contracts.models", - "ExecutionLimitsConfig", - )? - .call0(); + return imported(py, "sqlbuild._native", "ExecutionLimitsConfig")?.call0(); } call_with( &imported(py, "sqlbuild._native", "resolve_target_config")?, diff --git a/crates/sqlbuild-python/src/bindings/_helpers/dbt_interop/profile_load.rs b/crates/sqlbuild-python/src/bindings/_helpers/dbt_interop/profile_load.rs index dbc4ebfe5e..83a7dc39ea 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/dbt_interop/profile_load.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/dbt_interop/profile_load.rs @@ -15,7 +15,7 @@ use crate::bindings::_helpers::dbt_interop::profile::raised; use crate::bindings::_helpers::dbt_interop::profile_connection::profile_error; const PROFILE_LOAD_MODULE: &str = "sqlbuild.integrations.dbt._helpers.profile.load"; -const DBT_MODELS: &str = "sqlbuild.integrations.dbt.models"; +const DBT_MODELS: &str = "sqlbuild._native"; const DBT_PROJECT_FILE: &str = "dbt_project.yml"; const PROFILES_FILE: &str = "profiles.yml"; const DEFAULT_PROFILES_DIR: &str = "~/.dbt"; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/debug_command/run_debug.rs b/crates/sqlbuild-python/src/bindings/_helpers/debug_command/run_debug.rs index 76a82c12f0..de3f8f48f1 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/debug_command/run_debug.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/debug_command/run_debug.rs @@ -37,7 +37,7 @@ use crate::bindings::_helpers::presentation_text::terminal::stdout_supports_colo const NATIVE_MODULE: &str = "sqlbuild._native"; const ADAPTER_NAME_MODULE: &str = "sqlbuild._native"; -const CONTRACT_TYPES_MODULE: &str = "sqlbuild.spec.contracts.types"; +const CONTRACT_TYPES_MODULE: &str = "sqlbuild._native"; const DEBUG_OPTIONS: (&[&str], &[&str]) = ( &[], &[ diff --git a/crates/sqlbuild-python/src/bindings/_helpers/diff_report/query.rs b/crates/sqlbuild-python/src/bindings/_helpers/diff_report/query.rs index df0ee54d16..3fed5deff2 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/diff_report/query.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/diff_report/query.rs @@ -31,7 +31,7 @@ const GENERIC_DIALECT: &str = "generic"; const COMMAND_MODELS_MODULE: &str = "sqlbuild._native"; const LIFECYCLE_MODULE: &str = "sqlbuild.executor.diff.classes.query_artifact_lifecycle"; const RELATION_LOOKUP_MODULE: &str = "sqlbuild.adapter.relations.main.relation_lookup"; -const CURSOR_MODELS_MODULE: &str = "sqlbuild.cursor_algebra.models"; +const CURSOR_MODELS_MODULE: &str = "sqlbuild._native"; const EXECUTE_QUERY_MODULE: &str = "sqlbuild.executor.diff.main._execute_query"; const DIFF_MODELS_MODULE: &str = "sqlbuild.executor.diff.models"; const NAMESPACELESS_ADAPTERS: [&str; 2] = ["duckdb", "motherduck"]; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/discovery/declaration_records.rs b/crates/sqlbuild-python/src/bindings/_helpers/discovery/declaration_records.rs index 126c18bc21..75f730b91e 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/discovery/declaration_records.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/discovery/declaration_records.rs @@ -35,9 +35,9 @@ const NATIVE_RECORDS_MODULE: &str = "sqlbuild._native"; const MODELS_MODULE: &str = "sqlbuild.compiler.discovery.models"; const SCOPE_TYPES_MODULE: &str = "sqlbuild._native"; -const AUDITING_MODELS_MODULE: &str = "sqlbuild.compiler.auditing.models"; +const AUDITING_MODELS_MODULE: &str = "sqlbuild._native"; const AUDITING_TYPES_MODULE: &str = "sqlbuild.compiler.auditing.types"; -const CONTRACT_MODELS_MODULE: &str = "sqlbuild.spec.contracts.models"; +const CONTRACT_MODELS_MODULE: &str = "sqlbuild._native"; const PATHLIB_MODULE: &str = "pathlib"; const SESSION_MEMO_KEY: &str = "native_discovery_session"; const MEASUREMENT_MODE: &str = "MEASUREMENT"; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/discovery/description_inputs.rs b/crates/sqlbuild-python/src/bindings/_helpers/discovery/description_inputs.rs index 300f6fc990..70c35c02c4 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/discovery/description_inputs.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/discovery/description_inputs.rs @@ -8,7 +8,7 @@ use crate::bindings::_helpers::compile_driver::python_calls::{call_with, importe const NATIVE_MODULE: &str = "sqlbuild._native"; const DISCOVERY_MODELS_MODULE: &str = "sqlbuild.compiler.discovery.models"; -const CONTRACT_MODELS_MODULE: &str = "sqlbuild.spec.contracts.models"; +const CONTRACT_MODELS_MODULE: &str = "sqlbuild._native"; /// Models and model schemas, skipping files that cannot be read. #[pyfunction] diff --git a/crates/sqlbuild-python/src/bindings/_helpers/discovery/header_values.rs b/crates/sqlbuild-python/src/bindings/_helpers/discovery/header_values.rs index a5d2082a40..f0d83ff160 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/discovery/header_values.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/discovery/header_values.rs @@ -9,7 +9,7 @@ use crate::bindings::_helpers::boundary::keyword_request::Request; const NATIVE_MODULE: &str = "sqlbuild._native"; const DISCOVERY_ERRORS_MODULE: &str = "sqlbuild._native"; -const CONTRACT_MODELS_MODULE: &str = "sqlbuild.spec.contracts.models"; +const CONTRACT_MODELS_MODULE: &str = "sqlbuild._native"; const HEADER_CACHE_SIZE: usize = 4096; /// `(name, line, column, end line, end column)`. diff --git a/crates/sqlbuild-python/src/bindings/_helpers/discovery/model_file_record.rs b/crates/sqlbuild-python/src/bindings/_helpers/discovery/model_file_record.rs index d9177933af..f948566f07 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/discovery/model_file_record.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/discovery/model_file_record.rs @@ -7,7 +7,7 @@ use pyo3::types::{PyDict, PyDictMethods, PyString, PyTuple}; use pyo3::{FromPyObject, IntoPyObjectExt, pyclass, pymethods}; use sqlbuild_discovery::models::LineColumnSpan; -const SOURCE_LOCATION_MODULE: &str = "sqlbuild.spec.contracts.models"; +const SOURCE_LOCATION_MODULE: &str = "sqlbuild._native"; const SOURCE_LOCATION_CLASS: &str = "SourceLocation"; const FIELD_NAMES: [&str; 10] = [ "file_path", diff --git a/crates/sqlbuild-python/src/bindings/_helpers/discovery/scope_snapshot.rs b/crates/sqlbuild-python/src/bindings/_helpers/discovery/scope_snapshot.rs index 4be4207f44..b02c05c0be 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/discovery/scope_snapshot.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/discovery/scope_snapshot.rs @@ -27,7 +27,7 @@ use crate::bindings::_helpers::discovery::yaml_records::discover_source_files; const NATIVE_RECORDS_MODULE: &str = "sqlbuild._native"; const MODELS_MODULE: &str = "sqlbuild.compiler.discovery.models"; -const CONTRACT_MODELS_MODULE: &str = "sqlbuild.spec.contracts.models"; +const CONTRACT_MODELS_MODULE: &str = "sqlbuild._native"; const PROJECT_CONFIG_FILENAME: &str = "sqlbuild_project.toml"; const LOCAL_CONFIG_FILENAME: &str = "sqlbuild_local.toml"; const FALLBACK_PROJECT_NAME: &str = "project"; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/discovery/sql_test_blocks.rs b/crates/sqlbuild-python/src/bindings/_helpers/discovery/sql_test_blocks.rs index 2d4011a816..1075c3d825 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/discovery/sql_test_blocks.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/discovery/sql_test_blocks.rs @@ -16,7 +16,7 @@ const NATIVE_RECORDS_MODULE: &str = "sqlbuild._native"; const MODELS_MODULE: &str = "sqlbuild.compiler.discovery.models"; const ERRORS_MODULE: &str = "sqlbuild._native"; const TYPES_MODULE: &str = "sqlbuild.compiler.compile.types"; -const SQL_VALUE_TYPES_MODULE: &str = "sqlbuild.sql_values.types"; +const SQL_VALUE_TYPES_MODULE: &str = "sqlbuild._native"; pub(crate) type NativeBlock<'py> = (Bound<'py, PyDict>, String, Bound<'py, PyAny>); diff --git a/crates/sqlbuild-python/src/bindings/_helpers/fingerprints/latest_rows.rs b/crates/sqlbuild-python/src/bindings/_helpers/fingerprints/latest_rows.rs index 232b5681e6..b764698f81 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/fingerprints/latest_rows.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/fingerprints/latest_rows.rs @@ -14,7 +14,7 @@ use sqlbuild_analysis::fingerprint_storage::main::latest_index::latest_index; use sqlbuild_analysis::fingerprint_storage::main::selected_positions::selected_positions; use sqlbuild_analysis::fingerprint_storage::models::{FingerprintIdentity, LatestIndex}; -const MODELS_MODULE: &str = "sqlbuild.compiler.fingerprints.models"; +const MODELS_MODULE: &str = "sqlbuild._native"; const EXCEPTIONS_MODULE: &str = "sqlbuild._native"; const DATETIME_MODULE: &str = "datetime"; const BINASCII_MODULE: &str = "binascii"; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/fingerprints/storage_io.rs b/crates/sqlbuild-python/src/bindings/_helpers/fingerprints/storage_io.rs index 89b33f008c..05ce3603dc 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/fingerprints/storage_io.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/fingerprints/storage_io.rs @@ -21,7 +21,7 @@ use crate::bindings::_helpers::fingerprints::storage_sql::{ }; const FINGERPRINT_TABLE_NAME: &str = "_sqlbuild_fingerprints"; -const MODELS_MODULE: &str = "sqlbuild.compiler.fingerprints.models"; +const MODELS_MODULE: &str = "sqlbuild._native"; const ADAPTER_TYPES_MODULE: &str = "sqlbuild.adapter.contract.types"; const TIME_MODULE: &str = "time"; const LOGGING_MODULE: &str = "logging"; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/freshness/observation.rs b/crates/sqlbuild-python/src/bindings/_helpers/freshness/observation.rs index 458b0a11b9..531c0b7a83 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/freshness/observation.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/freshness/observation.rs @@ -591,7 +591,7 @@ fn source_freshness_record_from_observation<'py>( /// An aware freshness `datetime` in UTC; a naive value is an observation error. #[pyfunction] #[pyo3(signature = (*, value, field_name))] -fn require_aware_utc_datetime<'py>( +pub(crate) fn require_aware_utc_datetime<'py>( value: &Bound<'py, PyAny>, field_name: &str, ) -> PyResult> { diff --git a/crates/sqlbuild-python/src/bindings/_helpers/freshness/support.rs b/crates/sqlbuild-python/src/bindings/_helpers/freshness/support.rs index 2ce560bc3a..2a52df1756 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/freshness/support.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/freshness/support.rs @@ -9,10 +9,10 @@ use crate::bindings::_helpers::cursor::values::DATETIME_MODULE; use crate::bindings::_helpers::state_rows::timestamps::{date_time_typed, in_utc, offset_aware}; pub(crate) const EXCEPTIONS_MODULE: &str = "sqlbuild._native"; -pub(crate) const FRESHNESS_MODELS: &str = "sqlbuild.compiler.source_freshness.models"; +pub(crate) const FRESHNESS_MODELS: &str = "sqlbuild._native"; pub(crate) const FRESHNESS_CONSTANTS: &str = "sqlbuild.compiler.source_freshness.constants"; -pub(crate) const CONTRACT_MODELS: &str = "sqlbuild.spec.contracts.models"; -pub(crate) const CONTRACT_TYPES: &str = "sqlbuild.spec.contracts.types"; +pub(crate) const CONTRACT_MODELS: &str = "sqlbuild._native"; +pub(crate) const CONTRACT_TYPES: &str = "sqlbuild._native"; pub(crate) const ADAPTER_MODELS: &str = "sqlbuild.adapter.contract.models"; pub(crate) const ADAPTER_TYPES: &str = "sqlbuild.adapter.contract.types"; pub(crate) const ADAPTER_EXCEPTIONS: &str = "sqlbuild.adapter.contract.exceptions"; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/graph/python_lifecycle.rs b/crates/sqlbuild-python/src/bindings/_helpers/graph/python_lifecycle.rs index f6b6143dff..1acdd6cbbc 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/graph/python_lifecycle.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/graph/python_lifecycle.rs @@ -8,7 +8,7 @@ use pyo3::{pyfunction, wrap_pyfunction}; use sqlbuild_analysis::python_plan::main::run_lifecycle::run_lifecycle; use sqlbuild_analysis::python_plan::models::{LifecycleNode, RunLifecycle}; -const PYTHON_NODE_MODELS_MODULE: &str = "sqlbuild.compiler.python_nodes.models"; +const PYTHON_NODE_MODELS_MODULE: &str = "sqlbuild._native"; const EXTERNAL_CONNECTION_MODE: &str = "external"; /// `build_python_sql_run_lifecycle(*, selection, python_graph)`: a `PythonSqlRunLifecyclePlan`. diff --git a/crates/sqlbuild-python/src/bindings/_helpers/graph/python_node_graph.rs b/crates/sqlbuild-python/src/bindings/_helpers/graph/python_node_graph.rs index 480c8e128c..c5cf6c65d9 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/graph/python_node_graph.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/graph/python_node_graph.rs @@ -16,7 +16,7 @@ use crate::bindings::_helpers::graph::views::compiled_graph_keys; const NATIVE_RECORDS_MODULE: &str = "sqlbuild._native"; const USER_INVENTORY_MODULE: &str = "sqlbuild.compiler.python_nodes._helpers.user_inventory"; -const NODE_MODELS_MODULE: &str = "sqlbuild.compiler.python_nodes.models"; +const NODE_MODELS_MODULE: &str = "sqlbuild._native"; const NODE_TYPES_MODULE: &str = "sqlbuild.compiler.python_nodes.types"; const SQL_REF_MODULE: &str = "sqlbuild.python_nodes.models"; const NAME_KEY: &str = "name"; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/hand_built/compiled_model.rs b/crates/sqlbuild-python/src/bindings/_helpers/hand_built/compiled_model.rs index 159c6369c0..c20ac302ce 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/hand_built/compiled_model.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/hand_built/compiled_model.rs @@ -8,7 +8,7 @@ const NATIVE_MODULE: &str = "sqlbuild._native"; const COMPILE_MODELS_MODULE: &str = "sqlbuild.compiler.compile.models"; const COMPILE_TYPES_MODULE: &str = "sqlbuild.compiler.compile.types"; const LINEAGE_TYPES_MODULE: &str = "sqlbuild.compiler.lineage.types"; -const CONTRACT_MODELS_MODULE: &str = "sqlbuild.spec.contracts.models"; +const CONTRACT_MODELS_MODULE: &str = "sqlbuild._native"; type Object<'py> = Bound<'py, PyAny>; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/load/load_selection.rs b/crates/sqlbuild-python/src/bindings/_helpers/load/load_selection.rs index 70299b2a80..9f46362752 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/load/load_selection.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/load/load_selection.rs @@ -17,7 +17,7 @@ use crate::bindings::_helpers::compile_driver::python_calls::{ }; const ERRORS_MODULE: &str = "sqlbuild._native"; -const CONTRACT_MODELS: &str = "sqlbuild.spec.contracts.models"; +const CONTRACT_MODELS: &str = "sqlbuild._native"; const DESTINATION_MODULE: &str = "sqlbuild._native"; const CONNECTION_CONFIG_MODULE: &str = "sqlbuild._native"; const ADAPTER_NAME_MODULE: &str = "sqlbuild._native"; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/mod.rs b/crates/sqlbuild-python/src/bindings/_helpers/mod.rs index 962c1d5bad..d5320e8f37 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/mod.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/mod.rs @@ -23,6 +23,7 @@ pub(crate) mod compiler_errors; pub(crate) mod compiler_records; pub(crate) mod compute_logs; pub(crate) mod contract_command; +pub(crate) mod contract_records; pub(crate) mod contracts; pub(crate) mod cost_command; pub(crate) mod cost_report; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/model_config/contract_classes.rs b/crates/sqlbuild-python/src/bindings/_helpers/model_config/contract_classes.rs index 1cca55b389..67869c37d5 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/model_config/contract_classes.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/model_config/contract_classes.rs @@ -6,10 +6,10 @@ use pyo3::types::{PyDict, PyDictMethods}; use crate::bindings::_helpers::compile_driver::python_calls::imported; -const AUDITING_MODELS_MODULE: &str = "sqlbuild.compiler.auditing.models"; +const AUDITING_MODELS_MODULE: &str = "sqlbuild._native"; const AUDITING_TYPES_MODULE: &str = "sqlbuild.compiler.auditing.types"; -const CONTRACT_MODELS_MODULE: &str = "sqlbuild.spec.contracts.models"; -const CONTRACT_TYPES_MODULE: &str = "sqlbuild.spec.contracts.types"; +const CONTRACT_MODELS_MODULE: &str = "sqlbuild._native"; +const CONTRACT_TYPES_MODULE: &str = "sqlbuild._native"; const DLT_MODELS_MODULE: &str = "sqlbuild.integrations.dlt.models"; const INGESTR_MODELS_MODULE: &str = "sqlbuild.integrations.ingestr.models"; const CONTRACT_CLASSES: [(&str, &str, &str); 6] = [ @@ -126,10 +126,6 @@ fn built_contract_classes(py: Python<'_>) -> PyResult> { "threshold_operators", members_by_value(&imported(py, AUDITING_TYPES_MODULE, "ThresholdOperator")?)?, )?; - classes.set_item( - "allocate", - imported(py, "builtins", "object")?.getattr("__new__")?, - )?; Ok(classes) } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/model_config/header_metadata.rs b/crates/sqlbuild-python/src/bindings/_helpers/model_config/header_metadata.rs index 56e7cb86ce..2d91542cb0 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/model_config/header_metadata.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/model_config/header_metadata.rs @@ -21,7 +21,6 @@ pub(crate) struct ContractClasses<'py> { measurement_threshold_bound: Bound<'py, PyAny>, severities: Bound<'py, PyDict>, threshold_operators: Bound<'py, PyDict>, - allocate: Bound<'py, PyAny>, } pub(crate) fn column_tuple<'py>( @@ -50,18 +49,16 @@ pub(crate) fn audit_tuple<'py>( PyTuple::new(py, audits) } -/// Build a frozen contract dataclass the way `copy` does: allocate, then fill its `__dict__`. +/// `class(**fields)`. fn contract_object<'py>( - allocate: &Bound<'py, PyAny>, class: &Bound<'py, PyAny>, fields: &[(&str, Bound<'py, PyAny>)], ) -> PyResult> { - let object = allocate.call1((class,))?; - let attributes = object.getattr("__dict__")?.downcast_into::()?; + let keywords: Bound<'py, PyDict> = PyDict::new(class.py()); for (name, value) in fields { - attributes.set_item(name, value)?; + keywords.set_item(name, value)?; } - Ok(object) + class.call((), Some(&keywords)) } fn optional<'py>(py: Python<'py>, value: Option<&PyNode<'py>>) -> Bound<'py, PyAny> { @@ -83,7 +80,6 @@ fn schema_column<'py>( .map(|audit| audit_instance(py, classes, audit, &location)) .collect::>>()?; contract_object( - &classes.allocate, &classes.schema_column, &[ ("name", column.name.0.clone()), @@ -120,7 +116,6 @@ fn audit_instance<'py>( |node| node.0.clone(), ); contract_object( - &classes.allocate, &classes.schema_audit_instance, &[ ("definition_name", audit.definition_name.0.clone()), @@ -157,7 +152,6 @@ pub(crate) fn thresholds<'py>( return Ok(py.None().into_bound(py)); }; contract_object( - &classes.allocate, &classes.measurement_thresholds, &[ ("warn", threshold_bound(py, classes, parsed.warn)?), @@ -184,7 +178,6 @@ fn threshold_bound<'py>( .get_item(operator)? .unwrap_or_else(|| py.None().into_bound(py)); contract_object( - &classes.allocate, &classes.measurement_threshold_bound, &[ ("operator", operator), diff --git a/crates/sqlbuild-python/src/bindings/_helpers/plan_output/cursor_details.rs b/crates/sqlbuild-python/src/bindings/_helpers/plan_output/cursor_details.rs index f34fba5aab..1f5672ff48 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/plan_output/cursor_details.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/plan_output/cursor_details.rs @@ -9,7 +9,7 @@ use crate::bindings::_helpers::plan_output::display::{ use crate::bindings::_helpers::plan_output::values::{optional_text, text}; const BOUND_TYPES_MODULE: &str = "sqlbuild.cursor_algebra.types"; -const CURSOR_MODELS_MODULE: &str = "sqlbuild.cursor_algebra.models"; +const CURSOR_MODELS_MODULE: &str = "sqlbuild._native"; const MICROBATCH_MODE: &str = "microbatch"; const TIMESTAMP_CURSOR: &str = "timestamp"; const INTEGER_CURSOR: &str = "integer"; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/plan_output/display.rs b/crates/sqlbuild-python/src/bindings/_helpers/plan_output/display.rs index c5da29096c..51917f4b7d 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/plan_output/display.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/plan_output/display.rs @@ -26,7 +26,7 @@ use crate::bindings::_helpers::plan_output::values::{optional_text, text}; const PLANNER_TYPES: &str = "sqlbuild.compiler.planner.types"; const PLANNER_MODELS: &str = "sqlbuild.compiler.planner.models"; -const CURSOR_MODELS: &str = "sqlbuild.cursor_algebra.models"; +const CURSOR_MODELS: &str = "sqlbuild._native"; const DATETIME_MODULE: &str = "datetime"; fn label_facts(entry: &Bound<'_, PyAny>) -> PyResult { diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/model_planning.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/model_planning.rs index 5b39319d6f..99344dbef2 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/model_planning.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/model_planning.rs @@ -254,7 +254,7 @@ fn finish<'py>( )?; let project_config = runtime.getattr("project_config")?; let snapshots_config = if project_config.is_none() { - imported(py, "sqlbuild.spec.contracts.models", "SnapshotsConfig")?.call0()? + imported(py, "sqlbuild._native", "SnapshotsConfig")?.call0()? } else { project_config.getattr("snapshots")? }; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/old_name_entries.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/old_name_entries.rs index e45c22183f..42506b327f 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/old_name_entries.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/old_name_entries.rs @@ -23,7 +23,7 @@ impl<'py> OldNamePlanner<'py> { let py = runtime.py(); let project_config = runtime.getattr("project_config")?; let config = if project_config.is_none() { - imported(py, "sqlbuild.spec.contracts.models", "MigrationsConfig")?.call0()? + imported(py, "sqlbuild._native", "MigrationsConfig")?.call0()? } else { project_config.getattr("migrations")? }; @@ -282,7 +282,7 @@ impl<'py> OldNamePlanner<'py> { } } let retention = history.getattr("required")?.getattr("view_retention")?; - let duration = imported(py, "sqlbuild.cursor_algebra.models", "Duration")?.call_method1( + let duration = imported(py, "sqlbuild._native", "Duration")?.call_method1( "parse", (if retention.is_truthy()? { retention.clone() diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/old_name_objects.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/old_name_objects.rs index 9c8a9026c4..2ac012fb39 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/old_name_objects.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/old_name_objects.rs @@ -56,8 +56,8 @@ pub(super) fn expires<'py>( retention: &Bound<'py, PyAny>, now: &Bound<'py, PyAny>, ) -> PyResult> { - let duration = imported(now.py(), "sqlbuild.cursor_algebra.models", "Duration")? - .call_method1("parse", (retention,))?; + let duration = + imported(now.py(), "sqlbuild._native", "Duration")?.call_method1("parse", (retention,))?; if duration.is_none() { Ok(now.py().None().into_bound(now.py())) } else { diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_models/records.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_models/records.rs index bcba81d3e3..5885e93320 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_models/records.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_models/records.rs @@ -424,7 +424,7 @@ planner_record! { (invocation_time = "invocation_time", FieldDefault::NoneValue, true, true), (max_microbatches = "max_microbatches", FieldDefault::NoneValue, true, true), (max_microbatches_is_override = "max_microbatches_is_override", FieldDefault::Bool(false), true, true), - (microbatch_limit_action = "microbatch_limit_action", FieldDefault::Member(("sqlbuild.spec.contracts.types", "MicrobatchLimitAction"), "ERROR"), true, true), + (microbatch_limit_action = "microbatch_limit_action", FieldDefault::Member(("sqlbuild._native", "MicrobatchLimitAction"), "ERROR"), true, true), ] } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_phases/constants.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_phases/constants.rs index f80cc80940..ef2827ee14 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_phases/constants.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_phases/constants.rs @@ -6,7 +6,7 @@ pub(crate) const PLANNER_TYPES: &str = "sqlbuild.compiler.planner.types"; pub(crate) const PLANNER_EXCEPTIONS: &str = "sqlbuild.compiler.planner.exceptions"; pub(crate) const ADAPTER_MODELS: &str = "sqlbuild.adapter.contract.models"; pub(crate) const ADAPTER_TYPES: &str = "sqlbuild.adapter.contract.types"; -pub(crate) const SPEC_MODELS: &str = "sqlbuild.spec.contracts.models"; +pub(crate) const SPEC_MODELS: &str = "sqlbuild._native"; pub(crate) const SNOWFLAKE_ADAPTER: &str = "snowflake"; pub(crate) const BIGQUERY_ADAPTER: &str = "bigquery"; pub(crate) const TRANSIENT_TABLE_TYPE: &str = "transient"; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_records/constants.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_records/constants.rs index c2f70f51ba..76e2105e04 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_records/constants.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_records/constants.rs @@ -134,6 +134,92 @@ const R_DIRECT_SOURCE_FRESHNESS_PLANNING_RESULT: RecordSpec = RecordSpec { FieldSpec::new("propagation", FieldDefault::None), ], }; +const R_FINGERPRINT: RecordSpec = RecordSpec { + name: "Fingerprint", + doc: "One applied fingerprint record from a successful materialization.", + fields: &[ + FieldSpec::required("node_type"), + FieldSpec::required("node_name"), + FieldSpec::required("target_database"), + FieldSpec::required("target_schema"), + FieldSpec::required("target_name"), + FieldSpec::required("run_id"), + FieldSpec::required("definition_hash"), + FieldSpec::required("schema_fingerprint"), + FieldSpec::required("definition"), + FieldSpec::required("ts"), + FieldSpec::new("metadata_json", FieldDefault::Text("{}")), + FieldSpec::new("version_hash", FieldDefault::Text("")), + ], +}; +const R_FINGERPRINT_SET: RecordSpec = RecordSpec { + name: "FingerprintSet", + doc: "Latest fingerprints per node name for one target schema.", + fields: &[ + FieldSpec::required("schema"), + FieldSpec::required("fingerprints"), + FieldSpec::new("fingerprints_by_identity", FieldDefault::None), + ], +}; +const R_SOURCE_FRESHNESS_RECORD: RecordSpec = RecordSpec { + name: "SourceFreshnessRecord", + doc: "One append-only direct observation whose observed time is always aware UTC.", + fields: &[ + FieldSpec::required("source_name"), + FieldSpec::required("target_database"), + FieldSpec::required("target_schema"), + FieldSpec::required("target_name"), + FieldSpec::required("run_id"), + FieldSpec::required("strategy"), + FieldSpec::required("value_kind"), + FieldSpec::required("data_version"), + FieldSpec::required("data_version_hash"), + FieldSpec::required("observed_at"), + ], +}; +const R_TIMESTAMP_VALUE: RecordSpec = RecordSpec { + name: "TimestampValue", + doc: "Timestamp cursor value; aware datetimes are normalized to UTC at ingestion.", + fields: &[ + FieldSpec::required("value"), + FieldSpec { + name: "source_text", + default: FieldDefault::None, + compared: false, + }, + ], +}; +const R_DATE_VALUE: RecordSpec = RecordSpec { + name: "DateValue", + doc: "Date cursor value which must remain date-formatted when rendered.", + fields: &[FieldSpec::required("value")], +}; +const R_INTEGER_VALUE: RecordSpec = RecordSpec { + name: "IntegerValue", + doc: "Integral cursor value.", + fields: &[FieldSpec::required("value")], +}; +const R_ALIGNED_INTERVAL: RecordSpec = RecordSpec { + name: "AlignedInterval", + doc: "Validated half-open cursor interval.", + fields: &[ + FieldSpec::required("start"), + FieldSpec::required("end"), + FieldSpec::required("grain"), + ], +}; +const R_DURATION: RecordSpec = RecordSpec { + name: "Duration", + doc: "A calendar-aware duration (years/months plus fixed time units).", + fields: &[ + FieldSpec::new("years", FieldDefault::Int(0)), + FieldSpec::new("months", FieldDefault::Int(0)), + FieldSpec::new("days", FieldDefault::Int(0)), + FieldSpec::new("hours", FieldDefault::Int(0)), + FieldSpec::new("minutes", FieldDefault::Int(0)), + FieldSpec::new("seconds", FieldDefault::Int(0)), + ], +}; /// The planner, migration, cursor and fingerprint value records. pub(crate) const PLANNER_RECORDS: &[RecordSpec] = &[ @@ -148,4 +234,12 @@ pub(crate) const PLANNER_RECORDS: &[RecordSpec] = &[ R_DEFERRAL_INPUTS, R_PLANNER_POLICIES, R_DIRECT_SOURCE_FRESHNESS_PLANNING_RESULT, + R_FINGERPRINT, + R_FINGERPRINT_SET, + R_SOURCE_FRESHNESS_RECORD, + R_TIMESTAMP_VALUE, + R_DATE_VALUE, + R_INTEGER_VALUE, + R_ALIGNED_INTERVAL, + R_DURATION, ]; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_resolution/cursor_policies.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_resolution/cursor_policies.rs index f0bd2a13dc..fd73879b06 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_resolution/cursor_policies.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_resolution/cursor_policies.rs @@ -5,8 +5,8 @@ use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyModule, PyModuleMethods, PyRes use pyo3::types::PyString; use pyo3::{pyfunction, wrap_pyfunction}; -const SPEC_MODELS: &str = "sqlbuild.spec.contracts.models"; -const SPEC_TYPES: &str = "sqlbuild.spec.contracts.types"; +const SPEC_MODELS: &str = "sqlbuild._native"; +const SPEC_TYPES: &str = "sqlbuild._native"; const POLICY_DISABLED: &str = "disabled"; /// The policy class, its distance field, and the model keys overriding distance and action. diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_resolution/cursor_ranges.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_resolution/cursor_ranges.rs index dcf00e724a..a62f123cf0 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_resolution/cursor_ranges.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_resolution/cursor_ranges.rs @@ -24,7 +24,7 @@ use std::cmp::Ordering; const MODELS: &str = "sqlbuild.compiler.planner.models"; const CURSOR_TYPES: &str = "sqlbuild.cursor_algebra.types"; -const CURSOR_MODELS: &str = "sqlbuild.cursor_algebra.models"; +const CURSOR_MODELS: &str = "sqlbuild._native"; const COMPILE_EXCEPTIONS: &str = "sqlbuild.compiler.compile.exceptions"; const INCREMENTAL: &str = "incremental"; const MICROBATCH: &str = "microbatch"; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_resolution/cursor_safety.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_resolution/cursor_safety.rs index e26c003d12..3e72238c4b 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_resolution/cursor_safety.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_resolution/cursor_safety.rs @@ -44,15 +44,10 @@ pub(super) fn safety_duration(value: &Bound<'_, PyAny>, future: bool) -> PyResul pub(super) fn timestamp_bound(value: &Bound<'_, PyAny>) -> PyResult { let py = value.py(); - Ok(value.is_instance(&imported( - py, - "sqlbuild.cursor_algebra.models", - "DateValue", - )?)? || value.is_instance(&imported( - py, - "sqlbuild.cursor_algebra.models", - "TimestampValue", - )?)?) + Ok( + value.is_instance(&imported(py, "sqlbuild._native", "DateValue")?)? + || value.is_instance(&imported(py, "sqlbuild._native", "TimestampValue")?)?, + ) } pub(super) fn utc_clock<'py>(clock: &Bound<'py, PyAny>) -> PyResult> { @@ -68,11 +63,7 @@ pub(super) fn horizon<'py>( let py = session.py(); let (clock, grain) = clock_and_grain; let mut clock = utc_clock(clock)?; - let date_only = discovered.is_instance(&imported( - py, - "sqlbuild.cursor_algebra.models", - "DateValue", - )?)?; + let date_only = discovered.is_instance(&imported(py, "sqlbuild._native", "DateValue")?)?; if date_only || discovered.getattr("value")?.getattr("tzinfo")?.is_none() { clock = call_with( &clock.getattr("replace")?, @@ -80,7 +71,7 @@ pub(super) fn horizon<'py>( )?; } let clock = call_with( - &imported(py, "sqlbuild.cursor_algebra.models", "TimestampValue")?, + &imported(py, "sqlbuild._native", "TimestampValue")?, &[("value", clock)], )?; let grain = if grain.is_truthy()? { diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_resolution/cursor_windows.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_resolution/cursor_windows.rs index 4c268b6fb0..c3342d244e 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_resolution/cursor_windows.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_resolution/cursor_windows.rs @@ -94,7 +94,7 @@ fn empty_cursor_window<'py>( let mut bound = parse_value(&raw, &kind)?; if cursor_start.is_none() && !kind.eq("integer")? { bound = call_with( - &imported(py, "sqlbuild.cursor_algebra.models", "TimestampValue")?, + &imported(py, "sqlbuild._native", "TimestampValue")?, &[("value", bound.getattr("value")?)], )?; } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_resolution/future_safety.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_resolution/future_safety.rs index 0163650f9e..099ce98ded 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_resolution/future_safety.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_resolution/future_safety.rs @@ -127,7 +127,7 @@ fn resolve_future_cursor_safety<'py>( ( "invocation_time", call_with( - &imported(py, "sqlbuild.cursor_algebra.models", "TimestampValue")?, + &imported(py, "sqlbuild._native", "TimestampValue")?, &[("value", utc_clock(&clock)?)], )?, ), diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_resolution/maximum_safety.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_resolution/maximum_safety.rs index a14a30a3ea..6d078b4098 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_resolution/maximum_safety.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_resolution/maximum_safety.rs @@ -188,7 +188,7 @@ fn apply_maximum_start_policy<'py>( ( "invocation_time", call_with( - &imported(py, "sqlbuild.cursor_algebra.models", "TimestampValue")?, + &imported(py, "sqlbuild._native", "TimestampValue")?, &[("value", utc_clock(&clock)?)], )?, ), diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_scope/values.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_scope/values.rs index a9fd5203d5..e675a9ce32 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_scope/values.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_scope/values.rs @@ -9,7 +9,7 @@ use sqlbuild_analysis::planner_scope::models::LoaderGraph; pub(crate) const NATIVE: &str = "sqlbuild._native"; pub(crate) const MODELS: &str = "sqlbuild.compiler.planner.models"; -pub(crate) const SOURCE_MODELS: &str = "sqlbuild.spec.contracts.models"; +pub(crate) const SOURCE_MODELS: &str = "sqlbuild._native"; pub(crate) fn keys(values: &Bound<'_, PyAny>) -> PyResult> { values diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_strategy/microbatches.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_strategy/microbatches.rs index ea42c15438..596c9526f3 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_strategy/microbatches.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_strategy/microbatches.rs @@ -24,8 +24,8 @@ use crate::bindings::_helpers::plan_output::display::aligned_timestamp_bounds; use crate::bindings::_helpers::planner_changes::results::MODELS_MODULE; use crate::bindings::_helpers::planner_strategy::values::text; -const CURSOR_MODELS_MODULE: &str = "sqlbuild.cursor_algebra.models"; -pub(crate) const CONTRACT_TYPES_MODULE: &str = "sqlbuild.spec.contracts.types"; +const CURSOR_MODELS_MODULE: &str = "sqlbuild._native"; +pub(crate) const CONTRACT_TYPES_MODULE: &str = "sqlbuild._native"; const INTEGER_CURSOR: &str = "integer"; const TIMESTAMP_CURSOR: &str = "timestamp"; const MONTHS_PER_YEAR: i64 = 12; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_warehouse/constants.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_warehouse/constants.rs index 09151463ce..f4394e7f9b 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_warehouse/constants.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_warehouse/constants.rs @@ -46,7 +46,7 @@ pub(crate) const SQL_HOOK_ENTRY: (&str, &str) = ("sqlbuild.compiler.discovery.models", "SqlHookEntry"); pub(crate) const CURSOR_ALGEBRA_ERROR: (&str, &str) = ("sqlbuild.cursor_algebra.exceptions", "CursorAlgebraError"); -pub(crate) const DATE_VALUE: (&str, &str) = ("sqlbuild.cursor_algebra.models", "DateValue"); +pub(crate) const DATE_VALUE: (&str, &str) = ("sqlbuild._native", "DateValue"); pub(crate) const PLANNER_LOGGER: &str = "sqlbuild.planner"; pub(crate) const FINGERPRINT_TABLE_NAME: &str = "_sqlbuild_fingerprints"; pub(crate) const SOURCE_FRESHNESS_TABLE_NAME: &str = "_sqlbuild_source_freshness"; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/record_behaviours/auditing.rs b/crates/sqlbuild-python/src/bindings/_helpers/record_behaviours/auditing.rs new file mode 100644 index 0000000000..737a0b63b1 --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/record_behaviours/auditing.rs @@ -0,0 +1,106 @@ +//! Validation of the natively hosted measurement audit records. + +use pyo3::prelude::{Bound, Py, PyAny, PyAnyMethods, PyErr, PyResult, Python}; +use pyo3::types::{PyBool, PyFloat}; + +use sqlbuild_model_config::header_metadata::main::check_measurement_contract::check_measurement_contract; +use sqlbuild_model_config::header_metadata::main::check_threshold_bound::check_threshold_bound; +use sqlbuild_model_config::header_metadata::main::check_threshold_pair::check_threshold_pair; +use sqlbuild_model_config::header_metadata::models::{ThresholdBound, ThresholdInput}; + +use crate::bindings::_helpers::compile_driver::python_calls::imported; +use crate::bindings::_helpers::records::constants::NATIVE_MODULE; + +const MEASUREMENT_AUDIT_ERROR: &str = "MeasurementAuditError"; + +/// The `MeasurementAuditError` carrying `message`. +fn measurement_error(py: Python<'_>, message: &str) -> PyErr { + match imported(py, NATIVE_MODULE, MEASUREMENT_AUDIT_ERROR) + .and_then(|class| class.call1((message,))) + { + Ok(error) => PyErr::from_value(error), + Err(error) => error, + } +} + +/// How Python's threshold validation sees one limit value. +fn threshold_input(value: &Bound<'_, PyAny>) -> PyResult { + if value.is_none() { + return Ok(ThresholdInput::Absent); + } + if value.is_instance_of::() { + return Ok(ThresholdInput::Boolean); + } + Ok(ThresholdInput::Number(value.extract::()?)) +} + +/// `MeasurementThresholdBound`'s operator, limit, lower and upper, checked and made floats. +pub(crate) fn finish_threshold_bound( + py: Python<'_>, + values: Vec>, +) -> PyResult>> { + let operator: String = values[0].bind(py).str()?.extract()?; + let inputs = ( + threshold_input(values[1].bind(py))?, + threshold_input(values[2].bind(py))?, + threshold_input(values[3].bind(py))?, + ); + let float = |value: f64| PyFloat::new(py, value).into_any().unbind(); + let none = || py.None(); + let (limit, lower, upper) = match check_threshold_bound(&operator, inputs) { + Ok(ThresholdBound::Below(limit) | ThresholdBound::Above(limit)) => { + (float(limit), none(), none()) + } + Ok(ThresholdBound::Outside(lower, upper)) => (none(), float(lower), float(upper)), + Err(message) => return Err(measurement_error(py, &message)), + }; + let mut values: Vec> = values; + values[1] = limit; + values[2] = lower; + values[3] = upper; + Ok(values) +} + +/// The checked bound a `MeasurementThresholdBound` record holds, or `None`. +fn record_bound(bound: &Bound<'_, PyAny>) -> PyResult> { + if bound.is_none() { + return Ok(None); + } + let operator: String = bound.getattr("operator")?.str()?.extract()?; + let inputs = ( + threshold_input(&bound.getattr("limit")?)?, + threshold_input(&bound.getattr("lower")?)?, + threshold_input(&bound.getattr("upper")?)?, + ); + check_threshold_bound(&operator, inputs) + .map(Some) + .map_err(|message| measurement_error(bound.py(), &message)) +} + +/// `MeasurementThresholds`' warn and error bounds, checked as one policy. +pub(crate) fn finish_thresholds( + py: Python<'_>, + values: Vec>, +) -> PyResult>> { + let warn: Option = record_bound(values[0].bind(py))?; + let error: Option = record_bound(values[1].bind(py))?; + check_threshold_pair(warn, error).map_err(|message| measurement_error(py, message))?; + Ok(values) +} + +/// `MeasurementContract`'s named columns and unit, checked non-blank. +pub(crate) fn finish_measurement_contract( + py: Python<'_>, + values: Vec>, +) -> PyResult>> { + let value_column: String = values[0].bind(py).extract()?; + let sample_count_column: Option = values[1].bind(py).extract()?; + let sample_unit: Option = values[2].bind(py).extract()?; + check_measurement_contract( + &value_column, + sample_count_column.as_deref(), + sample_unit.as_deref(), + ) + .map_err(|message| measurement_error(py, message))?; + Ok(values) +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/record_behaviours/config.rs b/crates/sqlbuild-python/src/bindings/_helpers/record_behaviours/config.rs new file mode 100644 index 0000000000..5278ad1d62 --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/record_behaviours/config.rs @@ -0,0 +1,8 @@ +//! Computed attributes of the natively hosted project configuration records. + +use pyo3::prelude::{Bound, Py, PyAny, PyAnyMethods, PyResult}; + +/// `SettingsConfig.sql_validation`: the legacy name of the unified SQL analysis gate. +pub(crate) fn settings_sql_validation(instance: &Bound<'_, PyAny>) -> PyResult> { + Ok(instance.getattr("sql_analysis")?.unbind()) +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/record_behaviours/constants.rs b/crates/sqlbuild-python/src/bindings/_helpers/record_behaviours/constants.rs index f449aa3b85..fcc457bb4f 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/record_behaviours/constants.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/record_behaviours/constants.rs @@ -1,19 +1,80 @@ //! Native behaviour of the hosted shapes: computed attributes, methods, ordering and validation. +use crate::bindings::_helpers::record_behaviours::auditing::{ + finish_measurement_contract, finish_threshold_bound, finish_thresholds, +}; +use crate::bindings::_helpers::record_behaviours::config::settings_sql_validation; +use crate::bindings::_helpers::record_behaviours::durations::{ + duration_add_to, duration_fixed_seconds, duration_has_calendar_component, + duration_is_single_unit_in, duration_is_zero, duration_parse, duration_subtract_from, + duration_total_months, duration_units, finish_aligned_interval, +}; +use crate::bindings::_helpers::record_behaviours::freshness::{ + finish_freshness_record, freshness_record_identity, +}; +use crate::bindings::_helpers::record_behaviours::lexical::{ + finish_lexical_syntax, lexical_cache_key, lexical_native_mapping, lexical_reads_differently, +}; +use crate::bindings::_helpers::record_behaviours::python_nodes::graph_selected_sql_refs; use crate::bindings::_helpers::record_behaviours::scopes::{ completeness_complete, target_query_unknown, }; +use crate::bindings::_helpers::record_behaviours::values::{ + finish_fingerprint_set, logical_type_display_name, sql_value_kind, +}; use crate::bindings::_helpers::records::types::{ClassMethod, ClassProperty, RecordFinish}; /// Compiler records whose bound values are normalized or validated natively. -pub(crate) const BEHAVIOUR_FINISHERS: &[(&str, RecordFinish)] = &[]; +pub(crate) const BEHAVIOUR_FINISHERS: &[(&str, RecordFinish)] = &[ + ("MeasurementThresholdBound", finish_threshold_bound), + ("MeasurementThresholds", finish_thresholds), + ("MeasurementContract", finish_measurement_contract), + ("FingerprintSet", finish_fingerprint_set), + ("SourceFreshnessRecord", finish_freshness_record), + ("SqlLexicalSyntax", finish_lexical_syntax), + ("AlignedInterval", finish_aligned_interval), +]; /// Computed attributes of compiler shapes. pub(crate) const BEHAVIOUR_PROPERTIES: &[(&str, &str, ClassProperty)] = &[ ("ScopeCompleteness", "complete", completeness_complete), ("ScopeTargetQuery", "unknown", target_query_unknown), + ("SettingsConfig", "sql_validation", settings_sql_validation), + ("SqlLogicalType", "display_name", logical_type_display_name), + ("SqlValue", "kind", sql_value_kind), + ( + "SourceFreshnessRecord", + "identity", + freshness_record_identity, + ), + ("SqlLexicalSyntax", "cache_key", lexical_cache_key), + ("SqlLexicalSyntax", "native_mapping", lexical_native_mapping), + ("Duration", "is_zero", duration_is_zero), + ("Duration", "units", duration_units), + ( + "Duration", + "has_calendar_component", + duration_has_calendar_component, + ), + ("Duration", "fixed_seconds", duration_fixed_seconds), + ("Duration", "total_months", duration_total_months), ]; /// Methods of compiler shapes. -pub(crate) const BEHAVIOUR_METHODS: &[(&str, &str, ClassMethod)] = &[]; +pub(crate) const BEHAVIOUR_METHODS: &[(&str, &str, ClassMethod)] = &[ + ( + "SqlLexicalSyntax", + "reads_differently_from_generic", + lexical_reads_differently, + ), + ("Duration", "parse", duration_parse), + ( + "PythonNodeGraph", + "selected_sql_refs", + graph_selected_sql_refs, + ), + ("Duration", "is_single_unit_in", duration_is_single_unit_in), + ("Duration", "add_to", duration_add_to), + ("Duration", "subtract_from", duration_subtract_from), +]; /// Compiler records ordered by their compared fields. pub(crate) const BEHAVIOUR_ORDERED: &[&str] = &["ResourceIdentity", "DeclarationIdentity", "OwnershipRoot"]; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/record_behaviours/durations.rs b/crates/sqlbuild-python/src/bindings/_helpers/record_behaviours/durations.rs new file mode 100644 index 0000000000..c0aa28ca6d --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/record_behaviours/durations.rs @@ -0,0 +1,147 @@ +//! Calendar arithmetic and validation of the natively hosted cursor algebra records. + +use pyo3::IntoPyObjectExt; +use pyo3::prelude::{Bound, Py, PyAny, PyAnyMethods, PyResult, Python}; +use pyo3::types::{PyDict, PyDictMethods, PyFrozenSet, PyTuple, PyTupleMethods}; + +use crate::bindings::_helpers::cursor::durations::{ + parse_cursor_duration, shift_by_cursor_duration, +}; +use crate::bindings::_helpers::cursor::intervals::interval_validation; + +/// `(field, unit suffix)` of each duration part, in declaration order. +const DURATION_UNITS: [(&str, &str); 6] = [ + ("years", "y"), + ("months", "mo"), + ("days", "d"), + ("hours", "h"), + ("minutes", "m"), + ("seconds", "s"), +]; +const MONTHS_PER_YEAR: i64 = 12; +const SECONDS_PER_DAY: i64 = 86_400; +const SECONDS_PER_HOUR: i64 = 3_600; +const SECONDS_PER_MINUTE: i64 = 60; + +/// The one positional or keyword argument `name` of a native method call. +fn argument<'py>( + args: &Bound<'py, PyTuple>, + kwargs: Option<&Bound<'py, PyDict>>, + name: &str, +) -> PyResult> { + if let Some(kwargs) = kwargs + && let Some(value) = kwargs.get_item(name)? + { + return Ok(value); + } + args.get_item(0) +} + +fn part(duration: &Bound<'_, PyAny>, name: &str) -> PyResult { + duration.getattr(name)?.extract() +} + +fn total_months(duration: &Bound<'_, PyAny>) -> PyResult { + Ok(part(duration, "years")? * MONTHS_PER_YEAR + part(duration, "months")?) +} + +fn fixed_seconds(duration: &Bound<'_, PyAny>) -> PyResult { + Ok(part(duration, "days")? * SECONDS_PER_DAY + + part(duration, "hours")? * SECONDS_PER_HOUR + + part(duration, "minutes")? * SECONDS_PER_MINUTE + + part(duration, "seconds")?) +} + +fn units<'py>(duration: &Bound<'py, PyAny>) -> PyResult> { + let mut used: Vec<&str> = Vec::new(); + for (field, unit) in DURATION_UNITS { + if part(duration, field)? != 0 { + used.push(unit); + } + } + PyFrozenSet::new(duration.py(), used) +} + +/// `Duration.parse(value)`: the duration a text like `1y6mo` or `30d` spells, or `None`. +pub(crate) fn duration_parse( + class: &Bound<'_, PyAny>, + args: &Bound<'_, PyTuple>, + kwargs: Option<&Bound<'_, PyDict>>, +) -> PyResult> { + let py: Python<'_> = class.py(); + let value: String = argument(args, kwargs, "value")?.extract()?; + let Some(parts) = parse_cursor_duration(py, &value)? else { + return Ok(py.None()); + }; + let keywords: Bound<'_, PyDict> = PyDict::new(py); + for ((field, _), amount) in DURATION_UNITS.iter().zip(parts.iter()) { + keywords.set_item(*field, amount)?; + } + Ok(class.call((), Some(&keywords))?.unbind()) +} + +/// `Duration.is_zero`. +pub(crate) fn duration_is_zero(instance: &Bound<'_, PyAny>) -> PyResult> { + (total_months(instance)? == 0 && fixed_seconds(instance)? == 0).into_py_any(instance.py()) +} + +/// `Duration.units`: the unit suffixes with non-zero values. +pub(crate) fn duration_units(instance: &Bound<'_, PyAny>) -> PyResult> { + Ok(units(instance)?.into_any().unbind()) +} + +/// `Duration.has_calendar_component`: whether years or months are used. +pub(crate) fn duration_has_calendar_component(instance: &Bound<'_, PyAny>) -> PyResult> { + (total_months(instance)? != 0).into_py_any(instance.py()) +} + +/// `Duration.fixed_seconds`: days and below as whole seconds. +pub(crate) fn duration_fixed_seconds(instance: &Bound<'_, PyAny>) -> PyResult> { + fixed_seconds(instance)?.into_py_any(instance.py()) +} + +/// `Duration.total_months`: years folded into months. +pub(crate) fn duration_total_months(instance: &Bound<'_, PyAny>) -> PyResult> { + total_months(instance)?.into_py_any(instance.py()) +} + +/// `Duration.is_single_unit_in(allowed_units)`. +pub(crate) fn duration_is_single_unit_in( + instance: &Bound<'_, PyAny>, + args: &Bound<'_, PyTuple>, + kwargs: Option<&Bound<'_, PyDict>>, +) -> PyResult> { + let allowed: Bound<'_, PyAny> = argument(args, kwargs, "allowed_units")?; + let used: Bound<'_, PyFrozenSet> = units(instance)?; + let single: bool = used.len()? == 1 && used.as_any().le(&allowed)?; + single.into_py_any(instance.py()) +} + +/// `Duration.add_to(moment)`. +pub(crate) fn duration_add_to( + instance: &Bound<'_, PyAny>, + args: &Bound<'_, PyTuple>, + kwargs: Option<&Bound<'_, PyDict>>, +) -> PyResult> { + let moment: Bound<'_, PyAny> = argument(args, kwargs, "moment")?; + Ok(shift_by_cursor_duration(&moment, instance, 1)?.unbind()) +} + +/// `Duration.subtract_from(moment)`. +pub(crate) fn duration_subtract_from( + instance: &Bound<'_, PyAny>, + args: &Bound<'_, PyTuple>, + kwargs: Option<&Bound<'_, PyDict>>, +) -> PyResult> { + let moment: Bound<'_, PyAny> = argument(args, kwargs, "moment")?; + Ok(shift_by_cursor_duration(&moment, instance, -1)?.unbind()) +} + +/// `AlignedInterval`: its start, end and grain must form a valid aligned interval. +pub(crate) fn finish_aligned_interval( + py: Python<'_>, + values: Vec>, +) -> PyResult>> { + interval_validation(values[0].bind(py), values[1].bind(py), values[2].bind(py))?; + Ok(values) +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/record_behaviours/freshness.rs b/crates/sqlbuild-python/src/bindings/_helpers/record_behaviours/freshness.rs new file mode 100644 index 0000000000..7264226e08 --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/record_behaviours/freshness.rs @@ -0,0 +1,40 @@ +//! Validation and identity of the natively hosted source freshness state record. + +use pyo3::prelude::{Bound, Py, PyAny, PyAnyMethods, PyResult, Python}; +use pyo3::types::PyDict; + +use crate::bindings::_helpers::compile_driver::python_calls::imported; +use crate::bindings::_helpers::freshness::observation::require_aware_utc_datetime; +use crate::bindings::_helpers::records::constants::NATIVE_MODULE; + +/// `SourceFreshnessRecord.observed_at` position. +const OBSERVED_AT_FIELD: usize = 9; +const IDENTITY_FIELDS: [&str; 4] = [ + "source_name", + "target_database", + "target_schema", + "target_name", +]; + +/// `SourceFreshnessRecord`: its observed time made aware UTC, a naive time an error. +pub(crate) fn finish_freshness_record( + py: Python<'_>, + mut values: Vec>, +) -> PyResult>> { + let observed_at: Bound<'_, PyAny> = + require_aware_utc_datetime(values[OBSERVED_AT_FIELD].bind(py), "observed_at")?; + values[OBSERVED_AT_FIELD] = observed_at.unbind(); + Ok(values) +} + +/// `SourceFreshnessRecord.identity`: the source and target the observation belongs to. +pub(crate) fn freshness_record_identity(instance: &Bound<'_, PyAny>) -> PyResult> { + let py: Python<'_> = instance.py(); + let keywords: Bound<'_, PyDict> = PyDict::new(py); + for name in IDENTITY_FIELDS { + pyo3::types::PyDictMethods::set_item(&keywords, name, instance.getattr(name)?)?; + } + Ok(imported(py, NATIVE_MODULE, "SourceFreshnessIdentity")? + .call((), Some(&keywords))? + .unbind()) +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/record_behaviours/lexical.rs b/crates/sqlbuild-python/src/bindings/_helpers/record_behaviours/lexical.rs new file mode 100644 index 0000000000..dd5e7855fb --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/record_behaviours/lexical.rs @@ -0,0 +1,103 @@ +//! Validation and derived views of the natively hosted `SqlLexicalSyntax` record. + +use pyo3::IntoPyObjectExt; +use pyo3::prelude::{Bound, Py, PyAny, PyAnyMethods, PyErr, PyResult, Python}; +use pyo3::types::{PyDict, PyDictMethods, PyList, PyTuple, PyTupleMethods}; + +use sqlbuild_sqltext::sql_scan::constants::SUPPORTED_LINE_COMMENT_PREFIXES; +use sqlbuild_sqltext::sql_scan::main::lexical_syntax_key::lexical_syntax_key; +use sqlbuild_sqltext::sql_scan::main::reads_differently_from_generic::reads_differently_from_generic; +use sqlbuild_sqltext::sql_scan::main::unsupported_line_comment_prefixes::unsupported_line_comment_prefixes; +use sqlbuild_sqltext::sql_scan::models::LexicalSyntax; + +use crate::bindings::_helpers::compile_driver::python_calls::imported; +use crate::bindings::_helpers::records::constants::NATIVE_MODULE; + +/// `SqlLexicalSyntax.line_comment_prefixes` position. +const LINE_COMMENT_PREFIXES_FIELD: usize = 5; + +/// The sorted texts of a set of strings. +fn sorted_texts(values: &Bound<'_, PyAny>) -> PyResult> { + let mut texts: Vec = values + .try_iter()? + .map(|value| value?.extract::()) + .collect::>()?; + texts.sort_unstable(); + Ok(texts) +} + +/// The lexical rules a `SqlLexicalSyntax` record holds. +fn record_syntax(instance: &Bound<'_, PyAny>) -> PyResult { + let flag = |name: &str| -> PyResult { instance.getattr(name)?.is_truthy() }; + Ok(LexicalSyntax { + backslash_escape_quotes: sorted_texts(&instance.getattr("backslash_escape_quotes")?)?, + escape_string_prefix: flag("escape_string_prefix")?, + raw_string_prefix: flag("raw_string_prefix")?, + triple_quoted_strings: flag("triple_quoted_strings")?, + nested_block_comments: flag("nested_block_comments")?, + line_comment_prefixes: sorted_texts(&instance.getattr("line_comment_prefixes")?)?, + }) +} + +/// `SqlLexicalSyntax`: reject line comment prefixes the scanners cannot read. +pub(crate) fn finish_lexical_syntax( + py: Python<'_>, + values: Vec>, +) -> PyResult>> { + let prefixes: Vec = sorted_texts(values[LINE_COMMENT_PREFIXES_FIELD].bind(py))?; + let unsupported: Vec = unsupported_line_comment_prefixes(&prefixes); + if unsupported.is_empty() { + return Ok(values); + } + let supported: Vec<&str> = SUPPORTED_LINE_COMMENT_PREFIXES.to_vec(); + let message: String = format!( + "SQL line comment prefixes {} are not supported; SQLBuild scans only {}", + PyList::new(py, &unsupported)?.repr()?, + PyList::new(py, &supported)?.repr()? + ); + let error: Bound<'_, PyAny> = + imported(py, NATIVE_MODULE, "UnsupportedLexicalSyntaxError")?.call1((message,))?; + Err(PyErr::from_value(error)) +} + +/// `SqlLexicalSyntax.cache_key`: a process-stable text key for these rules. +pub(crate) fn lexical_cache_key(instance: &Bound<'_, PyAny>) -> PyResult> { + lexical_syntax_key(&record_syntax(instance)?).into_py_any(instance.py()) +} + +/// `SqlLexicalSyntax.native_mapping`: these rules as the mapping native scanners read. +pub(crate) fn lexical_native_mapping(instance: &Bound<'_, PyAny>) -> PyResult> { + let py: Python<'_> = instance.py(); + let syntax: LexicalSyntax = record_syntax(instance)?; + let mapping: Bound<'_, PyDict> = PyDict::new(py); + mapping.set_item( + "backslash_escape_quotes", + PyList::new(py, &syntax.backslash_escape_quotes)?, + )?; + mapping.set_item("escape_string_prefix", syntax.escape_string_prefix)?; + mapping.set_item("raw_string_prefix", syntax.raw_string_prefix)?; + mapping.set_item("triple_quoted_strings", syntax.triple_quoted_strings)?; + mapping.set_item("nested_block_comments", syntax.nested_block_comments)?; + mapping.set_item( + "line_comment_prefixes", + PyList::new(py, &syntax.line_comment_prefixes)?, + )?; + Ok(mapping.into_any().unbind()) +} + +/// `SqlLexicalSyntax.reads_differently_from_generic(sql)`. +pub(crate) fn lexical_reads_differently( + instance: &Bound<'_, PyAny>, + args: &Bound<'_, PyTuple>, + kwargs: Option<&Bound<'_, PyDict>>, +) -> PyResult> { + let keyword: Option> = match kwargs { + Some(kwargs) => kwargs.get_item("sql")?, + None => None, + }; + let sql: String = match keyword { + Some(sql) => sql.extract()?, + None => args.get_item(0)?.extract()?, + }; + reads_differently_from_generic(&record_syntax(instance)?, &sql).into_py_any(instance.py()) +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/record_behaviours/mod.rs b/crates/sqlbuild-python/src/bindings/_helpers/record_behaviours/mod.rs index 39ca7ba2c2..bd1cc759be 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/record_behaviours/mod.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/record_behaviours/mod.rs @@ -1,2 +1,9 @@ +pub(crate) mod auditing; +pub(crate) mod config; pub(crate) mod constants; +pub(crate) mod durations; +pub(crate) mod freshness; +pub(crate) mod lexical; +pub(crate) mod python_nodes; pub(crate) mod scopes; +pub(crate) mod values; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/record_behaviours/python_nodes.rs b/crates/sqlbuild-python/src/bindings/_helpers/record_behaviours/python_nodes.rs new file mode 100644 index 0000000000..24d78cf850 --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/record_behaviours/python_nodes.rs @@ -0,0 +1,50 @@ +//! Graph walks over the natively hosted Python-node graph record. + +use std::collections::HashSet; + +use pyo3::prelude::{Bound, Py, PyAny, PyAnyMethods, PyResult}; +use pyo3::types::{ + PyDict, PyDictMethods, PyFrozenSet, PySet, PySetMethods, PyTuple, PyTupleMethods, +}; + +/// `PythonNodeGraph.selected_sql_refs(selected_names=...)`: SQL references of the selected closure. +pub(crate) fn graph_selected_sql_refs( + graph: &Bound<'_, PyAny>, + args: &Bound<'_, PyTuple>, + kwargs: Option<&Bound<'_, PyDict>>, +) -> PyResult> { + let py = graph.py(); + let keyword: Option> = match kwargs { + Some(kwargs) => kwargs.get_item("selected_names")?, + None => None, + }; + let selected: Bound<'_, PyAny> = match keyword { + Some(selected) => selected, + None => args.get_item(0)?, + }; + let nodes_by_name: Bound<'_, PyAny> = graph.getattr("nodes_by_name")?; + let upstream: Bound<'_, PyAny> = graph.getattr("upstream_deps")?; + let refs: Bound<'_, PySet> = PySet::empty(py)?; + let mut pending: Vec = selected + .try_iter()? + .map(|name| name?.extract::()) + .collect::>()?; + let mut visited: HashSet = HashSet::new(); + while let Some(name) = pending.pop() { + if !visited.insert(name.clone()) { + continue; + } + let node: Bound<'_, PyAny> = nodes_by_name.call_method1("get", (name.as_str(),))?; + if !node.is_none() { + for reference in node.getattr("sql_deps")?.try_iter()? { + refs.add(reference?)?; + } + } + let parents: Bound<'_, PyAny> = + upstream.call_method1("get", (name.as_str(), PyTuple::empty(py)))?; + for parent in parents.try_iter()? { + pending.push(parent?.extract()?); + } + } + Ok(PyFrozenSet::new(py, refs.iter())?.into_any().unbind()) +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/record_behaviours/values.rs b/crates/sqlbuild-python/src/bindings/_helpers/record_behaviours/values.rs new file mode 100644 index 0000000000..a0b1704ce5 --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/record_behaviours/values.rs @@ -0,0 +1,45 @@ +//! Computed attributes of the natively hosted SQL value and fingerprint records. + +use pyo3::IntoPyObjectExt; +use pyo3::prelude::{Bound, Py, PyAny, PyAnyMethods, PyResult, Python}; +use pyo3::types::{PyDict, PyDictMethods, PyTuple}; + +/// `SqlLogicalType.display_name`: the kind, with its element type in angle brackets. +pub(crate) fn logical_type_display_name(instance: &Bound<'_, PyAny>) -> PyResult> { + let kind: String = instance.getattr("kind")?.getattr("value")?.extract()?; + let element: Bound<'_, PyAny> = instance.getattr("element_type")?; + if element.is_none() { + return kind.into_py_any(instance.py()); + } + let inner: String = element.getattr("display_name")?.extract()?; + format!("{kind}<{inner}>").into_py_any(instance.py()) +} + +/// `SqlValue.kind`: the outer logical kind. +pub(crate) fn sql_value_kind(instance: &Bound<'_, PyAny>) -> PyResult> { + Ok(instance.getattr("logical_type")?.getattr("kind")?.unbind()) +} + +/// `FingerprintSet`: index the fingerprints by `(node_type, node_name)` when not given. +pub(crate) fn finish_fingerprint_set( + py: Python<'_>, + mut values: Vec>, +) -> PyResult>> { + if !values[2].bind(py).is_none() { + return Ok(values); + } + let by_identity: Bound<'_, PyDict> = PyDict::new(py); + for fingerprint in values[1].bind(py).call_method0("values")?.try_iter()? { + let fingerprint: Bound<'_, PyAny> = fingerprint?; + let identity: Bound<'_, PyTuple> = PyTuple::new( + py, + [ + fingerprint.getattr("node_type")?, + fingerprint.getattr("node_name")?, + ], + )?; + by_identity.set_item(identity, fingerprint)?; + } + values[2] = by_identity.into_any().unbind(); + Ok(values) +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/records/constants.rs b/crates/sqlbuild-python/src/bindings/_helpers/records/constants.rs index 7e0a8c2d0e..2fe3621e30 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/records/constants.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/records/constants.rs @@ -4,6 +4,7 @@ use crate::bindings::_helpers::compile_records::constants::COMPILE_VALUE_RECORDS use crate::bindings::_helpers::compiler_enums::constants::COMPILER_ENUMS; use crate::bindings::_helpers::compiler_errors::constants::COMPILER_ERRORS; use crate::bindings::_helpers::compiler_records::constants::COMPILER_RECORDS; +use crate::bindings::_helpers::contract_records::constants::CONTRACT_RECORDS; use crate::bindings::_helpers::freshness::observation::normalize_observation; use crate::bindings::_helpers::planner_records::constants::PLANNER_RECORDS; use crate::bindings::_helpers::record_behaviours::constants::{ @@ -146,6 +147,7 @@ pub(crate) const RECORDS: &[&[RecordSpec]] = &[ SCOPE_RECORDS, PLANNER_RECORDS, RULE_RECORDS, + CONTRACT_RECORDS, ]; /// Records whose bound values are normalized or validated before the record is created. @@ -263,7 +265,7 @@ const BUILD_INVOCATION: RecordSpec = RecordSpec { FieldSpec::new("effective_target_name", FieldDefault::None), FieldSpec::new( "execution_limits", - FieldDefault::Call("sqlbuild.spec.contracts.models", "ExecutionLimitsConfig"), + FieldDefault::Call("sqlbuild._native", "ExecutionLimitsConfig"), ), ], }; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/records/fields.rs b/crates/sqlbuild-python/src/bindings/_helpers/records/fields.rs index 5126891bda..bfb0b4f31c 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/records/fields.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/records/fields.rs @@ -108,11 +108,13 @@ fn default_value<'py>( FieldDefault::Int(value) => value.into_bound_py_any(py), FieldDefault::Text(value) => Ok(PyString::new(py, value).into_any()), FieldDefault::TextTuple(texts) => Ok(PyTuple::new(py, texts)?.into_any()), + FieldDefault::TextFrozenset(texts) => Ok(PyFrozenSet::new(py, texts)?.into_any()), FieldDefault::EmptyTuple => Ok(PyTuple::empty(py).into_any()), FieldDefault::EmptyDict => Ok(PyDict::new(py).into_any()), FieldDefault::EmptyFrozenset => Ok(PyFrozenSet::empty(py)?.into_any()), FieldDefault::Member(module, class, member) => imported(py, module, class)?.getattr(member), FieldDefault::Call(module, name) => imported(py, module, name)?.call0(), + FieldDefault::CallText(module, name, text) => imported(py, module, name)?.call1((text,)), } } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/records/models.rs b/crates/sqlbuild-python/src/bindings/_helpers/records/models.rs index 59a6bb37f1..b8fb97993b 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/records/models.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/records/models.rs @@ -22,6 +22,8 @@ pub(crate) enum FieldDefault { Text(&'static str), /// A tuple of texts. TextTuple(&'static [&'static str]), + /// A frozenset of texts. + TextFrozenset(&'static [&'static str]), EmptyTuple, EmptyDict, EmptyFrozenset, @@ -29,6 +31,8 @@ pub(crate) enum FieldDefault { Member(&'static str, &'static str, &'static str), /// `getattr(import(module), name)()`, evaluated for every instance. Call(&'static str, &'static str), + /// `getattr(import(module), name)(text)`, evaluated for every instance. + CallText(&'static str, &'static str, &'static str), } /// One record field: its name, default and whether it takes part in equality and repr. diff --git a/crates/sqlbuild-python/src/bindings/_helpers/render/macro_expansion_result.rs b/crates/sqlbuild-python/src/bindings/_helpers/render/macro_expansion_result.rs index 005ee28870..a69975fb03 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/render/macro_expansion_result.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/render/macro_expansion_result.rs @@ -43,8 +43,8 @@ const GENERATED_REFERENCE_MARKERS: [&str; 6] = [ "__table_fn(", ]; const DIAGNOSTICS_MODULE: &str = "sqlbuild._native"; -const CONTRACT_MODELS_MODULE: &str = "sqlbuild.spec.contracts.models"; -const SQL_ANALYSIS_MODELS_MODULE: &str = "sqlbuild.compiler.sql_analysis.models"; +const CONTRACT_MODELS_MODULE: &str = "sqlbuild._native"; +const SQL_ANALYSIS_MODELS_MODULE: &str = "sqlbuild._native"; const BUILTINS_MODULE: &str = "builtins"; const MACRO_TOKEN: &str = "@"; const GENERATED_ALLOWED: &str = "macro-generated references"; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/scopes/scope_loading.rs b/crates/sqlbuild-python/src/bindings/_helpers/scopes/scope_loading.rs index a0b8ce015d..53daffc840 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/scopes/scope_loading.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/scopes/scope_loading.rs @@ -25,7 +25,7 @@ const NATIVE_RECORDS_MODULE: &str = "sqlbuild._native"; const NATIVE_MODULE: &str = "sqlbuild._native"; const SCOPE_CACHE_DIRECTORY_NAME: &str = "declaration-scopes-v2"; const SCOPE_CACHE_FILENAME: &str = "scope-index.json"; -const SQL_ANALYSIS_MODELS_MODULE: &str = "sqlbuild.compiler.sql_analysis.models"; +const SQL_ANALYSIS_MODELS_MODULE: &str = "sqlbuild._native"; const ADAPTER_NAME_MODULE: &str = "sqlbuild._native"; const COLLECTION_RENDERING_MODULE: &str = "sqlbuild._native"; const RESOLVE_ADAPTER_MODULE: &str = "sqlbuild.adapter.discovery.main.resolve_adapter"; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/semantic_checks/resource_sql.rs b/crates/sqlbuild-python/src/bindings/_helpers/semantic_checks/resource_sql.rs index d781c15a56..d403c7155f 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/semantic_checks/resource_sql.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/semantic_checks/resource_sql.rs @@ -19,7 +19,7 @@ use crate::bindings::types::CompilerDetach; const COMPILE_ERRORS_MODULE: &str = "sqlbuild.compiler.compile.exceptions"; const COMPILE_MODELS_MODULE: &str = "sqlbuild.compiler.compile.models"; -const CONTRACT_MODELS_MODULE: &str = "sqlbuild.spec.contracts.models"; +const CONTRACT_MODELS_MODULE: &str = "sqlbuild._native"; const SQL_ANALYSIS_CONFIG_KEY: &str = "sql_analysis"; const MODEL_HOOK_KEYS: [&str; 2] = ["pre_hooks", "post_hooks"]; const BUILT_IN_AUDIT_ROOT: &str = ""; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/semantic_checks/stage_objects.rs b/crates/sqlbuild-python/src/bindings/_helpers/semantic_checks/stage_objects.rs index fe679be210..e9ba4f5b49 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/semantic_checks/stage_objects.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/semantic_checks/stage_objects.rs @@ -9,7 +9,7 @@ const COMPILE_TYPES_MODULE: &str = "sqlbuild.compiler.compile.types"; const COMPILE_MODELS_MODULE: &str = "sqlbuild.compiler.compile.models"; const UNNEEDED_SQL_ANALYSIS_OPT_OUT_CODE: &str = "P009"; const SQL_ANALYSIS_CONFIG_KEY: &str = "sql_analysis"; -const CONTRACT_MODELS_MODULE: &str = "sqlbuild.spec.contracts.models"; +const CONTRACT_MODELS_MODULE: &str = "sqlbuild._native"; const PROJECT_WITH_MODULE: &str = "sqlbuild._native"; const TYPE_RECOVERY_MODULE: &str = "sqlbuild._native"; const BINDING_DIAGNOSTICS_MODULE: &str = "sqlbuild._native"; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/sql_test_glue/compiled_tests.rs b/crates/sqlbuild-python/src/bindings/_helpers/sql_test_glue/compiled_tests.rs index 96bbfa3179..56616a76da 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/sql_test_glue/compiled_tests.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/sql_test_glue/compiled_tests.rs @@ -13,7 +13,7 @@ use crate::bindings::_helpers::sql_test_glue::assembly::{ const COMPILE_MODELS_MODULE: &str = "sqlbuild.compiler.compile.models"; const COMPILE_TYPES_MODULE: &str = "sqlbuild.compiler.compile.types"; const GLUE_MODELS_MODULE: &str = "sqlbuild._native"; -const SPEC_MODELS_MODULE: &str = "sqlbuild.spec.contracts.models"; +const SPEC_MODELS_MODULE: &str = "sqlbuild._native"; /// Fields a direct-logic compiled payload copies from its input payload, in order. const DIRECT_PAYLOAD_FIELDS: [&str; 5] = [ "mode", diff --git a/crates/sqlbuild-python/src/bindings/_helpers/sql_test_glue/plan_errors.rs b/crates/sqlbuild-python/src/bindings/_helpers/sql_test_glue/plan_errors.rs index bb8bf430a2..ff3ae7f11c 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/sql_test_glue/plan_errors.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/sql_test_glue/plan_errors.rs @@ -7,7 +7,7 @@ use pyo3::types::{PyDict, PyDictMethods}; use sqlbuild_analysis::sql_test_attachment::main::sql_test_cte_location::sql_test_cte_location; use sqlbuild_analysis::sql_test_attachment::models::CteLocation; -const SOURCE_LOCATION_MODULE: &str = "sqlbuild.spec.contracts.models"; +const SOURCE_LOCATION_MODULE: &str = "sqlbuild._native"; /// The location of `call` inside one CTE of a SQL test block, else the CTE header, else the block. fn cte_call_location<'py>( diff --git a/crates/sqlbuild-python/src/bindings/_helpers/state_rows/constants.rs b/crates/sqlbuild-python/src/bindings/_helpers/state_rows/constants.rs index 7bdc577590..ba226c37d6 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/state_rows/constants.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/state_rows/constants.rs @@ -5,7 +5,7 @@ pub(crate) const MIGRATION_TYPES: &str = "sqlbuild.compiler.migrations.types"; pub(crate) const MIGRATION_EXCEPTIONS: &str = "sqlbuild._native"; pub(crate) const STATE_TABLE_DDL: &str = "sqlbuild.adapter.state_sql.main.render_state_table_create_sql"; -pub(crate) const SQL_VALUE_TYPES: &str = "sqlbuild.sql_values.types"; +pub(crate) const SQL_VALUE_TYPES: &str = "sqlbuild._native"; pub(crate) const OLD_NAME_VIEW_CONFIG_KEY: &str = "old_name_view"; pub(crate) const MODEL_KIND: &str = "model"; pub(crate) const COLUMN_KIND: &str = "column"; diff --git a/crates/sqlbuild-sqltext/src/sql_scan/constants.rs b/crates/sqlbuild-sqltext/src/sql_scan/constants.rs index b9bec2ab1a..f7d11818a4 100644 --- a/crates/sqlbuild-sqltext/src/sql_scan/constants.rs +++ b/crates/sqlbuild-sqltext/src/sql_scan/constants.rs @@ -6,3 +6,11 @@ pub(crate) const STRING_PREFIX_CHARACTERS: &[u8] = b"bBeErR"; pub(crate) const STRING_PREFIX_MAX_LENGTH: usize = 2; /// Length of a triple-quote string delimiter. pub(crate) const TRIPLE_QUOTE_LENGTH: usize = 3; +/// Line comment prefixes SQLBuild's scanners understand. +pub const SUPPORTED_LINE_COMMENT_PREFIXES: [&str; 3] = ["#", "--", "//"]; +/// The generic SQL line comment prefix. +pub const GENERIC_LINE_COMMENT_PREFIX: &str = "--"; +/// Quote characters a triple-quoted string may repeat. +pub const TRIPLE_QUOTES: [&str; 2] = ["'''", "\"\"\""]; +pub const ESCAPE_CHARACTER: char = '\\'; +pub const BLOCK_COMMENT_OPEN: &str = "/*"; diff --git a/crates/sqlbuild-sqltext/src/sql_scan/main/lexical_syntax_key.rs b/crates/sqlbuild-sqltext/src/sql_scan/main/lexical_syntax_key.rs new file mode 100644 index 0000000000..ff2cd882d3 --- /dev/null +++ b/crates/sqlbuild-sqltext/src/sql_scan/main/lexical_syntax_key.rs @@ -0,0 +1,30 @@ +//! The process-stable text key of one dialect's lexical rules. + +use crate::sql_scan::models::LexicalSyntax; + +/// `SqlLexicalSyntax.cache_key`: sorted quotes, the four flags as Python booleans, sorted prefixes. +#[must_use] +pub fn lexical_syntax_key(syntax: &LexicalSyntax) -> String { + let flag = |value: bool| if value { "True" } else { "False" }; + let mut quotes: Vec<&str> = syntax + .backslash_escape_quotes + .iter() + .map(String::as_str) + .collect(); + quotes.sort_unstable(); + let mut prefixes: Vec<&str> = syntax + .line_comment_prefixes + .iter() + .map(String::as_str) + .collect(); + prefixes.sort_unstable(); + [ + quotes.join(","), + flag(syntax.escape_string_prefix).to_owned(), + flag(syntax.raw_string_prefix).to_owned(), + flag(syntax.triple_quoted_strings).to_owned(), + flag(syntax.nested_block_comments).to_owned(), + prefixes.join(","), + ] + .join("|") +} diff --git a/crates/sqlbuild-sqltext/src/sql_scan/main/mod.rs b/crates/sqlbuild-sqltext/src/sql_scan/main/mod.rs index 2b5b0542f4..69cddeb649 100644 --- a/crates/sqlbuild-sqltext/src/sql_scan/main/mod.rs +++ b/crates/sqlbuild-sqltext/src/sql_scan/main/mod.rs @@ -2,9 +2,12 @@ pub mod comment_end; pub mod dialect_matching_paren; pub mod dialect_non_code_end; pub mod dialect_non_code_ranges; +pub mod lexical_syntax_key; pub mod matching_paren; pub mod non_code_end; pub mod quote_end; pub mod quote_policy; +pub mod reads_differently_from_generic; pub mod rules_dialect; pub mod skip_whitespace; +pub mod unsupported_line_comment_prefixes; diff --git a/crates/sqlbuild-sqltext/src/sql_scan/main/reads_differently_from_generic.rs b/crates/sqlbuild-sqltext/src/sql_scan/main/reads_differently_from_generic.rs new file mode 100644 index 0000000000..05479da834 --- /dev/null +++ b/crates/sqlbuild-sqltext/src/sql_scan/main/reads_differently_from_generic.rs @@ -0,0 +1,26 @@ +//! Whether one dialect's lexical rules can read a SQL text differently from generic SQL. + +use crate::sql_scan::constants::{ + BLOCK_COMMENT_OPEN, ESCAPE_CHARACTER, GENERIC_LINE_COMMENT_PREFIX, TRIPLE_QUOTES, +}; +use crate::sql_scan::models::LexicalSyntax; + +/// `SqlLexicalSyntax.reads_differently_from_generic`. +#[must_use] +pub fn reads_differently_from_generic(syntax: &LexicalSyntax, sql: &str) -> bool { + if sql.contains(ESCAPE_CHARACTER) + && (!syntax.backslash_escape_quotes.is_empty() || syntax.escape_string_prefix) + { + return true; + } + if syntax.triple_quoted_strings && TRIPLE_QUOTES.iter().any(|quote| sql.contains(quote)) { + return true; + } + if syntax.nested_block_comments && sql.matches(BLOCK_COMMENT_OPEN).count() > 1 { + return true; + } + syntax + .line_comment_prefixes + .iter() + .any(|prefix| prefix != GENERIC_LINE_COMMENT_PREFIX && sql.contains(prefix.as_str())) +} diff --git a/crates/sqlbuild-sqltext/src/sql_scan/main/unsupported_line_comment_prefixes.rs b/crates/sqlbuild-sqltext/src/sql_scan/main/unsupported_line_comment_prefixes.rs new file mode 100644 index 0000000000..02f3662da1 --- /dev/null +++ b/crates/sqlbuild-sqltext/src/sql_scan/main/unsupported_line_comment_prefixes.rs @@ -0,0 +1,16 @@ +//! The line comment prefixes a dialect declares that SQLBuild cannot scan. + +use crate::sql_scan::constants::SUPPORTED_LINE_COMMENT_PREFIXES; + +/// The declared prefixes outside the supported set, sorted and without repeats. +#[must_use] +pub fn unsupported_line_comment_prefixes(prefixes: &[String]) -> Vec { + let mut unsupported: Vec = prefixes + .iter() + .filter(|prefix| !SUPPORTED_LINE_COMMENT_PREFIXES.contains(&prefix.as_str())) + .cloned() + .collect(); + unsupported.sort_unstable(); + unsupported.dedup(); + unsupported +} diff --git a/crates/sqlbuild-sqltext/src/sql_scan/tests.rs b/crates/sqlbuild-sqltext/src/sql_scan/tests.rs index ab9f7b24dc..3da57ecf98 100644 --- a/crates/sqlbuild-sqltext/src/sql_scan/tests.rs +++ b/crates/sqlbuild-sqltext/src/sql_scan/tests.rs @@ -2,6 +2,8 @@ mod dialect_scan; #[path = "tests/helpers.rs"] mod helpers; +#[path = "tests/test_lexical_rules.rs"] +mod lexical_rules; #[path = "tests/test_sql_scan.rs"] mod sql_scan; #[path = "tests/test_types.rs"] diff --git a/crates/sqlbuild-sqltext/src/sql_scan/tests/helpers.rs b/crates/sqlbuild-sqltext/src/sql_scan/tests/helpers.rs index 3a9c34da1a..4adf633977 100644 --- a/crates/sqlbuild-sqltext/src/sql_scan/tests/helpers.rs +++ b/crates/sqlbuild-sqltext/src/sql_scan/tests/helpers.rs @@ -41,3 +41,15 @@ pub(crate) fn raw_strings() -> LexicalSyntax { ..generic() } } + +/// Lexical rules with the given quotes, `[escape, raw, triple, nested]` flags and prefixes. +pub(crate) fn syntax(quotes: &[&str], flags: [bool; 4], prefixes: &[&str]) -> LexicalSyntax { + LexicalSyntax { + backslash_escape_quotes: quotes.iter().map(|quote| (*quote).to_owned()).collect(), + escape_string_prefix: flags[0], + raw_string_prefix: flags[1], + triple_quoted_strings: flags[2], + nested_block_comments: flags[3], + line_comment_prefixes: prefixes.iter().map(|prefix| (*prefix).to_owned()).collect(), + } +} diff --git a/crates/sqlbuild-sqltext/src/sql_scan/tests/test_lexical_rules.rs b/crates/sqlbuild-sqltext/src/sql_scan/tests/test_lexical_rules.rs new file mode 100644 index 0000000000..8d862507f7 --- /dev/null +++ b/crates/sqlbuild-sqltext/src/sql_scan/tests/test_lexical_rules.rs @@ -0,0 +1,82 @@ +use crate::sql_scan::main::lexical_syntax_key::lexical_syntax_key; +use crate::sql_scan::main::reads_differently_from_generic::reads_differently_from_generic; +use crate::sql_scan::main::unsupported_line_comment_prefixes::unsupported_line_comment_prefixes; +use crate::sql_scan::tests::helpers::syntax; +use crate::sql_scan::tests::test_types::{LexicalRulesTestCase, PrefixCheckTestCase}; + +#[test] +fn given_lexical_rules_when_keyed_and_compared_then_match_python_properties() { + let test_cases = [ + LexicalRulesTestCase { + description: "generic rules read SQL as generic SQL", + syntax: syntax(&[], [false; 4], &["--"]), + sql: "select '\\' -- note", + expected_key: "|False|False|False|False|--", + expected_reads_differently: false, + }, + LexicalRulesTestCase { + description: "backslash quotes change escaped text", + syntax: syntax(&["\"", "'"], [false, true, false, false], &["--", "#"]), + sql: "select 'a\\'", + expected_key: "\",'|False|True|False|False|#,--", + expected_reads_differently: true, + }, + LexicalRulesTestCase { + description: "nested comments need two openers", + syntax: syntax(&[], [false, false, false, true], &["--"]), + sql: "/* a /* b */ */ select 1", + expected_key: "|False|False|False|True|--", + expected_reads_differently: true, + }, + LexicalRulesTestCase { + description: "a hash comment prefix reads hash text", + syntax: syntax(&[], [false; 4], &["--", "#"]), + sql: "select 1 # trailing", + expected_key: "|False|False|False|False|#,--", + expected_reads_differently: true, + }, + ]; + for test_case in test_cases { + assert_eq!( + ( + lexical_syntax_key(&test_case.syntax), + reads_differently_from_generic(&test_case.syntax, test_case.sql), + ), + ( + test_case.expected_key.to_owned(), + test_case.expected_reads_differently + ), + "{}", + test_case.description + ); + } +} + +#[test] +fn given_declared_prefixes_when_checking_then_only_unsupported_ones_are_returned() { + let test_cases = [ + PrefixCheckTestCase { + description: "unsupported prefixes are sorted without repeats", + declared: &["%", "--", "%", ";"], + expected_unsupported: &["%", ";"], + }, + PrefixCheckTestCase { + description: "supported prefixes pass", + declared: &["#", "//", "--"], + expected_unsupported: &[], + }, + ]; + for test_case in test_cases { + let declared: Vec = test_case + .declared + .iter() + .map(|prefix| (*prefix).to_owned()) + .collect(); + assert_eq!( + unsupported_line_comment_prefixes(&declared), + test_case.expected_unsupported, + "{}", + test_case.description + ); + } +} diff --git a/crates/sqlbuild-sqltext/src/sql_scan/tests/test_types.rs b/crates/sqlbuild-sqltext/src/sql_scan/tests/test_types.rs index ba72c5cbfb..86883ba217 100644 --- a/crates/sqlbuild-sqltext/src/sql_scan/tests/test_types.rs +++ b/crates/sqlbuild-sqltext/src/sql_scan/tests/test_types.rs @@ -1,5 +1,5 @@ -use crate::sql_scan::models::QuotePolicy; use crate::sql_scan::models::Unclosed; +use crate::sql_scan::models::{LexicalSyntax, QuotePolicy}; pub(crate) struct MatchingParenPolicyTestCase { pub(crate) description: &'static str, @@ -28,3 +28,17 @@ pub(crate) struct DialectNonCodeRangesTestCase { pub(crate) syntax: crate::sql_scan::models::LexicalSyntax, pub(crate) expected_marker_protected: bool, } + +pub(crate) struct LexicalRulesTestCase { + pub(crate) description: &'static str, + pub(crate) syntax: LexicalSyntax, + pub(crate) sql: &'static str, + pub(crate) expected_key: &'static str, + pub(crate) expected_reads_differently: bool, +} + +pub(crate) struct PrefixCheckTestCase { + pub(crate) description: &'static str, + pub(crate) declared: &'static [&'static str], + pub(crate) expected_unsupported: &'static [&'static str], +} diff --git a/python-boundary.toml b/python-boundary.toml index f6aaf0a3d0..71d8cf95cf 100644 --- a/python-boundary.toml +++ b/python-boundary.toml @@ -2,7 +2,7 @@ # migration work that moves to Rust. `remaining_lines` must equal the current count of that work: # a PR that moves code lowers it, and adding movable Python fails the check. # Native failures must surface as errors; `handled_native_errors` lists the reviewed exceptions. -remaining_lines = 5280 +remaining_lines = 3851 [[stays]] path = "executor" diff --git a/src/sqlbuild/_native.pyi b/src/sqlbuild/_native.pyi index e7bead5092..01f4c8b415 100644 --- a/src/sqlbuild/_native.pyi +++ b/src/sqlbuild/_native.pyi @@ -37,17 +37,13 @@ from sqlbuild.adapter.contract.types import ( ) from sqlbuild.adapter.relations.models import InspectionQueryRecord from sqlbuild.cli.compile_reuse.models import CompileReuseRequest -from sqlbuild.compiler.auditing.models import ( - AuditGateIdentity, - MeasurementContract, - MeasurementThresholds, -) from sqlbuild.compiler.auditing.types import ( AuditAttachmentKind, AuditEvaluationMode, AuditOutcome, AuditRunScope, AuditSeverity, + ThresholdOperator, ) from sqlbuild.compiler.compile.models import ( CompiledAudit, @@ -105,7 +101,6 @@ from sqlbuild.compiler.discovery.models import ( SqlTestParameterDeclaration, ) from sqlbuild.compiler.discovery.types import LoaderConnectionMode -from sqlbuild.compiler.fingerprints.models import Fingerprint, FingerprintSet from sqlbuild.compiler.lineage.types import ( InferredNullability, ) @@ -167,10 +162,6 @@ from sqlbuild.compiler.planner.types import ( PlanReason, ScenarioArtifactKind, ) -from sqlbuild.compiler.python_nodes.models import ( - PythonNodeGraph, - PythonSqlRunLifecyclePlan, -) from sqlbuild.compiler.python_nodes.types import PythonNodeKind from sqlbuild.compiler.references.models import ( ProjectRelation, @@ -466,6 +457,14 @@ type NativeSqlTestPlanRow = tuple[ type SqlTestAssemblyFailureKind = Literal["input", "internal", "decimal_overflow"] +type SqlScalar = str | int | bool | float | Decimal | None +type SqlValuePayload = SqlScalar | tuple[object, ...] + +class DbtInvoker(Protocol): + """Callable dbt process invocation contract.""" + + def __call__(self, *, argv: tuple[str, ...], cwd: Path | None) -> object: ... + class FingerprintWriteProgress(Protocol): def __call__(self, *, completed: int, total: int) -> None: ... @@ -14054,3 +14053,1467 @@ class ScopeReport(NativeRecord): diagnostics: tuple[ScopeDiagnostic, ...] = ..., complete: bool = ..., ) -> None: ... + +class EventExportSeverity(StrEnum): + """Severity used to filter exported lifecycle events.""" + + DEBUG = "debug" + INFO = "info" + WARNING = "warning" + ERROR = "error" + +class WarehouseGroup(StrEnum): + """Command group that selects a target's default warehouse.""" + + BUILD = "build" + QUERY = "query" + +class SourceWriteStrategy(StrEnum): + APPEND = "append" + DELETE_INSERT = "delete_insert" + MERGE = "merge" + TABLE = "table" + +class FutureCursorAction(StrEnum): + """Response when an effective cursor lies beyond the future horizon.""" + + CAP = "cap" + ERROR = "error" + +class MicrobatchLimitAction(StrEnum): + """Response when one model plans more than the configured batch limit.""" + + CAP_FROM_END = "cap_from_end" + CAP_FROM_START = "cap_from_start" + ERROR = "error" + WARN = "warn" + +class ColumnContractMode(StrEnum): + """Default contract behavior for model column declarations.""" + + IMPLICIT = "implicit" + EXPLICIT = "explicit" + +class SourceFreshnessStrategy(StrEnum): + ADAPTER = "adapter" + COLUMN = "column" + SQL = "sql" + +class SourceFreshnessValueKind(StrEnum): + TIMESTAMP = "timestamp" + INTEGER = "integer" + STRING = "string" + +class TimeTravelRetentionSource(StrEnum): + """Authored layer that supplied the effective retention policy.""" + + TARGET = "target" + MATERIALIZATION = "materialization" + MODEL = "model" + +class TableType(StrEnum): + """Snowflake managed table type.""" + + PERMANENT = "permanent" + TRANSIENT = "transient" + +class TableTypeSource(StrEnum): + """Authored layer that supplied the effective table type.""" + + DEFAULT = "default" + TARGET = "target" + MATERIALIZATION = "materialization" + MODEL = "model" + +class TableTypeDowngradePolicy(StrEnum): + """Permanent-to-transient conversion policy.""" + + DENY = "deny" + REQUIRE_CONFIRMATION = "require_confirmation" + ALLOW = "allow" + +class MissingMigrationOriginPolicy(StrEnum): + """Policy for a declared migrate_from whose origin does not exist in a target.""" + + DENY = "deny" + REQUIRE_CONFIRMATION = "require_confirmation" + ALLOW = "allow" + +class RetentionDecreasePolicy(StrEnum): + """Policy for lowering live time-travel retention.""" + + DENY = "deny" + REQUIRE_CONFIRMATION = "require_confirmation" + ALLOW = "allow" + +class ClonePolicy(NativeRecord): + """Environment clone policy.""" + + allow_as_clone_origin: bool + allow_as_clone_destination: bool + def __init__( + self, + allow_as_clone_origin: bool = ..., + allow_as_clone_destination: bool = ..., + ) -> None: ... + +class LocalClonePolicy(NativeRecord): + """Local environment clone policy overrides.""" + + allow_as_clone_origin: bool | None + allow_as_clone_destination: bool | None + def __init__( + self, + allow_as_clone_origin: bool | None = ..., + allow_as_clone_destination: bool | None = ..., + ) -> None: ... + +class AuthoredTimeTravelRetention(NativeRecord): + """One authored time-travel retention policy.""" + + desired_days: int | None + unmanaged: bool + def __init__( + self, + desired_days: int | None = ..., + unmanaged: bool = ..., + ) -> None: ... + +class MaterializationRetentionDefaults(NativeRecord): + """Retention defaults for one built-in materialization.""" + + time_travel_retention: AuthoredTimeTravelRetention | None + table_type: TableType | None + def __init__( + self, + time_travel_retention: AuthoredTimeTravelRetention | None = ..., + table_type: TableType | None = ..., + ) -> None: ... + +class MaterializationDefaultsConfig(NativeRecord): + """Project retention defaults by materialization type.""" + + table: MaterializationRetentionDefaults + incremental: MaterializationRetentionDefaults + snapshot: MaterializationRetentionDefaults + def __init__( + self, + table: MaterializationRetentionDefaults = ..., + incremental: MaterializationRetentionDefaults = ..., + snapshot: MaterializationRetentionDefaults = ..., + ) -> None: ... + +class ResolvedTimeTravelRetention(NativeRecord): + """Effective retention policy attached to a compiled model.""" + + desired_days: int | None + unmanaged: bool + source: TimeTravelRetentionSource | None + def __init__( + self, + desired_days: int | None = ..., + unmanaged: bool = ..., + source: TimeTravelRetentionSource | None = ..., + ) -> None: ... + +class ResolvedTableType(NativeRecord): + """Effective Snowflake table type attached to a compiled model.""" + + value: TableType + source: TableTypeSource + declared: bool + def __init__( + self, + value: TableType = ..., + source: TableTypeSource = ..., + declared: bool = ..., + ) -> None: ... + +class ExecutionLimitsConfig(NativeRecord): + """Optional target-scoped build execution limits.""" + + max_models: int | None + max_duration: str | None + max_duration_seconds: int | None + remediation: str | None + def __init__( + self, + max_models: int | None = ..., + max_duration: str | None = ..., + max_duration_seconds: int | None = ..., + remediation: str | None = ..., + ) -> None: ... + +class TargetDiffConfig(NativeRecord): + """Optional target-scoped data-diff limits.""" + + max_full_rows: int | Literal["unlimited"] | None + def __init__( + self, + max_full_rows: int | Literal["unlimited"] | None = ..., + ) -> None: ... + +class TargetWarehousesConfig(NativeRecord): + """Optional per-command-group default warehouses for one target.""" + + build: str | None + query: str | None + def __init__( + self, + build: str | None = ..., + query: str | None = ..., + ) -> None: ... + +class TargetConfig(NativeRecord): + """One named target configuration.""" + + connection: dict[str, object] + connection_name: str | None + vars: dict[str, str] + database: str | None + schema: str | None + loader_schema: str | None + defer_sources_to: str | None + defer_clone_from: str | None + clone: ClonePolicy + compile_cache: bool | None + time_travel_retention: AuthoredTimeTravelRetention | None + time_travel_retention_by_materialization: dict[str, AuthoredTimeTravelRetention] + owns_time_travel_retention_namespace: bool + default_table_type: TableType | None + table_type_downgrade: TableTypeDowngradePolicy + time_travel_retention_decrease: RetentionDecreasePolicy + missing_migration_origin: MissingMigrationOriginPolicy + execution_limits: ExecutionLimitsConfig + diff: TargetDiffConfig + warehouses: TargetWarehousesConfig + def __init__( + self, + connection: dict[str, object] = ..., + connection_name: str | None = ..., + vars: dict[str, str] = ..., + database: str | None = ..., + schema: str | None = ..., + loader_schema: str | None = ..., + defer_sources_to: str | None = ..., + defer_clone_from: str | None = ..., + clone: ClonePolicy = ..., + compile_cache: bool | None = ..., + time_travel_retention: AuthoredTimeTravelRetention | None = ..., + time_travel_retention_by_materialization: dict[str, AuthoredTimeTravelRetention] = ..., + owns_time_travel_retention_namespace: bool = ..., + default_table_type: TableType | None = ..., + table_type_downgrade: TableTypeDowngradePolicy = ..., + time_travel_retention_decrease: RetentionDecreasePolicy = ..., + missing_migration_origin: MissingMigrationOriginPolicy = ..., + execution_limits: ExecutionLimitsConfig = ..., + diff: TargetDiffConfig = ..., + warehouses: TargetWarehousesConfig = ..., + ) -> None: ... + +class LocalTargetConfig(NativeRecord): + """Local developer overrides for one named target.""" + + connection: dict[str, object] + connection_name: str | None + vars: dict[str, str] + database: str | None + schema: str | None + loader_schema: str | None + defer_sources_to: str | None + defer_clone_from: str | None + clone: LocalClonePolicy + compile_cache: bool | None + time_travel_retention: AuthoredTimeTravelRetention | None + time_travel_retention_by_materialization: dict[str, AuthoredTimeTravelRetention] | None + owns_time_travel_retention_namespace: bool | None + default_table_type: TableType | None + table_type_downgrade: TableTypeDowngradePolicy | None + time_travel_retention_decrease: RetentionDecreasePolicy | None + missing_migration_origin: MissingMigrationOriginPolicy | None + execution_limits: ExecutionLimitsConfig + diff: TargetDiffConfig + warehouses: TargetWarehousesConfig + def __init__( + self, + connection: dict[str, object] = ..., + connection_name: str | None = ..., + vars: dict[str, str] = ..., + database: str | None = ..., + schema: str | None = ..., + loader_schema: str | None = ..., + defer_sources_to: str | None = ..., + defer_clone_from: str | None = ..., + clone: LocalClonePolicy = ..., + compile_cache: bool | None = ..., + time_travel_retention: AuthoredTimeTravelRetention | None = ..., + time_travel_retention_by_materialization: dict[str, AuthoredTimeTravelRetention] + | None = ..., + owns_time_travel_retention_namespace: bool | None = ..., + default_table_type: TableType | None = ..., + table_type_downgrade: TableTypeDowngradePolicy | None = ..., + time_travel_retention_decrease: RetentionDecreasePolicy | None = ..., + missing_migration_origin: MissingMigrationOriginPolicy | None = ..., + execution_limits: ExecutionLimitsConfig = ..., + diff: TargetDiffConfig = ..., + warehouses: TargetWarehousesConfig = ..., + ) -> None: ... + +class SettingsConfig(NativeRecord): + """Global feature toggles.""" + + sql_analysis: bool + require_sql_analysis: bool + query_change_tracking: bool + column_contract_mode: ColumnContractMode + concurrency: int + auto_load_sources: bool + microbatch_concurrency: bool + microbatch_unaccounted_partition_policy: str + table_promotion_mode: str | None + default_audit_severity: AuditSeverity | None + default_audit_run_scope: str | None + def __init__( + self, + sql_analysis: bool = ..., + require_sql_analysis: bool = ..., + query_change_tracking: bool = ..., + column_contract_mode: ColumnContractMode = ..., + concurrency: int = ..., + auto_load_sources: bool = ..., + microbatch_concurrency: bool = ..., + microbatch_unaccounted_partition_policy: str = ..., + table_promotion_mode: str | None = ..., + default_audit_severity: AuditSeverity | None = ..., + default_audit_run_scope: str | None = ..., + ) -> None: ... + @property + def sql_validation(self) -> bool: ... + +class ScopesConfig(NativeRecord): + """Declaration scope policy configuration.""" + + enforce_placement: bool + def __init__( + self, + enforce_placement: bool = ..., + ) -> None: ... + +class ReferencesConfig(NativeRecord): + """Explicit-reference enforcement policy configuration.""" + + enforce_explicit: bool + def __init__( + self, + enforce_explicit: bool = ..., + ) -> None: ... + +class CostConfig(NativeRecord): + """Snowflake compute estimate configuration.""" + + usd_per_credit: Decimal + usd_per_credit_is_default: bool + def __init__( + self, + usd_per_credit: Decimal = ..., + usd_per_credit_is_default: bool = ..., + ) -> None: ... + +class ConstantsConfig(NativeRecord): + """Project-wide constant rendering configuration.""" + + collection_rendering: CollectionRendering + def __init__( + self, + collection_rendering: CollectionRendering = ..., + ) -> None: ... + +class FutureCursorsConfig(NativeRecord): + """Safety policy for automatically discovered future cursor watermarks.""" + + max_distance: str | None + action: FutureCursorAction + def __init__( + self, + max_distance: str | None = ..., + action: FutureCursorAction = ..., + ) -> None: ... + +class StartCursorsConfig(NativeRecord): + """Safety policy for automatically discovered incremental starts.""" + + max_ahead: str | None + action: FutureCursorAction + def __init__( + self, + max_ahead: str | None = ..., + action: FutureCursorAction = ..., + ) -> None: ... + +class CursorsConfig(NativeRecord): + """Project-wide cursor safety configuration.""" + + future: FutureCursorsConfig + start: StartCursorsConfig + def __init__( + self, + future: FutureCursorsConfig = ..., + start: StartCursorsConfig = ..., + ) -> None: ... + +class MicrobatchLimitsConfig(NativeRecord): + """Per-model microbatch planning safety limit.""" + + max_batches: int | None + action: MicrobatchLimitAction + def __init__( + self, + max_batches: int | None = ..., + action: MicrobatchLimitAction = ..., + ) -> None: ... + +class MicrobatchesConfig(NativeRecord): + """Project-wide microbatch configuration.""" + + limits: MicrobatchLimitsConfig + def __init__( + self, + limits: MicrobatchLimitsConfig = ..., + ) -> None: ... + +class DefaultsConfig(NativeRecord): + """Project-wide resource defaults.""" + + materialized: str | None + database: str | None + schema: str | None + seed_database: str | None + seed_schema: str | None + function_database: str | None + function_schema: str | None + incremental_strategy: str | None + incremental_mode: str | None + microbatch_strategy: str | None + cursor_watermark_mode: str | None + merge_exclude_columns: tuple[str, ...] + full_refresh: bool | None + append_cursor_inclusive: object | None + cursor_start: object | None + cursor_end: object | None + cursor_start_max_ahead: str | None + cursor_start_max_action: str | None + cursor_future_max_distance: str | None + cursor_future_action: str | None + lookback: str | None + batch_size: str | int | None + batch_concurrency: int | None + max_microbatches: int | None + unaccounted_partition_policy: str | None + replay_on_change: str | None + row_diff_exclude_columns: tuple[str, ...] + row_diff_tolerances: dict[str, object] + row_diff_sample_rows: int | None + row_diff_sample_seed: int | None + tags: tuple[str, ...] + contract: str | None + pre_hooks: object | None + post_hooks: object | None + def __init__( + self, + materialized: str | None = ..., + database: str | None = ..., + schema: str | None = ..., + seed_database: str | None = ..., + seed_schema: str | None = ..., + function_database: str | None = ..., + function_schema: str | None = ..., + incremental_strategy: str | None = ..., + incremental_mode: str | None = ..., + microbatch_strategy: str | None = ..., + cursor_watermark_mode: str | None = ..., + merge_exclude_columns: tuple[str, ...] = ..., + full_refresh: bool | None = ..., + append_cursor_inclusive: object | None = ..., + cursor_start: object | None = ..., + cursor_end: object | None = ..., + cursor_start_max_ahead: str | None = ..., + cursor_start_max_action: str | None = ..., + cursor_future_max_distance: str | None = ..., + cursor_future_action: str | None = ..., + lookback: str | None = ..., + batch_size: str | int | None = ..., + batch_concurrency: int | None = ..., + max_microbatches: int | None = ..., + unaccounted_partition_policy: str | None = ..., + replay_on_change: str | None = ..., + row_diff_exclude_columns: tuple[str, ...] = ..., + row_diff_tolerances: dict[str, object] = ..., + row_diff_sample_rows: int | None = ..., + row_diff_sample_seed: int | None = ..., + tags: tuple[str, ...] = ..., + contract: str | None = ..., + pre_hooks: object | None = ..., + post_hooks: object | None = ..., + ) -> None: ... + +class DiffConfig(NativeRecord): + """Project-wide data-diff configuration.""" + + query_artifact_ttl: str + def __init__( + self, + query_artifact_ttl: str = ..., + ) -> None: ... + +class JanitorConfig(NativeRecord): + """Janitor command defaults.""" + + enabled: bool + retention_days: int | None + archive_retention_days: int + direct_state_history_versions: int + delete_tracked_only: bool + exclude_patterns: tuple[str, ...] + def __init__( + self, + enabled: bool = ..., + retention_days: int | None = ..., + archive_retention_days: int = ..., + direct_state_history_versions: int = ..., + delete_tracked_only: bool = ..., + exclude_patterns: tuple[str, ...] = ..., + ) -> None: ... + +class MigrationsConfig(NativeRecord): + """Model migration defaults.""" + + old_name_views: str | None + def __init__( + self, + old_name_views: str | None = ..., + ) -> None: ... + +class SnapshotsConfig(NativeRecord): + """Snapshot materialization safety defaults.""" + + current_state_full_refresh: str + historical_full_refresh: str + schema_change: str + wildcard_check_schema_change: str + def __init__( + self, + current_state_full_refresh: str = ..., + historical_full_refresh: str = ..., + schema_change: str = ..., + wildcard_check_schema_change: str = ..., + ) -> None: ... + +class ScenarioSnapshotLimitsConfig(NativeRecord): + """Scenario snapshot capture safety limits.""" + + max_rows_per_relation: int | None + max_total_rows: int | None + max_bytes_per_relation: int | None + max_total_bytes: int | None + def __init__( + self, + max_rows_per_relation: int | None = ..., + max_total_rows: int | None = ..., + max_bytes_per_relation: int | None = ..., + max_total_bytes: int | None = ..., + ) -> None: ... + +class ScenarioConfig(NativeRecord): + """Scenario command configuration.""" + + run_namespace: str | None + local_type_overrides: dict[str, dict[str, str]] + snapshot_limits: ScenarioSnapshotLimitsConfig + def __init__( + self, + run_namespace: str | None = ..., + local_type_overrides: dict[str, dict[str, str]] = ..., + snapshot_limits: ScenarioSnapshotLimitsConfig = ..., + ) -> None: ... + +class DbtConfig(NativeRecord): + """dbt interop configuration.""" + + project_dir: str | None + profiles_dir: str | None + target: str | None + target_path: str | None + vars: dict[str, object] + def __init__( + self, + project_dir: str | None = ..., + profiles_dir: str | None = ..., + target: str | None = ..., + target_path: str | None = ..., + vars: dict[str, object] = ..., + ) -> None: ... + +class LocalDbtConfig(NativeRecord): + """Local dbt interop configuration overrides.""" + + target: str | None + vars: dict[str, object] + def __init__( + self, + target: str | None = ..., + vars: dict[str, object] = ..., + ) -> None: ... + +class LifecycleEventSinkFilterConfig(NativeRecord): + """Destination-neutral runtime narrowing for lifecycle-event sinks.""" + + event_kinds: frozenset[str] | None + min_severity: EventExportSeverity | None + def __init__( + self, + event_kinds: frozenset[str] | None = ..., + min_severity: EventExportSeverity | None = ..., + ) -> None: ... + +class LifecycleEventSinksConfig(NativeRecord): + """Global and named lifecycle-event sink runtime filters and delivery settings.""" + + defaults: LifecycleEventSinkFilterConfig + named: dict[str, LifecycleEventSinkFilterConfig] + shutdown_timeout: str | None + shutdown_timeout_seconds: int | None + def __init__( + self, + defaults: LifecycleEventSinkFilterConfig = ..., + named: dict[str, LifecycleEventSinkFilterConfig] = ..., + shutdown_timeout: str | None = ..., + shutdown_timeout_seconds: int | None = ..., + ) -> None: ... + +class SinksConfig(NativeRecord): + """Typed destination sink configuration.""" + + lifecycle: LifecycleEventSinksConfig + def __init__( + self, + lifecycle: LifecycleEventSinksConfig = ..., + ) -> None: ... + +class ProjectConfig(NativeRecord): + """Shared project configuration loaded from sqlbuild_project.toml.""" + + name: str + adapter: str + default_target: str | None + connection: dict[str, object] + connections: dict[str, dict[str, object]] + settings: SettingsConfig + scopes: ScopesConfig + references: ReferencesConfig + cost: CostConfig + constants: ConstantsConfig + cursors: CursorsConfig + microbatches: MicrobatchesConfig + defaults: DefaultsConfig + diff: DiffConfig + materialization_defaults: MaterializationDefaultsConfig + path_defaults: dict[str, dict[str, object]] + vars: dict[str, str] + targets: dict[str, TargetConfig] + janitor: JanitorConfig + migrations: MigrationsConfig + snapshots: SnapshotsConfig + scenario: ScenarioConfig + dbt: DbtConfig + sinks: SinksConfig + def __init__( + self, + name: str, + adapter: str, + default_target: str | None = ..., + connection: dict[str, object] = ..., + connections: dict[str, dict[str, object]] = ..., + settings: SettingsConfig = ..., + scopes: ScopesConfig = ..., + references: ReferencesConfig = ..., + cost: CostConfig = ..., + constants: ConstantsConfig = ..., + cursors: CursorsConfig = ..., + microbatches: MicrobatchesConfig = ..., + defaults: DefaultsConfig = ..., + diff: DiffConfig = ..., + materialization_defaults: MaterializationDefaultsConfig = ..., + path_defaults: dict[str, dict[str, object]] = ..., + vars: dict[str, str] = ..., + targets: dict[str, TargetConfig] = ..., + janitor: JanitorConfig = ..., + migrations: MigrationsConfig = ..., + snapshots: SnapshotsConfig = ..., + scenario: ScenarioConfig = ..., + dbt: DbtConfig = ..., + sinks: SinksConfig = ..., + ) -> None: ... + +class LocalConfig(NativeRecord): + """Local developer overrides from sqlbuild_local.toml.""" + + target: str | None + adapter: str | None + connection: dict[str, object] + connections: dict[str, dict[str, object]] + targets: dict[str, LocalTargetConfig] + settings: SettingsConfig + setting_overrides: frozenset[str] + vars: dict[str, str] + dbt: LocalDbtConfig + scenario: ScenarioConfig + def __init__( + self, + target: str | None = ..., + adapter: str | None = ..., + connection: dict[str, object] = ..., + connections: dict[str, dict[str, object]] = ..., + targets: dict[str, LocalTargetConfig] = ..., + settings: SettingsConfig = ..., + setting_overrides: frozenset[str] = ..., + vars: dict[str, str] = ..., + dbt: LocalDbtConfig = ..., + scenario: ScenarioConfig = ..., + ) -> None: ... + +class SourceLocation(NativeRecord): + """An authored source location for compiler diagnostics.""" + + path: Path + line: int + column: int + end_line: int | None + end_column: int | None + def __init__( + self, + path: Path, + line: int, + column: int, + end_line: int | None = ..., + end_column: int | None = ..., + ) -> None: ... + +class SeedCsvSettings(NativeRecord): + """CSV reader settings for a seed file.""" + + delimiter: str | None + quotechar: str | None + doublequote: bool | None + escapechar: str | None + skipinitialspace: bool | None + lineterminator: str | None + encoding: str | None + na_values: tuple[object, ...] | dict[str, tuple[object, ...]] | None + keep_default_na: bool | None + def __init__( + self, + delimiter: str | None = ..., + quotechar: str | None = ..., + doublequote: bool | None = ..., + escapechar: str | None = ..., + skipinitialspace: bool | None = ..., + lineterminator: str | None = ..., + encoding: str | None = ..., + na_values: tuple[object, ...] | dict[str, tuple[object, ...]] | None = ..., + keep_default_na: bool | None = ..., + ) -> None: ... + +class SchemaAuditInstance(NativeRecord): + """One audit instance attached to model, column, or seed metadata.""" + + definition_name: str + arguments: dict[str, object] + name: str | None + description: str | None + severity: AuditSeverity | None + run_scope: str | None + always_run: bool + thresholds: MeasurementThresholds | None + minimum_samples: int | None + evidence_limit: int | None + location: SourceLocation | None + def __init__( + self, + definition_name: str, + arguments: dict[str, object] = ..., + name: str | None = ..., + description: str | None = ..., + severity: AuditSeverity | None = ..., + run_scope: str | None = ..., + always_run: bool = ..., + thresholds: MeasurementThresholds | None = ..., + minimum_samples: int | None = ..., + evidence_limit: int | None = ..., + location: SourceLocation | None = ..., + ) -> None: ... + +class SchemaColumn(NativeRecord): + """One declared model, seed, or source column entry.""" + + name: str + type: str | None + nullable: bool | None + description: str | None + meta: dict[str, object] + audits: tuple[SchemaAuditInstance, ...] + location: SourceLocation | None + migrate_from: str | None + def __init__( + self, + name: str, + type: str | None = ..., + nullable: bool | None = ..., + description: str | None = ..., + meta: dict[str, object] = ..., + audits: tuple[SchemaAuditInstance, ...] = ..., + location: SourceLocation | None = ..., + migrate_from: str | None = ..., + ) -> None: ... + +class SchemaDynamicColumnFamily(NativeRecord): + """One declared family of columns generated by a runtime-dynamic pivot.""" + + name: str + pivot_column: str + value_column: str + aggregate: str + type: str + name_pattern: str | None + def __init__( + self, + name: str, + pivot_column: str, + value_column: str, + aggregate: str, + type: str, + name_pattern: str | None = ..., + ) -> None: ... + +class SchemaModelEntry(NativeRecord): + """One model metadata entry normalized from MODEL(...).""" + + name: str + model_schema: str | None + description: str | None + type_enforcement: bool | None + meta: dict[str, object] + columns: tuple[SchemaColumn, ...] + dynamic_columns: tuple[SchemaDynamicColumnFamily, ...] + audits: tuple[SchemaAuditInstance, ...] + tags: tuple[str, ...] + def __init__( + self, + name: str, + model_schema: str | None = ..., + description: str | None = ..., + type_enforcement: bool | None = ..., + meta: dict[str, object] = ..., + columns: tuple[SchemaColumn, ...] = ..., + dynamic_columns: tuple[SchemaDynamicColumnFamily, ...] = ..., + audits: tuple[SchemaAuditInstance, ...] = ..., + tags: tuple[str, ...] = ..., + ) -> None: ... + +class SchemaSeedEntry(NativeRecord): + """One seed metadata entry from seed YAML.""" + + name: str + description: str | None + database: str | None + schema: str | None + meta: dict[str, object] + csv_settings: SeedCsvSettings + columns: tuple[SchemaColumn, ...] + tags: tuple[str, ...] + audits: tuple[SchemaAuditInstance, ...] + def __init__( + self, + name: str, + description: str | None = ..., + database: str | None = ..., + schema: str | None = ..., + meta: dict[str, object] = ..., + csv_settings: SeedCsvSettings = ..., + columns: tuple[SchemaColumn, ...] = ..., + tags: tuple[str, ...] = ..., + audits: tuple[SchemaAuditInstance, ...] = ..., + ) -> None: ... + +class IntegrationLoaderConfig(NativeRecord): + """Typed configuration for one declarative integration loader.""" + + kind: str + config: object + def __init__( + self, + kind: str, + config: object, + ) -> None: ... + +class SourceFreshnessAgePolicy(NativeRecord): + """Age-based source freshness warning/error thresholds.""" + + warn_after: str | None + error_after: str | None + def __init__( + self, + warn_after: str | None = ..., + error_after: str | None = ..., + ) -> None: ... + +class SourceFreshnessConfig(NativeRecord): + """Configured source freshness observation for planning.""" + + strategy: SourceFreshnessStrategy + value_kind: SourceFreshnessValueKind | None + column: str | None + query: str | None + filter: str | None + lag_tolerance: str | None + age_policy: SourceFreshnessAgePolicy | None + def __init__( + self, + strategy: SourceFreshnessStrategy, + value_kind: SourceFreshnessValueKind | None = ..., + column: str | None = ..., + query: str | None = ..., + filter: str | None = ..., + lag_tolerance: str | None = ..., + age_policy: SourceFreshnessAgePolicy | None = ..., + ) -> None: ... + +class SourceColumnEntry(NativeRecord): + """One source column entry from sources/*.yml.""" + + name: str + type: str | None + nullable: bool | None + description: str | None + meta: dict[str, object] + audits: tuple[SchemaAuditInstance, ...] + def __init__( + self, + name: str, + type: str | None = ..., + nullable: bool | None = ..., + description: str | None = ..., + meta: dict[str, object] = ..., + audits: tuple[SchemaAuditInstance, ...] = ..., + ) -> None: ... + +class SourceEntry(NativeRecord): + """One source declaration from sources/*.yml.""" + + name: str + database: str | None + schema: str | None + table: str | None + loader: str | None + managed: bool + integration_loader: IntegrationLoaderConfig | None + freshness: SourceFreshnessConfig | None + write_strategy: SourceWriteStrategy | None + load_batch_size: int | None + cursor_column: str | None + unique_key: tuple[str, ...] + expression: str | None + description: str | None + type_enforcement: bool | None + contract: str | None + meta: dict[str, object] + columns: tuple[SourceColumnEntry, ...] + audits: tuple[SchemaAuditInstance, ...] + def __init__( + self, + name: str, + database: str | None = ..., + schema: str | None = ..., + table: str | None = ..., + loader: str | None = ..., + managed: bool = ..., + integration_loader: IntegrationLoaderConfig | None = ..., + freshness: SourceFreshnessConfig | None = ..., + write_strategy: SourceWriteStrategy | None = ..., + load_batch_size: int | None = ..., + cursor_column: str | None = ..., + unique_key: tuple[str, ...] = ..., + expression: str | None = ..., + description: str | None = ..., + type_enforcement: bool | None = ..., + contract: str | None = ..., + meta: dict[str, object] = ..., + columns: tuple[SourceColumnEntry, ...] = ..., + audits: tuple[SchemaAuditInstance, ...] = ..., + ) -> None: ... + +class LoaderDestinationParts(NativeRecord): + """Parsed warehouse relation parts for a loader destination.""" + + database: str | None + schema: str | None + table: str + def __init__( + self, + database: str | None, + schema: str | None, + table: str, + ) -> None: ... + +class MeasurementThresholdBound(NativeRecord): + """One directional threshold bound for a measurement audit.""" + + operator: ThresholdOperator + limit: float | None + lower: float | None + upper: float | None + def __init__( + self, + operator: ThresholdOperator, + limit: float | None = ..., + lower: float | None = ..., + upper: float | None = ..., + ) -> None: ... + +class MeasurementThresholds(NativeRecord): + """Warning and error policy for a measurement audit.""" + + warn: MeasurementThresholdBound | None + error: MeasurementThresholdBound | None + def __init__( + self, + warn: MeasurementThresholdBound | None = ..., + error: MeasurementThresholdBound | None = ..., + ) -> None: ... + +class MeasurementContract(NativeRecord): + """Columns and sample unit returned by a measurement query.""" + + value_column: str + sample_count_column: str | None + sample_unit: str | None + def __init__( + self, + value_column: str, + sample_count_column: str | None = ..., + sample_unit: str | None = ..., + ) -> None: ... + +class AuditIdentity(NativeRecord): + """Stable identity values for one planned audit binding.""" + + binding_key: str + audit_name: str + definition_fingerprint: str + execution_fingerprint: str + severity: AuditSeverity + run_scope_phase: str + attachment_kind: str + attached_target_name: str | None + attached_column_name: str | None + always_run: bool + def __init__( + self, + binding_key: str, + audit_name: str, + definition_fingerprint: str, + execution_fingerprint: str, + severity: AuditSeverity, + run_scope_phase: str, + attachment_kind: str, + attached_target_name: str | None = ..., + attached_column_name: str | None = ..., + always_run: bool = ..., + ) -> None: ... + +class AuditGateIdentity(NativeRecord): + """Aggregate identity for a model's runtime audit gate.""" + + binding_set_hash: str + blocking_set_hash: str + audits: tuple[AuditIdentity, ...] + def __init__( + self, + binding_set_hash: str, + blocking_set_hash: str, + audits: tuple[AuditIdentity, ...], + ) -> None: ... + +class SqlValueKind(StrEnum): + """Closed logical kinds accepted by the typed SQL value boundary.""" + + STRING = "string" + INTEGER = "integer" + BOOLEAN = "boolean" + FLOAT = "float" + DECIMAL = "decimal" + NULL = "null" + LIST = "list" + SET = "set" + OBJECT = "object" + +class StateSqlValueType(StrEnum): + """Closed physical value kinds used by warehouse state columns.""" + + STRING = "string" + INTEGER = "integer" + BOOLEAN = "boolean" + TIMESTAMP = "timestamp" + TEXT_TIMESTAMP = "text_timestamp" + DATE = "date" + JSON = "json" + +class SqlLogicalType(NativeRecord): + """Logical type inferred for a normalized SQL value.""" + + kind: SqlValueKind + element_type: SqlLogicalType | None + def __init__( + self, + kind: SqlValueKind, + element_type: SqlLogicalType | None = ..., + ) -> None: ... + @property + def display_name(self) -> str: ... + +class SqlValue(NativeRecord): + """One validated, canonical SQL value.""" + + logical_type: SqlLogicalType + value: SqlValuePayload + def __init__( + self, + logical_type: SqlLogicalType, + value: SqlValuePayload, + ) -> None: ... + @property + def kind(self) -> SqlValueKind: ... + +class Fingerprint(NativeRecord): + """One applied fingerprint record from a successful materialization.""" + + node_type: str + node_name: str + target_database: str | None + target_schema: str | None + target_name: str | None + run_id: str + definition_hash: str + schema_fingerprint: str + definition: str + ts: datetime + metadata_json: str + version_hash: str + def __init__( + self, + node_type: str, + node_name: str, + target_database: str | None, + target_schema: str | None, + target_name: str | None, + run_id: str, + definition_hash: str, + schema_fingerprint: str, + definition: str, + ts: datetime, + metadata_json: str = ..., + version_hash: str = ..., + ) -> None: ... + +class FingerprintSet(NativeRecord): + """Latest fingerprints per node name for one target schema.""" + + schema: str + fingerprints: dict[str, Fingerprint] + fingerprints_by_identity: dict[tuple[str, str], Fingerprint] | None + def __init__( + self, + schema: str, + fingerprints: dict[str, Fingerprint], + fingerprints_by_identity: dict[tuple[str, str], Fingerprint] | None = ..., + ) -> None: ... + +class SourceFreshnessRecord(NativeRecord): + """One append-only direct observation whose observed time is always aware UTC.""" + + source_name: str + target_database: str | None + target_schema: str | None + target_name: str | None + run_id: str + strategy: str + value_kind: str + data_version: str | None + data_version_hash: str + observed_at: datetime + def __init__( + self, + source_name: str, + target_database: str | None, + target_schema: str | None, + target_name: str | None, + run_id: str, + strategy: str, + value_kind: str, + data_version: str | None, + data_version_hash: str, + observed_at: datetime, + ) -> None: ... + @property + def identity(self) -> SourceFreshnessIdentity: ... + +class SelectedDbtProfileOutput(NativeRecord): + """Selected dbt profile target output before rendering.""" + + profile_name: str + target_name: str + output: dict[str, object] + def __init__( + self, + profile_name: str, + target_name: str, + output: dict[str, object], + ) -> None: ... + +class ResolvedDbtProfileOutput(NativeRecord): + """Rendered dbt profile target output.""" + + project_dir: Path + profiles_dir: Path + profile_name: str + target_name: str + output: dict[str, object] + def __init__( + self, + project_dir: Path, + profiles_dir: Path, + profile_name: str, + target_name: str, + output: dict[str, object], + ) -> None: ... + +class DiscoveredPythonNode(NativeRecord): + """Shared internal view of a discovered Python DAG node.""" + + kind: PythonNodeKind + file_path: Path + relative_path: Path + name: str + function: Callable[..., object] + depends_on: tuple[Callable[..., object] | SqlResourceRef, ...] + sql_deps: tuple[SqlResourceRef, ...] + tags: tuple[str, ...] + group: str | None + description: str | None + meta: dict[str, object] | None + provider_usages: tuple[DiscoveredProviderUsage, ...] + loader: DiscoveredPythonLoaderMetadata | None + task: DiscoveredPythonTaskMetadata | None + asset: DiscoveredPythonAssetMetadata | None + check: DiscoveredPythonCheckMetadata | None + identity: PythonNodeIdentity | None + def __init__( + self, + kind: PythonNodeKind, + file_path: Path, + relative_path: Path, + name: str, + function: Callable[..., object], + depends_on: tuple[Callable[..., object] | SqlResourceRef, ...] = ..., + sql_deps: tuple[SqlResourceRef, ...] = ..., + tags: tuple[str, ...] = ..., + group: str | None = ..., + description: str | None = ..., + meta: dict[str, object] | None = ..., + provider_usages: tuple[DiscoveredProviderUsage, ...] = ..., + loader: DiscoveredPythonLoaderMetadata | None = ..., + task: DiscoveredPythonTaskMetadata | None = ..., + asset: DiscoveredPythonAssetMetadata | None = ..., + check: DiscoveredPythonCheckMetadata | None = ..., + identity: PythonNodeIdentity | None = ..., + ) -> None: ... + +class PythonNodeGraph(NativeRecord): + """Internal graph inventory for discovered executable Python nodes.""" + + nodes: tuple[DiscoveredPythonNode, ...] + dependency_edges: tuple[PythonNodeDependencyEdge, ...] + upstream_deps: dict[str, tuple[str, ...]] + downstream_deps: dict[str, tuple[str, ...]] + tag_index: dict[str, frozenset[str]] + path_index: dict[str, str] + nodes_by_name: dict[str, DiscoveredPythonNode] + nodes_by_typed_selector: dict[str, DiscoveredPythonNode] + def __init__( + self, + nodes: tuple[DiscoveredPythonNode, ...], + dependency_edges: tuple[PythonNodeDependencyEdge, ...], + upstream_deps: dict[str, tuple[str, ...]], + downstream_deps: dict[str, tuple[str, ...]], + tag_index: dict[str, frozenset[str]], + path_index: dict[str, str], + nodes_by_name: dict[str, DiscoveredPythonNode], + nodes_by_typed_selector: dict[str, DiscoveredPythonNode], + ) -> None: ... + def selected_sql_refs(self, *, selected_names: frozenset[str]) -> frozenset[SqlResourceRef]: ... + +class PythonSqlRunLifecyclePlan(NativeRecord): + """Lifecycle-aware run classification for SQL resources and Python nodes.""" + + ingress_python_node_names: frozenset[str] + ingress_loader_names: frozenset[str] + read_side_python_node_names: frozenset[str] + def __init__( + self, + ingress_python_node_names: frozenset[str], + ingress_loader_names: frozenset[str], + read_side_python_node_names: frozenset[str], + ) -> None: ... + +class PythonIdentityDependency(NativeRecord): + """One first-party Python source object included in a Python node identity.""" + + kind: str + module: str + qualname: str + source_path: str + source_hash: str + source_text: str + def __init__( + self, + kind: str, + module: str, + qualname: str, + source_path: str, + source_hash: str, + source_text: str, + ) -> None: ... + +class PythonNodeIdentity(NativeRecord): + """Read-only identity metadata for a SQLBuild Python node.""" + + node_type: str + node_name: str + source_path: str + source_hash: str + definition_hash: str + version_hash: str + dependencies: tuple[PythonIdentityDependency, ...] + definition_json: str + metadata_json: str + def __init__( + self, + node_type: str, + node_name: str, + source_path: str, + source_hash: str, + definition_hash: str, + version_hash: str, + dependencies: tuple[PythonIdentityDependency, ...] = ..., + definition_json: str = ..., + metadata_json: str = ..., + ) -> None: ... + +class SqlLexicalSyntax(NativeRecord): + """Dialect rules for where SQL quoted text and comments end.""" + + backslash_escape_quotes: frozenset[str] + escape_string_prefix: bool + raw_string_prefix: bool + triple_quoted_strings: bool + nested_block_comments: bool + line_comment_prefixes: frozenset[str] + def __init__( + self, + backslash_escape_quotes: frozenset[str] = ..., + escape_string_prefix: bool = ..., + raw_string_prefix: bool = ..., + triple_quoted_strings: bool = ..., + nested_block_comments: bool = ..., + line_comment_prefixes: frozenset[str] = ..., + ) -> None: ... + @property + def cache_key(self) -> str: ... + @property + def native_mapping(self) -> dict[str, object]: ... + def reads_differently_from_generic(self, sql: str) -> bool: ... + +class TimestampValue(NativeRecord): + """Timestamp cursor value; aware datetimes are normalized to UTC at ingestion.""" + + value: datetime + source_text: str | None + def __init__( + self, + value: datetime, + source_text: str | None = ..., + ) -> None: ... + +class DateValue(NativeRecord): + """Date cursor value which must remain date-formatted when rendered.""" + + value: date + def __init__( + self, + value: date, + ) -> None: ... + +class IntegerValue(NativeRecord): + """Integral cursor value.""" + + value: int + def __init__( + self, + value: int, + ) -> None: ... + +class AlignedInterval(NativeRecord): + """Validated half-open cursor interval.""" + + start: TimestampValue | DateValue | IntegerValue + end: TimestampValue | DateValue | IntegerValue + grain: CursorGrain | None + def __init__( + self, + start: TimestampValue | DateValue | IntegerValue, + end: TimestampValue | DateValue | IntegerValue, + grain: CursorGrain | None, + ) -> None: ... + +class Duration(NativeRecord): + """A calendar-aware duration (years/months plus fixed time units).""" + + years: int + months: int + days: int + hours: int + minutes: int + seconds: int + def __init__( + self, + years: int = ..., + months: int = ..., + days: int = ..., + hours: int = ..., + minutes: int = ..., + seconds: int = ..., + ) -> None: ... + @classmethod + def parse(cls, value: str) -> Duration | None: ... + @property + def is_zero(self) -> bool: ... + @property + def units(self) -> frozenset[str]: ... + def is_single_unit_in(self, allowed_units: frozenset[str]) -> bool: ... + @property + def has_calendar_component(self) -> bool: ... + @property + def fixed_seconds(self) -> int: ... + @property + def total_months(self) -> int: ... + def add_to(self, moment: datetime) -> datetime: ... + def subtract_from(self, moment: datetime) -> datetime: ... diff --git a/src/sqlbuild/adapter/contract/_helpers/seed_csv.py b/src/sqlbuild/adapter/contract/_helpers/seed_csv.py index ad71e9899b..84daf70c80 100644 --- a/src/sqlbuild/adapter/contract/_helpers/seed_csv.py +++ b/src/sqlbuild/adapter/contract/_helpers/seed_csv.py @@ -2,7 +2,7 @@ from __future__ import annotations -from sqlbuild.spec.contracts.models import SeedCsvSettings +from sqlbuild._native import SeedCsvSettings _DEFAULT_NULL_VALUE: str = "" diff --git a/src/sqlbuild/adapter/contract/classes/base_adapter.py b/src/sqlbuild/adapter/contract/classes/base_adapter.py index cb33a480c7..ad513a0bc5 100644 --- a/src/sqlbuild/adapter/contract/classes/base_adapter.py +++ b/src/sqlbuild/adapter/contract/classes/base_adapter.py @@ -9,6 +9,13 @@ from pathlib import Path from typing import Any, ClassVar, cast +from sqlbuild._native import ( + SeedCsvSettings, + SourceFreshnessRecord, + SqlLexicalSyntax, + SqlValue, + SqlValueKind, +) from sqlbuild.adapter.contract.classes.historical_check_snapshot_sql import ( HistoricalCheckSnapshotSql, ) @@ -77,13 +84,8 @@ render_insert_source_freshness_records_sql, ) from sqlbuild.compiler.compile.types import FunctionLanguage -from sqlbuild.compiler.source_freshness.models import SourceFreshnessRecord -from sqlbuild.compiler.sql_analysis.models import SqlLexicalSyntax from sqlbuild.spec.contracts.constants import DEFAULT_SEED_CSV_SETTINGS -from sqlbuild.spec.contracts.models import SeedCsvSettings from sqlbuild.sql_values.exceptions import SqlValueRenderingError -from sqlbuild.sql_values.models import SqlValue -from sqlbuild.sql_values.types import SqlValueKind _SCALAR_SQL_VALUE_KINDS: frozenset[SqlValueKind] = frozenset( { diff --git a/src/sqlbuild/adapter/contract/classes/duckdb_backed_adapter.py b/src/sqlbuild/adapter/contract/classes/duckdb_backed_adapter.py index ec8ecff6f5..447c10131c 100644 --- a/src/sqlbuild/adapter/contract/classes/duckdb_backed_adapter.py +++ b/src/sqlbuild/adapter/contract/classes/duckdb_backed_adapter.py @@ -12,6 +12,7 @@ from types import ModuleType from typing import Any, ClassVar +from sqlbuild._native import SeedCsvSettings, SourceFreshnessRecord, SqlLexicalSyntax, SqlValue from sqlbuild.adapter.contract._helpers.seed_csv import get_seed_csv_null_values from sqlbuild.adapter.contract.classes.base_adapter import ( BaseAdapter, @@ -85,12 +86,8 @@ from sqlbuild.adapter.type_system.main.normalize_numeric_family import normalize_numeric_family from sqlbuild.adapter.type_system.main.types_equal import types_equal from sqlbuild.compiler.compile.types import FunctionLanguage -from sqlbuild.compiler.source_freshness.models import SourceFreshnessRecord -from sqlbuild.compiler.sql_analysis.models import SqlLexicalSyntax from sqlbuild.diagnostics.main.log_sql import log_sql from sqlbuild.spec.contracts.constants import DEFAULT_SEED_CSV_SETTINGS -from sqlbuild.spec.contracts.models import SeedCsvSettings -from sqlbuild.sql_values.models import SqlValue class DuckDbBackedAdapter(UnkeyedDiffMixin, BaseAdapter): diff --git a/src/sqlbuild/adapter/contract/classes/materialization.py b/src/sqlbuild/adapter/contract/classes/materialization.py index bc499048cc..6d86984026 100644 --- a/src/sqlbuild/adapter/contract/classes/materialization.py +++ b/src/sqlbuild/adapter/contract/classes/materialization.py @@ -6,10 +6,10 @@ from pathlib import Path from typing import Any +from sqlbuild._native import SeedCsvSettings from sqlbuild.adapter.contract.classes.statement_recorder import StatementRecorder from sqlbuild.adapter.contract.models import ColumnInfo, FunctionDefinition from sqlbuild.spec.contracts.constants import DEFAULT_SEED_CSV_SETTINGS -from sqlbuild.spec.contracts.models import SeedCsvSettings class MaterializationMixin(ABC): diff --git a/src/sqlbuild/adapter/contract/classes/strict_adapter.py b/src/sqlbuild/adapter/contract/classes/strict_adapter.py index 4ce382bf8b..7b676a97bb 100644 --- a/src/sqlbuild/adapter/contract/classes/strict_adapter.py +++ b/src/sqlbuild/adapter/contract/classes/strict_adapter.py @@ -6,6 +6,7 @@ from decimal import Decimal from typing import Any, ClassVar +from sqlbuild._native import SourceFreshnessRecord, SqlValue from sqlbuild.adapter.contract.classes.connection import ConnectionMixin from sqlbuild.adapter.contract.classes.diff import DiffMixin from sqlbuild.adapter.contract.classes.materialization import MaterializationMixin @@ -32,8 +33,6 @@ TablePromotionMode, ) from sqlbuild.compiler.compile.types import FunctionLanguage -from sqlbuild.compiler.source_freshness.models import SourceFreshnessRecord -from sqlbuild.sql_values.models import SqlValue class StrictAdapter( diff --git a/src/sqlbuild/adapter/contract/main/normalize_seed_csv_value.py b/src/sqlbuild/adapter/contract/main/normalize_seed_csv_value.py index f93382334c..09abd3c23d 100644 --- a/src/sqlbuild/adapter/contract/main/normalize_seed_csv_value.py +++ b/src/sqlbuild/adapter/contract/main/normalize_seed_csv_value.py @@ -2,8 +2,8 @@ from __future__ import annotations +from sqlbuild._native import SeedCsvSettings from sqlbuild.adapter.contract._helpers.seed_csv import normalize_seed_csv_value as _normalize -from sqlbuild.spec.contracts.models import SeedCsvSettings def normalize_seed_csv_value( diff --git a/src/sqlbuild/adapter/state_sql/_helpers/source_freshness.py b/src/sqlbuild/adapter/state_sql/_helpers/source_freshness.py index 7edd187e8a..904604cbe9 100644 --- a/src/sqlbuild/adapter/state_sql/_helpers/source_freshness.py +++ b/src/sqlbuild/adapter/state_sql/_helpers/source_freshness.py @@ -4,7 +4,7 @@ from collections.abc import Callable -from sqlbuild._native import render_state_sql_literal +from sqlbuild._native import SourceFreshnessRecord, render_state_sql_literal from sqlbuild.adapter.contract.exceptions import AdapterUserError from sqlbuild.compiler.source_freshness.constants import ( COLUMN_DATA_VERSION, @@ -21,7 +21,6 @@ SOURCE_FRESHNESS_COLUMNS, SOURCE_FRESHNESS_TABLE_NAME, ) -from sqlbuild.compiler.source_freshness.models import SourceFreshnessRecord def render_insert_source_freshness_records_sql( diff --git a/src/sqlbuild/adapter/state_sql/_helpers/table_ddl.py b/src/sqlbuild/adapter/state_sql/_helpers/table_ddl.py index 5312637c8c..86fb8e7095 100644 --- a/src/sqlbuild/adapter/state_sql/_helpers/table_ddl.py +++ b/src/sqlbuild/adapter/state_sql/_helpers/table_ddl.py @@ -4,8 +4,8 @@ from collections.abc import Callable, Mapping +from sqlbuild._native import StateSqlValueType from sqlbuild.adapter.contract.types import FrameworkType -from sqlbuild.sql_values.types import StateSqlValueType def render_state_table_create_sql_impl( diff --git a/src/sqlbuild/adapter/state_sql/main/render_insert_source_freshness_records_sql.py b/src/sqlbuild/adapter/state_sql/main/render_insert_source_freshness_records_sql.py index 3434747e8f..33a2db2e93 100644 --- a/src/sqlbuild/adapter/state_sql/main/render_insert_source_freshness_records_sql.py +++ b/src/sqlbuild/adapter/state_sql/main/render_insert_source_freshness_records_sql.py @@ -2,10 +2,10 @@ from collections.abc import Callable +from sqlbuild._native import SourceFreshnessRecord from sqlbuild.adapter.state_sql._helpers.source_freshness import ( render_insert_source_freshness_records_sql as _render_sql, ) -from sqlbuild.compiler.source_freshness.models import SourceFreshnessRecord def render_insert_source_freshness_records_sql( diff --git a/src/sqlbuild/adapter/state_sql/main/render_state_table_create_sql.py b/src/sqlbuild/adapter/state_sql/main/render_state_table_create_sql.py index e0fe88aac6..4b9c1fe1ff 100644 --- a/src/sqlbuild/adapter/state_sql/main/render_state_table_create_sql.py +++ b/src/sqlbuild/adapter/state_sql/main/render_state_table_create_sql.py @@ -2,9 +2,9 @@ from collections.abc import Callable, Mapping +from sqlbuild._native import StateSqlValueType from sqlbuild.adapter.contract.types import FrameworkType from sqlbuild.adapter.state_sql._helpers.table_ddl import render_state_table_create_sql_impl -from sqlbuild.sql_values.types import StateSqlValueType def render_state_table_create_sql( diff --git a/src/sqlbuild/adapters/bigquery/classes/bigquery_adapter.py b/src/sqlbuild/adapters/bigquery/classes/bigquery_adapter.py index 4601cc88e8..32ed39021c 100644 --- a/src/sqlbuild/adapters/bigquery/classes/bigquery_adapter.py +++ b/src/sqlbuild/adapters/bigquery/classes/bigquery_adapter.py @@ -10,6 +10,13 @@ from pathlib import Path from typing import Any, ClassVar, cast +from sqlbuild._native import ( + SeedCsvSettings, + SourceFreshnessRecord, + SqlLexicalSyntax, + SqlValue, + SqlValueKind, +) from sqlbuild.adapter.contract.classes.base_adapter import ( _COLLECTION_SQL_VALUE_KINDS, BaseAdapter, @@ -135,14 +142,9 @@ TIMESTAMP_TYPE_TOKEN, ) from sqlbuild.compiler.compile.types import FunctionLanguage -from sqlbuild.compiler.source_freshness.models import SourceFreshnessRecord -from sqlbuild.compiler.sql_analysis.models import SqlLexicalSyntax from sqlbuild.diagnostics.main.log_sql import log_sql from sqlbuild.runtime.observability.classes.statement_lifecycle import StatementLifecycle from sqlbuild.spec.contracts.constants import DEFAULT_SEED_CSV_SETTINGS -from sqlbuild.spec.contracts.models import SeedCsvSettings -from sqlbuild.sql_values.models import SqlValue -from sqlbuild.sql_values.types import SqlValueKind class BigQueryAdapter(MicrobatchMixin, UnkeyedDiffMixin, BaseAdapter): diff --git a/src/sqlbuild/adapters/databricks/classes/databricks_adapter.py b/src/sqlbuild/adapters/databricks/classes/databricks_adapter.py index a15d065224..68ead70ade 100644 --- a/src/sqlbuild/adapters/databricks/classes/databricks_adapter.py +++ b/src/sqlbuild/adapters/databricks/classes/databricks_adapter.py @@ -13,6 +13,7 @@ from pathlib import Path from typing import Any, ClassVar +from sqlbuild._native import SeedCsvSettings, SourceFreshnessRecord, SqlLexicalSyntax, SqlValue from sqlbuild.adapter.contract.classes.base_adapter import ( BaseAdapter, _encode_typed_json, @@ -126,12 +127,8 @@ UNDEFINED_TABLE_SQLSTATE, ) from sqlbuild.compiler.compile.types import FunctionLanguage -from sqlbuild.compiler.source_freshness.models import SourceFreshnessRecord -from sqlbuild.compiler.sql_analysis.models import SqlLexicalSyntax from sqlbuild.diagnostics.main.log_sql import log_sql from sqlbuild.spec.contracts.constants import DEFAULT_SEED_CSV_SETTINGS -from sqlbuild.spec.contracts.models import SeedCsvSettings -from sqlbuild.sql_values.models import SqlValue class DatabricksAdapter(MicrobatchMixin, UnkeyedDiffMixin, BaseAdapter): diff --git a/src/sqlbuild/adapters/postgres/classes/postgres_adapter.py b/src/sqlbuild/adapters/postgres/classes/postgres_adapter.py index 5cd919f515..740c59410c 100644 --- a/src/sqlbuild/adapters/postgres/classes/postgres_adapter.py +++ b/src/sqlbuild/adapters/postgres/classes/postgres_adapter.py @@ -10,6 +10,7 @@ from pathlib import Path from typing import Any, ClassVar +from sqlbuild._native import SeedCsvSettings, SourceFreshnessRecord, SqlLexicalSyntax, SqlValue from sqlbuild.adapter.contract.classes.base_adapter import ( BaseAdapter, _build_names_filter, @@ -104,12 +105,8 @@ TABLE_FUNCTION_RETURN_TYPE, ) from sqlbuild.compiler.compile.types import FunctionLanguage -from sqlbuild.compiler.source_freshness.models import SourceFreshnessRecord -from sqlbuild.compiler.sql_analysis.models import SqlLexicalSyntax from sqlbuild.diagnostics.main.log_sql import log_sql from sqlbuild.spec.contracts.constants import DEFAULT_SEED_CSV_SETTINGS -from sqlbuild.spec.contracts.models import SeedCsvSettings -from sqlbuild.sql_values.models import SqlValue class PostgresAdapter(MicrobatchMixin, UnkeyedDiffMixin, BaseAdapter): diff --git a/src/sqlbuild/adapters/snowflake/classes/snowflake_adapter.py b/src/sqlbuild/adapters/snowflake/classes/snowflake_adapter.py index c20afa3e80..2b6e20dad3 100644 --- a/src/sqlbuild/adapters/snowflake/classes/snowflake_adapter.py +++ b/src/sqlbuild/adapters/snowflake/classes/snowflake_adapter.py @@ -12,7 +12,14 @@ from pathlib import Path from typing import Any, ClassVar, cast -from sqlbuild._native import collect_snowflake_cost +from sqlbuild._native import ( + SeedCsvSettings, + SourceFreshnessRecord, + SqlLexicalSyntax, + SqlValue, + TableType, + collect_snowflake_cost, +) from sqlbuild.adapter.contract.classes.base_adapter import ( BaseAdapter, _encode_typed_json, @@ -151,16 +158,11 @@ VIEW_RELATION_TYPE_TOKEN, ) from sqlbuild.compiler.compile.types import FunctionLanguage -from sqlbuild.compiler.source_freshness.models import SourceFreshnessRecord -from sqlbuild.compiler.sql_analysis.models import SqlLexicalSyntax from sqlbuild.cost.models import RunCostSummary from sqlbuild.cost.types import CostCapability, CostStatus from sqlbuild.diagnostics.main.log_sql import log_sql from sqlbuild.runtime.observability.classes.statement_lifecycle import StatementLifecycle from sqlbuild.spec.contracts.constants import DEFAULT_SEED_CSV_SETTINGS -from sqlbuild.spec.contracts.models import SeedCsvSettings -from sqlbuild.spec.contracts.types import TableType -from sqlbuild.sql_values.models import SqlValue _EXACT_COLUMN_INSPECTION_LIMIT: int = 32 _STATE_TABLE_RETENTION_ATTRIBUTE: str = "_state_table_retention_fallbacks_state" diff --git a/src/sqlbuild/adapters/sqlserver/classes/sqlserver_adapter.py b/src/sqlbuild/adapters/sqlserver/classes/sqlserver_adapter.py index ccfda8d0b2..a6c30bf6e4 100644 --- a/src/sqlbuild/adapters/sqlserver/classes/sqlserver_adapter.py +++ b/src/sqlbuild/adapters/sqlserver/classes/sqlserver_adapter.py @@ -10,6 +10,13 @@ from pathlib import Path from typing import Any, ClassVar +from sqlbuild._native import ( + SeedCsvSettings, + SourceFreshnessRecord, + SqlLexicalSyntax, + SqlValue, + SqlValueKind, +) from sqlbuild.adapter.contract.classes.base_adapter import ( BaseAdapter, _build_names_filter, @@ -109,13 +116,8 @@ USER_TABLE_OBJECT_TYPE, ) from sqlbuild.compiler.compile.types import FunctionLanguage -from sqlbuild.compiler.source_freshness.models import SourceFreshnessRecord -from sqlbuild.compiler.sql_analysis.models import SqlLexicalSyntax from sqlbuild.diagnostics.main.log_sql import log_sql from sqlbuild.spec.contracts.constants import DEFAULT_SEED_CSV_SETTINGS -from sqlbuild.spec.contracts.models import SeedCsvSettings -from sqlbuild.sql_values.models import SqlValue -from sqlbuild.sql_values.types import SqlValueKind def _render_tsql_distinct_condition(*, left: str, right: str) -> str: diff --git a/src/sqlbuild/assets.py b/src/sqlbuild/assets.py index 49143893ab..296ae1f5d2 100644 --- a/src/sqlbuild/assets.py +++ b/src/sqlbuild/assets.py @@ -4,6 +4,7 @@ from collections.abc import Callable, Mapping, Sequence +from sqlbuild._native import SourceColumnEntry from sqlbuild.compiler.python_nodes.types import SkipMode from sqlbuild.executor.python_nodes.models import AssetContext from sqlbuild.python_nodes.main.apply_asset import apply_asset @@ -15,7 +16,6 @@ SqlResourceRef, ) from sqlbuild.python_nodes.types import ColumnLineageRefSpec, PythonNodeColumnSpec -from sqlbuild.spec.contracts.models import SourceColumnEntry __all__ = ("AssetContext", "SkipMode", "asset", "get_asset_definition") diff --git a/src/sqlbuild/assets.pyi b/src/sqlbuild/assets.pyi index 5740b14106..ae882c433e 100644 --- a/src/sqlbuild/assets.pyi +++ b/src/sqlbuild/assets.pyi @@ -1,6 +1,7 @@ from collections.abc import Callable, Mapping, Sequence from typing import overload +from sqlbuild._native import SourceColumnEntry from sqlbuild.compiler.python_nodes.types import SkipMode as SkipMode from sqlbuild.executor.python_nodes.models import AssetContext as AssetContext from sqlbuild.python_nodes.models import ( @@ -10,7 +11,6 @@ from sqlbuild.python_nodes.models import ( SqlResourceRef, ) from sqlbuild.python_nodes.types import ColumnLineageRefSpec, PythonNodeColumnSpec -from sqlbuild.spec.contracts.models import SourceColumnEntry __all__ = ("AssetContext", "SkipMode", "asset", "get_asset_definition") diff --git a/src/sqlbuild/audits.py b/src/sqlbuild/audits.py index 800380b82a..a686467180 100644 --- a/src/sqlbuild/audits.py +++ b/src/sqlbuild/audits.py @@ -1,5 +1,7 @@ """Public authoring API for generated SQL audit attachments.""" +from sqlbuild._native import MeasurementThresholdBound as MeasurementThresholdBound +from sqlbuild._native import MeasurementThresholds as MeasurementThresholds from sqlbuild.compiler.auditing.main.above import above as above from sqlbuild.compiler.auditing.main.audit_factory import audit_factory as audit_factory from sqlbuild.compiler.auditing.main.below import below as below @@ -7,12 +9,6 @@ get_audit_factory_definition as get_audit_factory_definition, ) from sqlbuild.compiler.auditing.main.outside import outside as outside -from sqlbuild.compiler.auditing.models import ( - MeasurementThresholdBound as MeasurementThresholdBound, -) -from sqlbuild.compiler.auditing.models import ( - MeasurementThresholds as MeasurementThresholds, -) from sqlbuild.compiler.auditing.types import ( AuditSeverity as AuditSeverity, ) diff --git a/src/sqlbuild/cli/commands/_helpers/freshness/source_freshness.py b/src/sqlbuild/cli/commands/_helpers/freshness/source_freshness.py index 53387605c0..0db7e631c1 100644 --- a/src/sqlbuild/cli/commands/_helpers/freshness/source_freshness.py +++ b/src/sqlbuild/cli/commands/_helpers/freshness/source_freshness.py @@ -9,12 +9,12 @@ CostContext, DirectSourceFreshnessPlanningResult, SourceFreshnessIdentity, + SourceFreshnessRecord, SourceFreshnessRenderers, write_source_freshness_records, ) from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter from sqlbuild.compiler.planner.models import ModelPlanEntry, PlanOutput -from sqlbuild.compiler.source_freshness.models import SourceFreshnessRecord from sqlbuild.executor.build.models import BuildExecutionResult from sqlbuild.executor.scheduling.types import ExecutionStatus from sqlbuild.runtime.observability.classes.operation_lifecycle import OperationLifecycle diff --git a/src/sqlbuild/compiler/auditing/main/above.py b/src/sqlbuild/compiler/auditing/main/above.py index ede46717e8..544e4516fc 100644 --- a/src/sqlbuild/compiler/auditing/main/above.py +++ b/src/sqlbuild/compiler/auditing/main/above.py @@ -1,6 +1,6 @@ """Build an above-threshold audit bound.""" -from sqlbuild.compiler.auditing.models import MeasurementThresholdBound +from sqlbuild._native import MeasurementThresholdBound from sqlbuild.compiler.auditing.types import ThresholdOperator diff --git a/src/sqlbuild/compiler/auditing/main/below.py b/src/sqlbuild/compiler/auditing/main/below.py index 0e891b0e96..c38ebe6402 100644 --- a/src/sqlbuild/compiler/auditing/main/below.py +++ b/src/sqlbuild/compiler/auditing/main/below.py @@ -1,6 +1,6 @@ """Build a below-threshold audit bound.""" -from sqlbuild.compiler.auditing.models import MeasurementThresholdBound +from sqlbuild._native import MeasurementThresholdBound from sqlbuild.compiler.auditing.types import ThresholdOperator diff --git a/src/sqlbuild/compiler/auditing/main/outside.py b/src/sqlbuild/compiler/auditing/main/outside.py index 3f31c2cfdc..4d94ba1938 100644 --- a/src/sqlbuild/compiler/auditing/main/outside.py +++ b/src/sqlbuild/compiler/auditing/main/outside.py @@ -1,6 +1,6 @@ """Build an outside-range audit bound.""" -from sqlbuild.compiler.auditing.models import MeasurementThresholdBound +from sqlbuild._native import MeasurementThresholdBound from sqlbuild.compiler.auditing.types import ThresholdOperator diff --git a/src/sqlbuild/compiler/auditing/models.py b/src/sqlbuild/compiler/auditing/models.py deleted file mode 100644 index ef187e02d3..0000000000 --- a/src/sqlbuild/compiler/auditing/models.py +++ /dev/null @@ -1,122 +0,0 @@ -"""Auditing identity models.""" - -from __future__ import annotations - -from dataclasses import dataclass -from math import isfinite -from typing import cast - -from sqlbuild._native import MeasurementAuditError -from sqlbuild.compiler.auditing.types import AuditSeverity, ThresholdOperator - - -@dataclass(frozen=True, kw_only=True) -class MeasurementThresholdBound: - """One directional threshold bound for a measurement audit.""" - - operator: ThresholdOperator - limit: float | None = None - lower: float | None = None - upper: float | None = None - - def __post_init__(self) -> None: - if self.operator in {ThresholdOperator.BELOW, ThresholdOperator.ABOVE}: - if self.limit is None or self.lower is not None or self.upper is not None: - raise MeasurementAuditError(f"{self.operator.value} threshold requires limit only") - if isinstance(self.limit, bool): - raise MeasurementAuditError("measurement threshold limit must not be boolean") - object.__setattr__(self, "limit", float(self.limit)) - if not isfinite(self.limit): - raise MeasurementAuditError("measurement threshold limit must be finite") - return - - if self.limit is not None or self.lower is None or self.upper is None: - raise MeasurementAuditError("outside threshold requires lower and upper only") - if isinstance(self.lower, bool): - raise MeasurementAuditError("measurement threshold lower must not be boolean") - if isinstance(self.upper, bool): - raise MeasurementAuditError("measurement threshold upper must not be boolean") - object.__setattr__(self, "lower", float(self.lower)) - object.__setattr__(self, "upper", float(self.upper)) - if not isfinite(self.lower): - raise MeasurementAuditError("measurement threshold lower must be finite") - if not isfinite(self.upper): - raise MeasurementAuditError("measurement threshold upper must be finite") - if self.lower > self.upper: - raise MeasurementAuditError( - "outside threshold lower must be less than or equal to upper" - ) - - -@dataclass(frozen=True, kw_only=True) -class MeasurementThresholds: - """Warning and error policy for a measurement audit.""" - - warn: MeasurementThresholdBound | None = None - error: MeasurementThresholdBound | None = None - - def __post_init__(self) -> None: - if self.warn is None and self.error is None: - raise MeasurementAuditError("at least one measurement threshold is required") - if self.warn is None or self.error is None: - return - if self.warn.operator != self.error.operator: - raise MeasurementAuditError( - "mixed measurement threshold operators are unsupported in v1" - ) - if self.warn.operator == ThresholdOperator.BELOW: - if cast(float, self.error.limit) >= cast(float, self.warn.limit): - raise MeasurementAuditError("below error limit must be less than warn limit") - return - if self.warn.operator == ThresholdOperator.ABOVE: - if cast(float, self.error.limit) <= cast(float, self.warn.limit): - raise MeasurementAuditError("above error limit must be greater than warn limit") - return - if cast(float, self.error.lower) >= cast(float, self.warn.lower) or cast( - float, self.error.upper - ) <= cast(float, self.warn.upper): - raise MeasurementAuditError("outside error range must strictly contain warn range") - - -@dataclass(frozen=True, kw_only=True) -class MeasurementContract: - """Columns and sample unit returned by a measurement query.""" - - value_column: str - sample_count_column: str | None = None - sample_unit: str | None = None - - def __post_init__(self) -> None: - if not self.value_column.strip(): - raise MeasurementAuditError("measurement contract value_column must be non-empty") - if self.sample_count_column is not None and not self.sample_count_column.strip(): - raise MeasurementAuditError( - "measurement contract sample_count_column must be non-empty" - ) - if self.sample_unit is not None and not self.sample_unit.strip(): - raise MeasurementAuditError("measurement contract sample_unit must be non-empty") - - -@dataclass(frozen=True) -class AuditIdentity: - """Stable identity values for one planned audit binding.""" - - binding_key: str - audit_name: str - definition_fingerprint: str - execution_fingerprint: str - severity: AuditSeverity - run_scope_phase: str - attachment_kind: str - attached_target_name: str | None = None - attached_column_name: str | None = None - always_run: bool = False - - -@dataclass(frozen=True) -class AuditGateIdentity: - """Aggregate identity for a model's runtime audit gate.""" - - binding_set_hash: str - blocking_set_hash: str - audits: tuple[AuditIdentity, ...] diff --git a/src/sqlbuild/compiler/compile/_helpers/diagnostics/python_resources.py b/src/sqlbuild/compiler/compile/_helpers/diagnostics/python_resources.py index 778eaaefb4..121f5334a9 100644 --- a/src/sqlbuild/compiler/compile/_helpers/diagnostics/python_resources.py +++ b/src/sqlbuild/compiler/compile/_helpers/diagnostics/python_resources.py @@ -7,10 +7,9 @@ from functools import partial from pathlib import Path -from sqlbuild._native import snippet_help +from sqlbuild._native import SourceLocation, snippet_help from sqlbuild.compiler.compile.types import PythonResourceRow, SourceInput from sqlbuild.compiler.discovery.models import DiscoveredProjectInputs, DiscoveredProvider -from sqlbuild.spec.contracts.models import SourceLocation def python_resource_rows( diff --git a/src/sqlbuild/compiler/compile/_helpers/macro_bridge/call_store.py b/src/sqlbuild/compiler/compile/_helpers/macro_bridge/call_store.py index 48af17b0f7..e62351fb10 100644 --- a/src/sqlbuild/compiler/compile/_helpers/macro_bridge/call_store.py +++ b/src/sqlbuild/compiler/compile/_helpers/macro_bridge/call_store.py @@ -5,13 +5,13 @@ from pathlib import Path from sqlbuild._native import ( + TargetConfig, build_compile_target_context, ) from sqlbuild.compiler.discovery.models import DiscoveredProjectInputs from sqlbuild.compiler.macro_bridge.classes.macro_bridge import MacroBridge from sqlbuild.compiler.macro_bridge.main._claim_first_compile import claim_first_compile from sqlbuild.compiler.macro_bridge.main.active_macro_bridge import active_macro_bridge -from sqlbuild.spec.contracts.models import TargetConfig def target_context_with_macro_call_store( diff --git a/src/sqlbuild/compiler/compile/_helpers/render/user_macros.py b/src/sqlbuild/compiler/compile/_helpers/render/user_macros.py index dcc9979e2e..2c325615f7 100644 --- a/src/sqlbuild/compiler/compile/_helpers/render/user_macros.py +++ b/src/sqlbuild/compiler/compile/_helpers/render/user_macros.py @@ -18,6 +18,8 @@ DeclarationIdentity, DeclarationKind, DeclarationRecord, + SqlValue, + SqlValueKind, validate_resource_identity, ) from sqlbuild.compiler.compile.exceptions import ( @@ -37,8 +39,6 @@ DiscoveredMacroFile, EnumDeclaration, ) -from sqlbuild.sql_values.models import SqlValue -from sqlbuild.sql_values.types import SqlValueKind _OBJECT_ENTRY_LENGTH: int = 2 _MACRO_MODULE_LOAD_LOCK: threading.RLock = threading.RLock() diff --git a/src/sqlbuild/compiler/compile/exceptions.py b/src/sqlbuild/compiler/compile/exceptions.py index d038ba2808..3d4caa356b 100644 --- a/src/sqlbuild/compiler/compile/exceptions.py +++ b/src/sqlbuild/compiler/compile/exceptions.py @@ -4,7 +4,7 @@ from pathlib import Path -from sqlbuild.spec.contracts.models import SourceLocation +from sqlbuild._native import SourceLocation class CompileInputError(ValueError): diff --git a/src/sqlbuild/compiler/compile/models.py b/src/sqlbuild/compiler/compile/models.py index fd68f9c4aa..46b9adeb17 100644 --- a/src/sqlbuild/compiler/compile/models.py +++ b/src/sqlbuild/compiler/compile/models.py @@ -34,14 +34,22 @@ DiscoveredSqlTestBlock, DynamicColumnContractProof, LiteralSqlRelation, + MeasurementContract, + MeasurementThresholds, ModelSchemaDeclaration, RelatedLocation, ResourceIdentity, + ScenarioConfig, + SchemaModelEntry, ScopeIndex, + SettingsConfig, + SourceLocation, + SqlLexicalSyntax, + SqlValue, + TargetConfig, UsageRecord, VisibilityRecord, ) -from sqlbuild.compiler.auditing.models import MeasurementContract, MeasurementThresholds from sqlbuild.compiler.auditing.types import AuditEvaluationMode, AuditSeverity from sqlbuild.compiler.compile.constants import DEFAULT_SQL_TEST_MODE, MACRO_CONTEXT_PARAMETER_NAME from sqlbuild.compiler.compile.exceptions import CompileInputError @@ -70,15 +78,6 @@ ) from sqlbuild.compiler.lineage.types import InferredNullability from sqlbuild.compiler.references.types import ExternalSqlReferenceResolver, SqlReferenceKind -from sqlbuild.compiler.sql_analysis.models import SqlLexicalSyntax -from sqlbuild.spec.contracts.models import ( - ScenarioConfig, - SchemaModelEntry, - SettingsConfig, - SourceLocation, - TargetConfig, -) -from sqlbuild.sql_values.models import SqlValue @dataclass(frozen=True) diff --git a/src/sqlbuild/compiler/compile/types.py b/src/sqlbuild/compiler/compile/types.py index 927c6a14fd..d60ad95c12 100644 --- a/src/sqlbuild/compiler/compile/types.py +++ b/src/sqlbuild/compiler/compile/types.py @@ -5,7 +5,7 @@ from enum import StrEnum from typing import TYPE_CHECKING, Any, Protocol -from sqlbuild.sql_values.models import SqlValue +from sqlbuild._native import SqlValue if TYPE_CHECKING: from collections.abc import Callable, Sequence @@ -27,10 +27,18 @@ FunctionArgument, FunctionReturnColumn, LiteralSqlRelation, + MeasurementContract, + MeasurementThresholds, + ScenarioConfig, + SchemaModelEntry, + SchemaSeedEntry, ScopeIndex, + SettingsConfig, + SourceEntry, + SourceLocation, + SqlLexicalSyntax, UsageRecord, ) - from sqlbuild.compiler.auditing.models import MeasurementContract, MeasurementThresholds from sqlbuild.compiler.auditing.types import AuditEvaluationMode, AuditSeverity from sqlbuild.compiler.compile.models import ( CompiledObjectKey, @@ -51,15 +59,6 @@ EnumDeclaration, ) from sqlbuild.compiler.references.types import ExternalSqlReferenceResolver - from sqlbuild.compiler.sql_analysis.models import SqlLexicalSyntax - from sqlbuild.spec.contracts.models import ( - ScenarioConfig, - SchemaModelEntry, - SchemaSeedEntry, - SettingsConfig, - SourceEntry, - SourceLocation, - ) type RelationKeyRow = tuple[ diff --git a/src/sqlbuild/compiler/discovery/models.py b/src/sqlbuild/compiler/discovery/models.py index 9570fb3258..b5b1732ec8 100644 --- a/src/sqlbuild/compiler/discovery/models.py +++ b/src/sqlbuild/compiler/discovery/models.py @@ -21,8 +21,17 @@ DiscoveredSqlModelFile, DiscoveredSqlTestBlock, EnumMember, + EventExportSeverity, + LocalConfig, NativeDiscoverySession, + ProjectConfig, ScopeKind, + SourceColumnEntry, + SourceEntry, + SourceWriteStrategy, + SqlLogicalType, + SqlValue, + SqlValueKind, ) from sqlbuild.compiler.discovery.constants import ( SQL_SCENARIOS_OWNERSHIP_ROOT, @@ -36,15 +45,6 @@ from sqlbuild.python_nodes.types import PythonCheckSeverity from sqlbuild.runtime.event_exporting.constants import EVENT_EXPORT_KINDS from sqlbuild.runtime.output_capture.types import CommandOutputStream -from sqlbuild.spec.contracts.models import ( - LocalConfig, - ProjectConfig, - SourceColumnEntry, - SourceEntry, -) -from sqlbuild.spec.contracts.types import EventExportSeverity, SourceWriteStrategy -from sqlbuild.sql_values.models import SqlLogicalType, SqlValue -from sqlbuild.sql_values.types import SqlValueKind @dataclass(frozen=True) diff --git a/src/sqlbuild/compiler/fingerprints/models.py b/src/sqlbuild/compiler/fingerprints/models.py deleted file mode 100644 index c205f5e659..0000000000 --- a/src/sqlbuild/compiler/fingerprints/models.py +++ /dev/null @@ -1,41 +0,0 @@ -"""Fingerprint domain models.""" - -from __future__ import annotations - -from dataclasses import dataclass -from datetime import datetime - - -@dataclass(frozen=True) -class Fingerprint: - """One applied fingerprint record from a successful materialization.""" - - node_type: str - node_name: str - target_database: str | None - target_schema: str | None - target_name: str | None - run_id: str - definition_hash: str - schema_fingerprint: str - definition: str - ts: datetime - metadata_json: str = "{}" - version_hash: str = "" - - -@dataclass(frozen=True) -class FingerprintSet: - """Latest fingerprints per node name for one target schema.""" - - schema: str - fingerprints: dict[str, Fingerprint] - fingerprints_by_identity: dict[tuple[str, str], Fingerprint] | None = None - - def __post_init__(self) -> None: - if self.fingerprints_by_identity is None: - object.__setattr__( - self, - "fingerprints_by_identity", - {(fp.node_type, fp.node_name): fp for fp in self.fingerprints.values()}, - ) diff --git a/src/sqlbuild/compiler/planner/models.py b/src/sqlbuild/compiler/planner/models.py index eedaf65977..2b466b4075 100644 --- a/src/sqlbuild/compiler/planner/models.py +++ b/src/sqlbuild/compiler/planner/models.py @@ -12,11 +12,19 @@ BackfillResult, ColumnRenameHint, CursorWatermarkMode, + DateValue, DirectSourceFreshnessPlanningResult, + Fingerprint, FunctionReturnColumn, + FutureCursorAction, + FutureCursorsConfig, GraphResourceKind, + IntegerValue, + MeasurementThresholds, + MicrobatchLimitAction, MigrationCompatibility, MigrationDiscovery, + MissingMigrationOriginPolicy, OldNameViewAction, PlanProviderUsage, PlanWarning, @@ -25,12 +33,19 @@ SchemaAction, SchemaChangeKind, SchemaColumnSource, + SchemaDynamicColumnFamily, + SeedCsvSettings, + SourceEntry, + SourceWriteStrategy, SqlTestAssertionStep, + SqlValue, + StartCursorsConfig, + TimestampValue, ) +from sqlbuild._native import Duration as Duration from sqlbuild._native import cursor_parse as parse from sqlbuild.adapter.contract.models import ColumnInfo, RetentionRequest from sqlbuild.adapter.contract.types import MigrationTransfer -from sqlbuild.compiler.auditing.models import MeasurementThresholds from sqlbuild.compiler.auditing.types import ( AuditAttachmentKind, AuditEvaluationMode, @@ -43,7 +58,6 @@ FunctionLanguage, ) from sqlbuild.compiler.discovery.models import DiscoveredHookFunction, SqlTestParameterDeclaration -from sqlbuild.compiler.fingerprints.models import Fingerprint from sqlbuild.compiler.migrations.types import ( ColumnMigrationDecision, MigrationDecision, @@ -63,30 +77,8 @@ ScenarioArtifactKind, ) from sqlbuild.cursor_algebra.exceptions import CursorAlgebraError -from sqlbuild.cursor_algebra.models import ( - DateValue, - IntegerValue, - TimestampValue, -) -from sqlbuild.cursor_algebra.models import ( - Duration as Duration, -) from sqlbuild.cursor_algebra.types import Bound, BoundSentinel, CursorScalar from sqlbuild.runtime.contracts.types import ExecutionResourceKind -from sqlbuild.spec.contracts.models import ( - FutureCursorsConfig, - SchemaDynamicColumnFamily, - SeedCsvSettings, - SourceEntry, - StartCursorsConfig, -) -from sqlbuild.spec.contracts.types import ( - FutureCursorAction, - MicrobatchLimitAction, - MissingMigrationOriginPolicy, - SourceWriteStrategy, -) -from sqlbuild.sql_values.models import SqlValue @dataclass(frozen=True, kw_only=True, init=False) diff --git a/src/sqlbuild/compiler/python_nodes/_helpers/identity.py b/src/sqlbuild/compiler/python_nodes/_helpers/identity.py index 24edafabb6..fbc7f1b172 100644 --- a/src/sqlbuild/compiler/python_nodes/_helpers/identity.py +++ b/src/sqlbuild/compiler/python_nodes/_helpers/identity.py @@ -8,11 +8,12 @@ import json import textwrap from collections.abc import Callable, Iterable, Mapping +from dataclasses import asdict from pathlib import Path from types import ModuleType from typing import Any, cast -from sqlbuild.compiler.python_nodes.models import PythonIdentityDependency, PythonNodeIdentity +from sqlbuild._native import PythonIdentityDependency, PythonNodeIdentity _IGNORED_PATH_PARTS: frozenset[str] = frozenset( { @@ -66,13 +67,13 @@ def build_python_identity( "source_text": source, } metadata_payload: dict[str, object] = { - "dependencies": tuple(vars(dependency) for dependency in dependencies) + "dependencies": tuple(asdict(dependency) for dependency in dependencies) } version_hash: str = _hash_text( _stable_json( { "definition_hash": definition_hash, - "dependencies": tuple(vars(dependency) for dependency in dependencies), + "dependencies": tuple(asdict(dependency) for dependency in dependencies), } ) ) diff --git a/src/sqlbuild/compiler/python_nodes/main/_hook_identities.py b/src/sqlbuild/compiler/python_nodes/main/_hook_identities.py index 7621223123..9d802bcc6c 100644 --- a/src/sqlbuild/compiler/python_nodes/main/_hook_identities.py +++ b/src/sqlbuild/compiler/python_nodes/main/_hook_identities.py @@ -4,10 +4,10 @@ from pathlib import Path +from sqlbuild._native import PythonNodeIdentity from sqlbuild.compiler.discovery.models import DiscoveredHookFunction from sqlbuild.compiler.fingerprints.constants import NODE_TYPE_HOOK from sqlbuild.compiler.python_nodes.main.identity import build_python_node_identity -from sqlbuild.compiler.python_nodes.models import PythonNodeIdentity def build_hook_identities( diff --git a/src/sqlbuild/compiler/python_nodes/main/identity.py b/src/sqlbuild/compiler/python_nodes/main/identity.py index 7e1296d2d7..8018b394dd 100644 --- a/src/sqlbuild/compiler/python_nodes/main/identity.py +++ b/src/sqlbuild/compiler/python_nodes/main/identity.py @@ -5,8 +5,8 @@ from collections.abc import Callable, Mapping from pathlib import Path +from sqlbuild._native import PythonNodeIdentity from sqlbuild.compiler.python_nodes._helpers.identity import build_python_identity -from sqlbuild.compiler.python_nodes.models import PythonNodeIdentity def build_python_node_identity( diff --git a/src/sqlbuild/compiler/python_nodes/models.py b/src/sqlbuild/compiler/python_nodes/models.py deleted file mode 100644 index 601582f2dd..0000000000 --- a/src/sqlbuild/compiler/python_nodes/models.py +++ /dev/null @@ -1,108 +0,0 @@ -"""Internal Python-node discovery models.""" - -from __future__ import annotations - -from collections.abc import Callable -from dataclasses import dataclass, field -from pathlib import Path - -from sqlbuild._native import ( - DiscoveredPythonAssetMetadata, - DiscoveredPythonCheckMetadata, - DiscoveredPythonLoaderMetadata, - DiscoveredPythonTaskMetadata, - PythonNodeDependencyEdge, -) -from sqlbuild.compiler.discovery.models import DiscoveredProviderUsage -from sqlbuild.compiler.python_nodes.types import PythonNodeKind -from sqlbuild.python_nodes.models import SqlResourceRef - - -@dataclass(frozen=True) -class DiscoveredPythonNode: - """Shared internal view of a discovered Python DAG node.""" - - kind: PythonNodeKind - file_path: Path - relative_path: Path - name: str - function: Callable[..., object] - depends_on: tuple[Callable[..., object] | SqlResourceRef, ...] = field(default_factory=tuple) - sql_deps: tuple[SqlResourceRef, ...] = field(default_factory=tuple) - tags: tuple[str, ...] = field(default_factory=tuple) - group: str | None = None - description: str | None = None - meta: dict[str, object] | None = None - provider_usages: tuple[DiscoveredProviderUsage, ...] = field(default_factory=tuple) - loader: DiscoveredPythonLoaderMetadata | None = None - task: DiscoveredPythonTaskMetadata | None = None - asset: DiscoveredPythonAssetMetadata | None = None - check: DiscoveredPythonCheckMetadata | None = None - identity: PythonNodeIdentity | None = None - - -@dataclass(frozen=True) -class PythonNodeGraph: - """Internal graph inventory for discovered executable Python nodes.""" - - nodes: tuple[DiscoveredPythonNode, ...] - dependency_edges: tuple[PythonNodeDependencyEdge, ...] - upstream_deps: dict[str, tuple[str, ...]] - downstream_deps: dict[str, tuple[str, ...]] - tag_index: dict[str, frozenset[str]] - path_index: dict[str, str] - nodes_by_name: dict[str, DiscoveredPythonNode] - nodes_by_typed_selector: dict[str, DiscoveredPythonNode] - - def selected_sql_refs(self, *, selected_names: frozenset[str]) -> frozenset[SqlResourceRef]: - """Return SQL references required by a selected Python-node closure.""" - - refs: set[SqlResourceRef] = set() - pending: list[str] = list(selected_names) - visited: set[str] = set() - while pending: - node_name: str = pending.pop() - if node_name in visited: - continue - visited.add(node_name) - node: DiscoveredPythonNode | None = self.nodes_by_name.get(node_name) - if node is not None: - refs.update(node.sql_deps) - pending.extend(self.upstream_deps.get(node_name, ())) - return frozenset(refs) - - -@dataclass(frozen=True) -class PythonSqlRunLifecyclePlan: - """Lifecycle-aware run classification for SQL resources and Python nodes.""" - - ingress_python_node_names: frozenset[str] - ingress_loader_names: frozenset[str] - read_side_python_node_names: frozenset[str] - - -@dataclass(frozen=True) -class PythonIdentityDependency: - """One first-party Python source object included in a Python node identity.""" - - kind: str - module: str - qualname: str - source_path: str - source_hash: str - source_text: str - - -@dataclass(frozen=True) -class PythonNodeIdentity: - """Read-only identity metadata for a SQLBuild Python node.""" - - node_type: str - node_name: str - source_path: str - source_hash: str - definition_hash: str - version_hash: str - dependencies: tuple[PythonIdentityDependency, ...] = field(default_factory=tuple) - definition_json: str = "{}" - metadata_json: str = "{}" diff --git a/src/sqlbuild/compiler/source_freshness/constants.py b/src/sqlbuild/compiler/source_freshness/constants.py index 0cd633a952..971545a6c7 100644 --- a/src/sqlbuild/compiler/source_freshness/constants.py +++ b/src/sqlbuild/compiler/source_freshness/constants.py @@ -2,7 +2,7 @@ from __future__ import annotations -from sqlbuild.sql_values.types import StateSqlValueType +from sqlbuild._native import StateSqlValueType SOURCE_FRESHNESS_TABLE_NAME: str = "_sqlbuild_source_freshness" diff --git a/src/sqlbuild/compiler/source_freshness/models.py b/src/sqlbuild/compiler/source_freshness/models.py deleted file mode 100644 index 8b20d3d929..0000000000 --- a/src/sqlbuild/compiler/source_freshness/models.py +++ /dev/null @@ -1,43 +0,0 @@ -"""Direct source freshness state models.""" - -from __future__ import annotations - -from dataclasses import dataclass -from datetime import datetime - -from sqlbuild._native import ( - SourceFreshnessIdentity, - require_aware_utc_datetime, -) - - -@dataclass(frozen=True) -class SourceFreshnessRecord: - """One append-only direct observation whose observed time is always aware UTC.""" - - source_name: str - target_database: str | None - target_schema: str | None - target_name: str | None - run_id: str - strategy: str - value_kind: str - data_version: str | None - data_version_hash: str - observed_at: datetime - - def __post_init__(self) -> None: - object.__setattr__( - self, - "observed_at", - require_aware_utc_datetime(value=self.observed_at, field_name="observed_at"), - ) - - @property - def identity(self) -> SourceFreshnessIdentity: - return SourceFreshnessIdentity( - source_name=self.source_name, - target_database=self.target_database, - target_schema=self.target_schema, - target_name=self.target_name, - ) diff --git a/src/sqlbuild/compiler/sql_analysis/constants.py b/src/sqlbuild/compiler/sql_analysis/constants.py index c7e0b21087..d68a9dc000 100644 --- a/src/sqlbuild/compiler/sql_analysis/constants.py +++ b/src/sqlbuild/compiler/sql_analysis/constants.py @@ -2,12 +2,6 @@ from __future__ import annotations -SQL_ESCAPABLE_QUOTE_CHARACTERS: frozenset[str] = frozenset({"'", '"'}) -SQL_ESCAPE_CHARACTER: str = "\\" -SQL_TRIPLE_QUOTE_LENGTH: int = 3 -SQL_LINE_COMMENT_PREFIX: str = "--" -SUPPORTED_SQL_LINE_COMMENT_PREFIXES: frozenset[str] = frozenset({"--", "//", "#"}) -SQL_BLOCK_COMMENT_OPEN: str = "/*" POLYGLOT_ANALYZE_QUERY_ENTRY_POINT: str = "analyze_query" POLYGLOT_ANALYZE_QUERY_OPTIONS_ARGUMENT_COUNT: int = 2 POLYGLOT_ANALYZE_QUERY_OPTIONS_ARGUMENT_INDEX: int = 1 diff --git a/src/sqlbuild/compiler/sql_analysis/models.py b/src/sqlbuild/compiler/sql_analysis/models.py deleted file mode 100644 index 47f779ab2b..0000000000 --- a/src/sqlbuild/compiler/sql_analysis/models.py +++ /dev/null @@ -1,84 +0,0 @@ -"""Stable SQLBuild-owned schema validation models.""" - -from __future__ import annotations - -from dataclasses import dataclass - -from sqlbuild._native import UnsupportedLexicalSyntaxError -from sqlbuild.compiler.sql_analysis.constants import ( - SQL_BLOCK_COMMENT_OPEN, - SQL_ESCAPABLE_QUOTE_CHARACTERS, - SQL_ESCAPE_CHARACTER, - SQL_LINE_COMMENT_PREFIX, - SQL_TRIPLE_QUOTE_LENGTH, - SUPPORTED_SQL_LINE_COMMENT_PREFIXES, -) - - -@dataclass(frozen=True) -class SqlLexicalSyntax: - """Dialect rules for where SQL quoted text and comments end.""" - - backslash_escape_quotes: frozenset[str] = frozenset() - escape_string_prefix: bool = False - raw_string_prefix: bool = False - triple_quoted_strings: bool = False - nested_block_comments: bool = False - line_comment_prefixes: frozenset[str] = frozenset({SQL_LINE_COMMENT_PREFIX}) - - def __post_init__(self) -> None: - unsupported: frozenset[str] = ( - self.line_comment_prefixes - SUPPORTED_SQL_LINE_COMMENT_PREFIXES - ) - if unsupported: - raise UnsupportedLexicalSyntaxError( - f"SQL line comment prefixes {sorted(unsupported)} are not supported; SQLBuild " - f"scans only {sorted(SUPPORTED_SQL_LINE_COMMENT_PREFIXES)}" - ) - - @property - def cache_key(self) -> str: - """Return a process-stable text key for these lexical rules.""" - - return "|".join( - ( - ",".join(sorted(self.backslash_escape_quotes)), - str(self.escape_string_prefix), - str(self.raw_string_prefix), - str(self.triple_quoted_strings), - str(self.nested_block_comments), - ",".join(sorted(self.line_comment_prefixes)), - ) - ) - - @property - def native_mapping(self) -> dict[str, object]: - """Return these lexical rules as the mapping native scanners read.""" - - return { - "backslash_escape_quotes": sorted(self.backslash_escape_quotes), - "escape_string_prefix": self.escape_string_prefix, - "raw_string_prefix": self.raw_string_prefix, - "triple_quoted_strings": self.triple_quoted_strings, - "nested_block_comments": self.nested_block_comments, - "line_comment_prefixes": sorted(self.line_comment_prefixes), - } - - def reads_differently_from_generic(self, sql: str) -> bool: - """Return whether these rules can read the SQL differently from generic SQL.""" - - if SQL_ESCAPE_CHARACTER in sql and ( - self.backslash_escape_quotes or self.escape_string_prefix - ): - return True - if self.triple_quoted_strings and any( - quote * SQL_TRIPLE_QUOTE_LENGTH in sql for quote in SQL_ESCAPABLE_QUOTE_CHARACTERS - ): - return True - if self.nested_block_comments and sql.count(SQL_BLOCK_COMMENT_OPEN) > 1: - return True - return any( - prefix in sql - for prefix in self.line_comment_prefixes - if prefix != SQL_LINE_COMMENT_PREFIX - ) diff --git a/src/sqlbuild/cost/constants.py b/src/sqlbuild/cost/constants.py index 43664c4c4a..5e618485cd 100644 --- a/src/sqlbuild/cost/constants.py +++ b/src/sqlbuild/cost/constants.py @@ -1,6 +1,3 @@ """Cost observability defaults.""" -from decimal import Decimal - -DEFAULT_USD_PER_CREDIT: Decimal = Decimal("3.00") SQLBUILD_QUERY_TAG_APP: str = "sqlbuild" diff --git a/src/sqlbuild/cursor_algebra/constants.py b/src/sqlbuild/cursor_algebra/constants.py index 4c93c60836..094a033b7e 100644 --- a/src/sqlbuild/cursor_algebra/constants.py +++ b/src/sqlbuild/cursor_algebra/constants.py @@ -10,10 +10,3 @@ CursorGrain.MONTH: 4, CursorGrain.YEAR: 5, } - -DURATION_YEAR_UNIT: str = "y" -DURATION_MONTH_UNIT: str = "mo" -DURATION_DAY_UNIT: str = "d" -DURATION_HOUR_UNIT: str = "h" -DURATION_MINUTE_UNIT: str = "m" -DURATION_SECOND_UNIT: str = "s" diff --git a/src/sqlbuild/cursor_algebra/models.py b/src/sqlbuild/cursor_algebra/models.py deleted file mode 100644 index 02ec9e9332..0000000000 --- a/src/sqlbuild/cursor_algebra/models.py +++ /dev/null @@ -1,143 +0,0 @@ -"""Immutable cursor algebra values.""" - -from __future__ import annotations - -from dataclasses import dataclass, field -from datetime import date, datetime -from typing import ClassVar - -from sqlbuild._native import ( - parse_cursor_duration, - shift_by_cursor_duration, - validate_cursor_interval, -) -from sqlbuild.compiler.planner.types import CursorGrain -from sqlbuild.cursor_algebra.constants import ( - DURATION_DAY_UNIT, - DURATION_HOUR_UNIT, - DURATION_MINUTE_UNIT, - DURATION_MONTH_UNIT, - DURATION_SECOND_UNIT, - DURATION_YEAR_UNIT, -) - - -@dataclass(frozen=True, kw_only=True) -class TimestampValue: - """Timestamp cursor value; aware datetimes are normalized to UTC at ingestion.""" - - value: datetime - source_text: str | None = field(default=None, compare=False, repr=False) - - -@dataclass(frozen=True, kw_only=True) -class DateValue: - """Date cursor value which must remain date-formatted when rendered.""" - - value: date - - -@dataclass(frozen=True, kw_only=True) -class IntegerValue: - """Integral cursor value.""" - - value: int - - -@dataclass(frozen=True, kw_only=True) -class AlignedInterval: - """Validated half-open cursor interval.""" - - start: TimestampValue | DateValue | IntegerValue - end: TimestampValue | DateValue | IntegerValue - grain: CursorGrain | None - - def __post_init__(self) -> None: - validate_cursor_interval(start=self.start, end=self.end, grain=self.grain) - - -@dataclass(frozen=True) -class Duration: - """A calendar-aware duration (years/months plus fixed time units).""" - - _MONTHS_PER_YEAR: ClassVar[int] = 12 - _SECONDS_PER_DAY: ClassVar[int] = 86_400 - _SECONDS_PER_HOUR: ClassVar[int] = 3_600 - _SECONDS_PER_MINUTE: ClassVar[int] = 60 - - years: int = 0 - months: int = 0 - days: int = 0 - hours: int = 0 - minutes: int = 0 - seconds: int = 0 - - @classmethod - def parse(cls, value: str) -> Duration | None: - """Parse a string like '1mo', '1y6mo', '30d', or '6h' into a duration.""" - - parts: tuple[int, int, int, int, int, int] | None = parse_cursor_duration(value) - if parts is None: - return None - years, months, days, hours, minutes, seconds = parts - return cls( - years=years, months=months, days=days, hours=hours, minutes=minutes, seconds=seconds - ) - - @property - def is_zero(self) -> bool: - """Return whether the duration is empty.""" - - return self.total_months == 0 and self.fixed_seconds == 0 - - @property - def units(self) -> frozenset[str]: - """Return the unit suffixes with non-zero values.""" - - values_by_unit: tuple[tuple[str, int], ...] = ( - (DURATION_YEAR_UNIT, self.years), - (DURATION_MONTH_UNIT, self.months), - (DURATION_DAY_UNIT, self.days), - (DURATION_HOUR_UNIT, self.hours), - (DURATION_MINUTE_UNIT, self.minutes), - (DURATION_SECOND_UNIT, self.seconds), - ) - return frozenset(unit for unit, amount in values_by_unit if amount != 0) - - def is_single_unit_in(self, allowed_units: frozenset[str]) -> bool: - """Return whether exactly one used unit belongs to the allowed vocabulary.""" - - return len(self.units) == 1 and self.units <= allowed_units - - @property - def has_calendar_component(self) -> bool: - """Return whether the duration includes variable-length years or months.""" - - return self.total_months != 0 - - @property - def fixed_seconds(self) -> int: - """Return the fixed-length portion (days and below) as whole seconds.""" - - return ( - self.days * self._SECONDS_PER_DAY - + self.hours * self._SECONDS_PER_HOUR - + self.minutes * self._SECONDS_PER_MINUTE - + self.seconds - ) - - @property - def total_months(self) -> int: - """Return the whole-month portion (years folded into months).""" - - return self.years * self._MONTHS_PER_YEAR + self.months - - def add_to(self, moment: datetime) -> datetime: - """Return the moment advanced by this duration.""" - - return shift_by_cursor_duration(moment=moment, duration=self, direction=1) - - def subtract_from(self, moment: datetime) -> datetime: - """Return the moment moved back by this duration.""" - - return shift_by_cursor_duration(moment=moment, duration=self, direction=-1) diff --git a/src/sqlbuild/cursor_algebra/types.py b/src/sqlbuild/cursor_algebra/types.py index 6d86a5f165..4fb5c4e2f9 100644 --- a/src/sqlbuild/cursor_algebra/types.py +++ b/src/sqlbuild/cursor_algebra/types.py @@ -2,7 +2,7 @@ from enum import StrEnum -from sqlbuild.cursor_algebra.models import DateValue, IntegerValue, TimestampValue +from sqlbuild._native import DateValue, IntegerValue, TimestampValue type CursorScalar = TimestampValue | DateValue | IntegerValue diff --git a/src/sqlbuild/executor/audit_results/constants.py b/src/sqlbuild/executor/audit_results/constants.py index eec3a1ae12..551379eaaf 100644 --- a/src/sqlbuild/executor/audit_results/constants.py +++ b/src/sqlbuild/executor/audit_results/constants.py @@ -1,6 +1,6 @@ """Audit result warehouse storage constants.""" -from sqlbuild.sql_values.types import StateSqlValueType +from sqlbuild._native import StateSqlValueType AUDIT_RESULTS_TABLE_NAME: str = "_sqlbuild_audit_results" AUDIT_RESULT_SCHEMA_VERSION: int = 1 diff --git a/src/sqlbuild/executor/auditing/_helpers/execution.py b/src/sqlbuild/executor/auditing/_helpers/execution.py index 6b16e9c4a3..4161711549 100644 --- a/src/sqlbuild/executor/auditing/_helpers/execution.py +++ b/src/sqlbuild/executor/auditing/_helpers/execution.py @@ -11,6 +11,7 @@ from sqlbuild._native import ( CostContext, + SourceEntry, assert_no_unresolved_sql_markers, evaluate_measurement, render_audit_sql, @@ -32,7 +33,6 @@ from sqlbuild.executor.auditing.models import AuditExecutionContext, AuditExecutionResult from sqlbuild.runtime.observability.classes.operation_lifecycle import OperationLifecycle from sqlbuild.runtime.observability.models import OperationAttributes -from sqlbuild.spec.contracts.models import SourceEntry def execute_audit_impl( # noqa: PLR0913 diff --git a/src/sqlbuild/executor/auditing/_helpers/result_projection.py b/src/sqlbuild/executor/auditing/_helpers/result_projection.py index dbe7f1d4ec..f61f547cba 100644 --- a/src/sqlbuild/executor/auditing/_helpers/result_projection.py +++ b/src/sqlbuild/executor/auditing/_helpers/result_projection.py @@ -13,9 +13,8 @@ from datetime import UTC, datetime from typing import Any -from sqlbuild._native import build_audit_gate_identity +from sqlbuild._native import AuditIdentity, MeasurementThresholdBound, build_audit_gate_identity from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter -from sqlbuild.compiler.auditing.models import AuditIdentity, MeasurementThresholdBound from sqlbuild.compiler.auditing.types import AuditEvaluationMode, ThresholdOperator from sqlbuild.compiler.compile.models import CompiledRelationLocation from sqlbuild.compiler.planner.models import AuditPlanEntry, PlanOutput diff --git a/src/sqlbuild/executor/auditing/main/_execute.py b/src/sqlbuild/executor/auditing/main/_execute.py index 715addf1dd..2e5f1472a0 100644 --- a/src/sqlbuild/executor/auditing/main/_execute.py +++ b/src/sqlbuild/executor/auditing/main/_execute.py @@ -4,13 +4,13 @@ from typing import Any +from sqlbuild._native import SourceEntry from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter from sqlbuild.compiler.auditing.types import AuditRunScope from sqlbuild.compiler.compile.models import CompiledRelationLocation from sqlbuild.compiler.planner.models import AuditPlanEntry from sqlbuild.executor.auditing._helpers.execution import execute_audit_impl from sqlbuild.executor.auditing.models import AuditExecutionResult -from sqlbuild.spec.contracts.models import SourceEntry def execute_audit( diff --git a/src/sqlbuild/executor/auditing/main/_execute_reporting_errors.py b/src/sqlbuild/executor/auditing/main/_execute_reporting_errors.py index 3339ec1915..c98c402be6 100644 --- a/src/sqlbuild/executor/auditing/main/_execute_reporting_errors.py +++ b/src/sqlbuild/executor/auditing/main/_execute_reporting_errors.py @@ -4,13 +4,13 @@ from typing import Any +from sqlbuild._native import SourceEntry from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter from sqlbuild.compiler.auditing.types import AuditRunScope from sqlbuild.compiler.compile.models import CompiledRelationLocation from sqlbuild.compiler.planner.models import AuditPlanEntry from sqlbuild.executor.auditing._helpers.execution import execute_audit_impl, failed_audit_result from sqlbuild.executor.auditing.models import AuditExecutionResult -from sqlbuild.spec.contracts.models import SourceEntry def execute_audit_reporting_errors( diff --git a/src/sqlbuild/executor/auditing/models.py b/src/sqlbuild/executor/auditing/models.py index 01fffea0c8..13ff6176e0 100644 --- a/src/sqlbuild/executor/auditing/models.py +++ b/src/sqlbuild/executor/auditing/models.py @@ -6,8 +6,8 @@ from dataclasses import dataclass from typing import Any +from sqlbuild._native import MeasurementThresholds, SourceEntry from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter -from sqlbuild.compiler.auditing.models import MeasurementThresholds from sqlbuild.compiler.auditing.types import ( AuditAttachmentKind, AuditEvaluationMode, @@ -17,7 +17,6 @@ ) from sqlbuild.compiler.compile.models import CompiledRelationLocation from sqlbuild.compiler.compile.types import AttachedAuditTargetKind -from sqlbuild.spec.contracts.models import SourceEntry @dataclass(frozen=True) diff --git a/src/sqlbuild/executor/build/_helpers/end_audits.py b/src/sqlbuild/executor/build/_helpers/end_audits.py index 3169b537d9..9671bbf12f 100644 --- a/src/sqlbuild/executor/build/_helpers/end_audits.py +++ b/src/sqlbuild/executor/build/_helpers/end_audits.py @@ -4,7 +4,7 @@ from typing import Any -from sqlbuild._native import audit_resource_id +from sqlbuild._native import SourceEntry, audit_resource_id from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter from sqlbuild.compiler.auditing.types import AuditOutcome, AuditRunScope from sqlbuild.compiler.compile.models import CompiledRelationLocation @@ -19,7 +19,6 @@ from sqlbuild.runtime.observability.classes.resource_attempt_lifecycle import ( ResourceAttemptLifecycle, ) -from sqlbuild.spec.contracts.models import SourceEntry def run_end_audits( diff --git a/src/sqlbuild/executor/build/_helpers/external_source_loads.py b/src/sqlbuild/executor/build/_helpers/external_source_loads.py index c622624d7a..063650d68b 100644 --- a/src/sqlbuild/executor/build/_helpers/external_source_loads.py +++ b/src/sqlbuild/executor/build/_helpers/external_source_loads.py @@ -4,6 +4,7 @@ from collections.abc import Callable +from sqlbuild._native import SourceEntry from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter from sqlbuild.compiler.compile.models import CompiledObjectKey from sqlbuild.compiler.discovery.models import DiscoveredLoaderFunction @@ -22,7 +23,6 @@ from sqlbuild.runtime.observability.classes.resource_attempt_lifecycle import ( ResourceAttemptLifecycle, ) -from sqlbuild.spec.contracts.models import SourceEntry def run_external_source_loads_before_connections( diff --git a/src/sqlbuild/executor/build/_helpers/retention.py b/src/sqlbuild/executor/build/_helpers/retention.py index b5a4d06249..0b2dd5f845 100644 --- a/src/sqlbuild/executor/build/_helpers/retention.py +++ b/src/sqlbuild/executor/build/_helpers/retention.py @@ -4,6 +4,7 @@ from typing import Any +from sqlbuild._native import TableType from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter from sqlbuild.adapter.contract.models import ( RelationInfo, @@ -35,7 +36,6 @@ from sqlbuild.runtime.observability.main.canonicalize_operation_adapter import ( canonicalize_operation_adapter, ) -from sqlbuild.spec.contracts.types import TableType def materialization_recreates_relation(entry: ModelPlanEntry) -> bool: diff --git a/src/sqlbuild/executor/build/_helpers/scheduler.py b/src/sqlbuild/executor/build/_helpers/scheduler.py index 1e41b5847a..480429e6de 100644 --- a/src/sqlbuild/executor/build/_helpers/scheduler.py +++ b/src/sqlbuild/executor/build/_helpers/scheduler.py @@ -5,6 +5,7 @@ from collections.abc import Callable, Mapping from sqlbuild._native import ( + SnapshotsConfig, assert_no_unresolved_sql_markers, replaces_incremental_relation, resolve_cursor_intrinsics, @@ -47,7 +48,6 @@ from sqlbuild.executor.scheduling.types import ExecutionStatus from sqlbuild.executor.testing.models import SqlTestExecutionResult from sqlbuild.runtime.contracts.types import ExecutionResourceKind -from sqlbuild.spec.contracts.models import SnapshotsConfig type _BuildWorkerResult = ( ModelExecutionResult diff --git a/src/sqlbuild/executor/build/_helpers/source_node.py b/src/sqlbuild/executor/build/_helpers/source_node.py index 028b2af56b..9992c266c6 100644 --- a/src/sqlbuild/executor/build/_helpers/source_node.py +++ b/src/sqlbuild/executor/build/_helpers/source_node.py @@ -7,7 +7,7 @@ from collections.abc import Callable from typing import Any -from sqlbuild._native import CostContext +from sqlbuild._native import CostContext, SourceEntry from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter from sqlbuild.adapter.contract.classes.statement_recorder import StatementRecorder from sqlbuild.compiler.compile.models import CompiledObjectKey @@ -20,7 +20,6 @@ from sqlbuild.executor.node_results.main._direct_store import build_direct_node_result_store from sqlbuild.executor.node_results.models import NodeResultRecord from sqlbuild.runtime.contracts.types import ExecutionResourceKind -from sqlbuild.spec.contracts.models import SourceEntry def execute_build_source_node( diff --git a/src/sqlbuild/executor/build/classes/build_scheduler.py b/src/sqlbuild/executor/build/classes/build_scheduler.py index 6284493d5c..f5fc1f4c14 100644 --- a/src/sqlbuild/executor/build/classes/build_scheduler.py +++ b/src/sqlbuild/executor/build/classes/build_scheduler.py @@ -15,7 +15,7 @@ from pathlib import Path from typing import Any, cast -from sqlbuild._native import CostContext, sql_test_resource_id +from sqlbuild._native import CostContext, SnapshotsConfig, SourceEntry, sql_test_resource_id from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter from sqlbuild.adapter.contract.classes.statement_recorder import StatementRecorder from sqlbuild.adapter.contract.models import RelationInfo @@ -124,7 +124,6 @@ from sqlbuild.runtime.observability.classes.resource_attempt_lifecycle import ( ResourceAttemptLifecycle, ) -from sqlbuild.spec.contracts.models import SnapshotsConfig, SourceEntry _DEBUG_LOGGER: logging.Logger = logging.getLogger("sqlbuild.execution") diff --git a/src/sqlbuild/executor/build/models.py b/src/sqlbuild/executor/build/models.py index 15bd6e6c8b..5586233fc3 100644 --- a/src/sqlbuild/executor/build/models.py +++ b/src/sqlbuild/executor/build/models.py @@ -7,6 +7,7 @@ from datetime import datetime from pathlib import Path +from sqlbuild._native import SnapshotsConfig from sqlbuild.adapter.contract.models import LifeCycleEvent, RelationInfo from sqlbuild.adapter.contract.types import TablePromotionMode from sqlbuild.compiler.compile.models import CompiledObjectKey, CompiledRelationLocation @@ -33,7 +34,6 @@ from sqlbuild.provider.main.runtime import ProviderContainer from sqlbuild.python_nodes.models import SqlResourceRef from sqlbuild.runtime.contracts.types import ConnectionElapsedCallback, NodeStartCallback -from sqlbuild.spec.contracts.models import SnapshotsConfig @dataclass(frozen=True) diff --git a/src/sqlbuild/executor/clone/_helpers/fingerprinting.py b/src/sqlbuild/executor/clone/_helpers/fingerprinting.py index 347dbba958..c890a87ab9 100644 --- a/src/sqlbuild/executor/clone/_helpers/fingerprinting.py +++ b/src/sqlbuild/executor/clone/_helpers/fingerprinting.py @@ -8,7 +8,12 @@ from datetime import UTC, datetime from typing import Any -from sqlbuild._native import read_latest_fingerprints, write_fingerprints +from sqlbuild._native import ( + Fingerprint, + FingerprintSet, + read_latest_fingerprints, + write_fingerprints, +) from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter from sqlbuild.adapter.contract.models import RelationLookup from sqlbuild.adapter.relations.main.relation_lookup import build_relation_lookup @@ -17,7 +22,6 @@ NODE_TYPE_MODEL, NODE_TYPE_SEED, ) -from sqlbuild.compiler.fingerprints.models import Fingerprint, FingerprintSet from sqlbuild.compiler.planner.models import ModelPlanEntry, SeedPlanEntry from sqlbuild.compiler.planner.types import MaterializationType from sqlbuild.executor.clone.models import CloneExecutionResult diff --git a/src/sqlbuild/executor/clone/_helpers/retention.py b/src/sqlbuild/executor/clone/_helpers/retention.py index 0740cf885c..d3a6a3fb34 100644 --- a/src/sqlbuild/executor/clone/_helpers/retention.py +++ b/src/sqlbuild/executor/clone/_helpers/retention.py @@ -4,6 +4,7 @@ from typing import Any +from sqlbuild._native import ResolvedTimeTravelRetention from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter from sqlbuild.adapter.contract.models import ( RenderedRetentionChange, @@ -21,7 +22,6 @@ from sqlbuild.runtime.observability.main.canonicalize_operation_adapter import ( canonicalize_operation_adapter, ) -from sqlbuild.spec.contracts.models import ResolvedTimeTravelRetention def build_clone_retention_requests( diff --git a/src/sqlbuild/executor/clone/main/_build_destination_transient_models.py b/src/sqlbuild/executor/clone/main/_build_destination_transient_models.py index 778d01aabc..1f5dbbc6e2 100644 --- a/src/sqlbuild/executor/clone/main/_build_destination_transient_models.py +++ b/src/sqlbuild/executor/clone/main/_build_destination_transient_models.py @@ -1,7 +1,7 @@ """Collect destination models whose resolved table type is transient.""" +from sqlbuild._native import TableType from sqlbuild.compiler.compile.types import CompiledProjectView -from sqlbuild.spec.contracts.types import TableType def build_destination_transient_models(*, project: CompiledProjectView) -> frozenset[str]: diff --git a/src/sqlbuild/executor/diff/_helpers/query_artifacts.py b/src/sqlbuild/executor/diff/_helpers/query_artifacts.py index 1d0196fc8e..fe57c3fbbc 100644 --- a/src/sqlbuild/executor/diff/_helpers/query_artifacts.py +++ b/src/sqlbuild/executor/diff/_helpers/query_artifacts.py @@ -8,7 +8,13 @@ from datetime import UTC, datetime from typing import Any -from sqlbuild._native import compute_query_hash, read_latest_fingerprints, write_fingerprint +from sqlbuild._native import ( + Fingerprint, + FingerprintSet, + compute_query_hash, + read_latest_fingerprints, + write_fingerprint, +) from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter from sqlbuild.adapter.contract.classes.statement_recorder import StatementRecorder from sqlbuild.adapter.contract.models import ColumnInfo, RelationInfo, RelationLookup @@ -17,7 +23,6 @@ FINGERPRINT_TABLE_NAME, NODE_TYPE_QUERY_DIFF_ARTIFACT, ) -from sqlbuild.compiler.fingerprints.models import Fingerprint, FingerprintSet from sqlbuild.errors.contracts.exceptions import ExecutorInputError from sqlbuild.executor.diff.models import ( QueryDiffArtifact, diff --git a/src/sqlbuild/executor/functions/main/_execute.py b/src/sqlbuild/executor/functions/main/_execute.py index f00bba06d6..2061b580c4 100644 --- a/src/sqlbuild/executor/functions/main/_execute.py +++ b/src/sqlbuild/executor/functions/main/_execute.py @@ -7,6 +7,7 @@ from typing import Any from sqlbuild._native import ( + Fingerprint, compute_function_definition_hash, error_code, error_help, @@ -18,7 +19,6 @@ from sqlbuild.adapter.contract.classes.statement_recorder import StatementRecorder from sqlbuild.adapter.contract.models import FunctionDefinition from sqlbuild.compiler.compile.types import FunctionLanguage -from sqlbuild.compiler.fingerprints.models import Fingerprint from sqlbuild.compiler.planner.models import FunctionPlanEntry from sqlbuild.errors.contracts.exceptions import ExecutorInputError from sqlbuild.executor.build.models import FunctionExecutionResult diff --git a/src/sqlbuild/executor/janitor_events/constants.py b/src/sqlbuild/executor/janitor_events/constants.py index 721f93d887..9e30530b97 100644 --- a/src/sqlbuild/executor/janitor_events/constants.py +++ b/src/sqlbuild/executor/janitor_events/constants.py @@ -1,6 +1,6 @@ """Janitor audit event warehouse storage constants.""" -from sqlbuild.sql_values.types import StateSqlValueType +from sqlbuild._native import StateSqlValueType JANITOR_EVENTS_TABLE_NAME: str = "_sqlbuild_janitor_events" JANITOR_EVENT_SCHEMA_VERSION: int = 1 diff --git a/src/sqlbuild/executor/load/_helpers/cursors.py b/src/sqlbuild/executor/load/_helpers/cursors.py index 4f91b530d7..dfe099793b 100644 --- a/src/sqlbuild/executor/load/_helpers/cursors.py +++ b/src/sqlbuild/executor/load/_helpers/cursors.py @@ -6,12 +6,12 @@ from decimal import Decimal from typing import Any +from sqlbuild._native import DateValue, IntegerValue, TimestampValue from sqlbuild._native import cursor_parse as parse from sqlbuild._native import cursor_render as render from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter from sqlbuild.adapter.contract.classes.statement_recorder import StatementRecorder from sqlbuild.adapter.contract.types import CursorKind -from sqlbuild.cursor_algebra.models import DateValue, IntegerValue, TimestampValue from sqlbuild.cursor_algebra.types import CursorScalar from sqlbuild.errors.contracts.exceptions import ExecutorInputError diff --git a/src/sqlbuild/executor/load/_helpers/dag_runtime.py b/src/sqlbuild/executor/load/_helpers/dag_runtime.py index d4e93068f9..0695bf42ec 100644 --- a/src/sqlbuild/executor/load/_helpers/dag_runtime.py +++ b/src/sqlbuild/executor/load/_helpers/dag_runtime.py @@ -7,6 +7,7 @@ from dataclasses import replace from typing import Any +from sqlbuild._native import SourceEntry from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter from sqlbuild.adapter.contract.classes.statement_recorder import StatementRecorder from sqlbuild.compiler.python_nodes.types import SkipMode @@ -39,7 +40,6 @@ current_execution_identity, ) from sqlbuild.runtime.observability.models import ExecutionIdentity -from sqlbuild.spec.contracts.models import SourceEntry def build_load_dag_state( diff --git a/src/sqlbuild/executor/load/_helpers/execution.py b/src/sqlbuild/executor/load/_helpers/execution.py index 3cca33cf04..4e516e8c92 100644 --- a/src/sqlbuild/executor/load/_helpers/execution.py +++ b/src/sqlbuild/executor/load/_helpers/execution.py @@ -4,13 +4,13 @@ from collections.abc import Callable +from sqlbuild._native import SourceEntry from sqlbuild.compiler.discovery.models import DiscoveredLoaderFunction from sqlbuild.compiler.python_nodes.types import SkipMode from sqlbuild.executor.load.models import LoadExecutionIndexes, LoadExecutionResult from sqlbuild.executor.scheduling.types import ExecutionStatus from sqlbuild.python_nodes.models import SqlResourceRef from sqlbuild.runtime.contracts.types import ExecutionResourceKind -from sqlbuild.spec.contracts.models import SourceEntry def build_load_execution_indexes( diff --git a/src/sqlbuild/executor/load/_helpers/external.py b/src/sqlbuild/executor/load/_helpers/external.py index 2436ffb75d..5b041b73b1 100644 --- a/src/sqlbuild/executor/load/_helpers/external.py +++ b/src/sqlbuild/executor/load/_helpers/external.py @@ -6,6 +6,7 @@ import time from collections.abc import Callable +from sqlbuild._native import SourceEntry from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter from sqlbuild.adapter.contract.classes.statement_recorder import StatementRecorder from sqlbuild.compiler.discovery.models import DiscoveredLoaderFunction @@ -26,7 +27,6 @@ ) from sqlbuild.runtime.contracts.types import ExecutionResourceKind from sqlbuild.runtime.observability.classes.operation_lifecycle import OperationLifecycle -from sqlbuild.spec.contracts.models import SourceEntry def execute_external_source_load( diff --git a/src/sqlbuild/executor/load/_helpers/loader_invocation.py b/src/sqlbuild/executor/load/_helpers/loader_invocation.py index 21db821933..193c8dccfe 100644 --- a/src/sqlbuild/executor/load/_helpers/loader_invocation.py +++ b/src/sqlbuild/executor/load/_helpers/loader_invocation.py @@ -7,6 +7,7 @@ from collections.abc import Callable, Mapping from typing import Any +from sqlbuild._native import SourceEntry, SourceWriteStrategy from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter from sqlbuild.adapter.contract.classes.statement_recorder import StatementRecorder from sqlbuild.compiler.discovery.models import DiscoveredLoaderFunction @@ -35,8 +36,6 @@ from sqlbuild.python_nodes.models import SqlResourceRef from sqlbuild.python_nodes.types import SqlResourceRefKind from sqlbuild.runtime.contracts.types import ExecutionResourceKind -from sqlbuild.spec.contracts.models import SourceEntry -from sqlbuild.spec.contracts.types import SourceWriteStrategy _SUPPORTED_WRITE_STRATEGIES: frozenset[SourceWriteStrategy] = frozenset( { diff --git a/src/sqlbuild/executor/load/_helpers/relation_refs.py b/src/sqlbuild/executor/load/_helpers/relation_refs.py index 2b2a97a894..64cf6af316 100644 --- a/src/sqlbuild/executor/load/_helpers/relation_refs.py +++ b/src/sqlbuild/executor/load/_helpers/relation_refs.py @@ -5,13 +5,13 @@ from collections.abc import Callable, Mapping from typing import Any +from sqlbuild._native import SourceEntry from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter from sqlbuild.adapter.contract.classes.statement_recorder import StatementRecorder from sqlbuild.adapter.relations.main.resolve_qualified_name_parts import ( resolve_qualified_name_parts, ) from sqlbuild.executor.load.models import LoaderRelationRef -from sqlbuild.spec.contracts.models import SourceEntry def build_loader_relation_refs( diff --git a/src/sqlbuild/executor/load/_helpers/rows.py b/src/sqlbuild/executor/load/_helpers/rows.py index 9df83270e5..83f1bcab53 100644 --- a/src/sqlbuild/executor/load/_helpers/rows.py +++ b/src/sqlbuild/executor/load/_helpers/rows.py @@ -6,12 +6,12 @@ from datetime import date, datetime from decimal import Decimal +from sqlbuild._native import SourceColumnEntry from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter from sqlbuild.adapter.contract.models import ColumnInfo from sqlbuild.adapter.contract.types import LoaderLogicalType from sqlbuild.errors.contracts.exceptions import ExecutorInputError from sqlbuild.executor.load.models import LoaderRowsSchema -from sqlbuild.spec.contracts.models import SourceColumnEntry def iter_loader_row_batches( diff --git a/src/sqlbuild/executor/load/_helpers/scheduling.py b/src/sqlbuild/executor/load/_helpers/scheduling.py index fde0ced4ed..074e624949 100644 --- a/src/sqlbuild/executor/load/_helpers/scheduling.py +++ b/src/sqlbuild/executor/load/_helpers/scheduling.py @@ -10,6 +10,7 @@ from functools import partial from typing import Any +from sqlbuild._native import SourceEntry from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter from sqlbuild.compiler.discovery.models import DiscoveredLoaderFunction from sqlbuild.compiler.discovery.types import LoaderConnectionMode @@ -38,7 +39,6 @@ from sqlbuild.executor.load.types import LoadProgressCallback from sqlbuild.executor.scheduling.types import ExecutionStatus from sqlbuild.runtime.contracts.types import ConnectionElapsedCallback -from sqlbuild.spec.contracts.models import SourceEntry def validate_external_loaders_are_preconnect_runnable( diff --git a/src/sqlbuild/executor/load/_helpers/schema.py b/src/sqlbuild/executor/load/_helpers/schema.py index aa21d5cd7a..40b3b2842b 100644 --- a/src/sqlbuild/executor/load/_helpers/schema.py +++ b/src/sqlbuild/executor/load/_helpers/schema.py @@ -4,13 +4,13 @@ from typing import Any +from sqlbuild._native import SourceEntry from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter from sqlbuild.adapter.contract.classes.statement_recorder import StatementRecorder from sqlbuild.adapter.contract.models import ColumnInfo from sqlbuild.adapter.type_system.main.types_equal import types_equal from sqlbuild.compiler.planner.types import ContractPolicy from sqlbuild.errors.contracts.exceptions import ExecutorInputError -from sqlbuild.spec.contracts.models import SourceEntry def validate_and_evolve_existing_target( diff --git a/src/sqlbuild/executor/load/_helpers/staging.py b/src/sqlbuild/executor/load/_helpers/staging.py index 237ae5b80b..6e566cb5c4 100644 --- a/src/sqlbuild/executor/load/_helpers/staging.py +++ b/src/sqlbuild/executor/load/_helpers/staging.py @@ -4,7 +4,7 @@ from typing import Any -from sqlbuild._native import fit_auxiliary_relation_name +from sqlbuild._native import SourceEntry, fit_auxiliary_relation_name from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter from sqlbuild.adapter.contract.classes.statement_recorder import StatementRecorder from sqlbuild.adapter.contract.types import LoaderLogicalType @@ -19,7 +19,6 @@ ) from sqlbuild.executor.load.models import LoaderRowsSchema from sqlbuild.executor.run.constants import STAGING_RELATION_SUFFIX -from sqlbuild.spec.contracts.models import SourceEntry def write_loader_rows_to_staging( diff --git a/src/sqlbuild/executor/load/main/_build_execution_indexes.py b/src/sqlbuild/executor/load/main/_build_execution_indexes.py index 174c422334..68b1f01212 100644 --- a/src/sqlbuild/executor/load/main/_build_execution_indexes.py +++ b/src/sqlbuild/executor/load/main/_build_execution_indexes.py @@ -2,12 +2,12 @@ from __future__ import annotations +from sqlbuild._native import SourceEntry from sqlbuild.compiler.discovery.models import DiscoveredLoaderFunction from sqlbuild.executor.load._helpers.execution import ( build_load_execution_indexes as _build_load_execution_indexes, ) from sqlbuild.executor.load.models import LoadExecutionIndexes -from sqlbuild.spec.contracts.models import SourceEntry def build_load_execution_indexes( diff --git a/src/sqlbuild/executor/load/main/_execute.py b/src/sqlbuild/executor/load/main/_execute.py index 4db2a54708..48d8d44d90 100644 --- a/src/sqlbuild/executor/load/main/_execute.py +++ b/src/sqlbuild/executor/load/main/_execute.py @@ -7,7 +7,7 @@ from dataclasses import replace from typing import Any -from sqlbuild._native import CostContext +from sqlbuild._native import CostContext, SourceEntry, SourceWriteStrategy from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter from sqlbuild.adapter.contract.classes.statement_recorder import StatementRecorder from sqlbuild.adapter.contract.models import ColumnInfo @@ -46,8 +46,6 @@ from sqlbuild.provider.main.runtime import invoke_with_providers from sqlbuild.runtime.contracts.types import ExecutionResourceKind from sqlbuild.runtime.observability.classes.operation_lifecycle import OperationLifecycle -from sqlbuild.spec.contracts.models import SourceEntry -from sqlbuild.spec.contracts.types import SourceWriteStrategy def execute_source_load( diff --git a/src/sqlbuild/executor/load/main/_resource_kind.py b/src/sqlbuild/executor/load/main/_resource_kind.py index f02a8d9364..d7ce0fcb9e 100644 --- a/src/sqlbuild/executor/load/main/_resource_kind.py +++ b/src/sqlbuild/executor/load/main/_resource_kind.py @@ -2,9 +2,9 @@ from __future__ import annotations +from sqlbuild._native import SourceEntry from sqlbuild.executor.load._helpers.execution import load_resource_kind as _load_resource_kind from sqlbuild.runtime.contracts.types import ExecutionResourceKind -from sqlbuild.spec.contracts.models import SourceEntry def load_resource_kind(source: SourceEntry) -> ExecutionResourceKind: diff --git a/src/sqlbuild/executor/load/main/_skipped_result.py b/src/sqlbuild/executor/load/main/_skipped_result.py index f7a3ff7221..e85eb356c9 100644 --- a/src/sqlbuild/executor/load/main/_skipped_result.py +++ b/src/sqlbuild/executor/load/main/_skipped_result.py @@ -2,10 +2,10 @@ from __future__ import annotations +from sqlbuild._native import SourceEntry from sqlbuild.compiler.python_nodes.types import SkipMode from sqlbuild.executor.load._helpers.execution import skipped_load_result as _skipped_load_result from sqlbuild.executor.load.models import LoadExecutionResult -from sqlbuild.spec.contracts.models import SourceEntry def skipped_load_result( diff --git a/src/sqlbuild/executor/load/main/run.py b/src/sqlbuild/executor/load/main/run.py index 45333c282e..26697faa72 100644 --- a/src/sqlbuild/executor/load/main/run.py +++ b/src/sqlbuild/executor/load/main/run.py @@ -4,6 +4,7 @@ from typing import Any +from sqlbuild._native import SourceEntry from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter from sqlbuild.compiler.discovery.models import DiscoveredLoaderFunction from sqlbuild.executor.load._helpers.execution import build_load_execution_indexes @@ -20,7 +21,6 @@ LoadExecutionResult, LoadRuntimeParams, ) -from sqlbuild.spec.contracts.models import SourceEntry def run_load_pipeline( diff --git a/src/sqlbuild/executor/load/models.py b/src/sqlbuild/executor/load/models.py index 7bf948b434..2eeb1349b1 100644 --- a/src/sqlbuild/executor/load/models.py +++ b/src/sqlbuild/executor/load/models.py @@ -9,6 +9,7 @@ from pathlib import Path from typing import Any +from sqlbuild._native import SourceEntry from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter from sqlbuild.adapter.contract.classes.statement_recorder import StatementRecorder from sqlbuild.adapter.contract.models import ColumnInfo, LifeCycleEvent @@ -29,7 +30,6 @@ from sqlbuild.python_nodes.models import SqlResourceRef from sqlbuild.python_nodes.types import SqlResourceRefKind from sqlbuild.runtime.contracts.types import ConnectionElapsedCallback, ExecutionResourceKind -from sqlbuild.spec.contracts.models import SourceEntry @dataclass(frozen=True) diff --git a/src/sqlbuild/executor/load/types.py b/src/sqlbuild/executor/load/types.py index 7fe3787ea8..231e9958ad 100644 --- a/src/sqlbuild/executor/load/types.py +++ b/src/sqlbuild/executor/load/types.py @@ -5,7 +5,7 @@ from typing import TYPE_CHECKING, Protocol if TYPE_CHECKING: - from sqlbuild.spec.contracts.models import SourceEntry + from sqlbuild._native import SourceEntry class LoadProgressCallback(Protocol): diff --git a/src/sqlbuild/executor/migrations/_helpers/old_name_views.py b/src/sqlbuild/executor/migrations/_helpers/old_name_views.py index c83c502598..304f590be0 100644 --- a/src/sqlbuild/executor/migrations/_helpers/old_name_views.py +++ b/src/sqlbuild/executor/migrations/_helpers/old_name_views.py @@ -31,7 +31,6 @@ ) from sqlbuild.compiler.migrations.types import OldNameViewEventType from sqlbuild.compiler.planner.models import OldNameView, OldNameViewPlanEntry, PlanOutput -from sqlbuild.cursor_algebra.models import Duration from sqlbuild.errors.contracts.exceptions import ExecutorInputError from sqlbuild.executor.migrations._helpers.old_name_facts import ( old_name_fact, diff --git a/src/sqlbuild/executor/node_results/constants.py b/src/sqlbuild/executor/node_results/constants.py index 9408696776..c57129c44c 100644 --- a/src/sqlbuild/executor/node_results/constants.py +++ b/src/sqlbuild/executor/node_results/constants.py @@ -1,6 +1,6 @@ """Runtime node result storage constants.""" -from sqlbuild.sql_values.types import StateSqlValueType +from sqlbuild._native import StateSqlValueType NODE_RESULTS_TABLE_NAME: str = "_sqlbuild_node_results" NODE_RESULT_MATERIALIZED_TRUE_VALUE: str = "true" diff --git a/src/sqlbuild/executor/pipeline/_helpers/scenario.py b/src/sqlbuild/executor/pipeline/_helpers/scenario.py index 60f819b29f..9141520410 100644 --- a/src/sqlbuild/executor/pipeline/_helpers/scenario.py +++ b/src/sqlbuild/executor/pipeline/_helpers/scenario.py @@ -17,6 +17,7 @@ from sqlbuild._native import ( NativeProgressProjector, + SqlLexicalSyntax, current_native_progress_projector, error_code, error_help, @@ -29,7 +30,6 @@ from sqlbuild.compiler.compile.models import CompiledSqlScenario from sqlbuild.compiler.pipeline.models import CompilePipelineResult from sqlbuild.compiler.planner.models import ScenarioExecutionPlan -from sqlbuild.compiler.sql_analysis.models import SqlLexicalSyntax from sqlbuild.diagnostics.main.log_debug_event import log_debug_event from sqlbuild.executor.pipeline._helpers.connections import ( close_connections, diff --git a/src/sqlbuild/executor/pipeline/_helpers/schema_preparation.py b/src/sqlbuild/executor/pipeline/_helpers/schema_preparation.py index b792ee08e2..d8ac9b14b9 100644 --- a/src/sqlbuild/executor/pipeline/_helpers/schema_preparation.py +++ b/src/sqlbuild/executor/pipeline/_helpers/schema_preparation.py @@ -4,10 +4,10 @@ from typing import Any +from sqlbuild._native import SourceEntry from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter from sqlbuild.adapter.contract.classes.statement_recorder import StatementRecorder from sqlbuild.compiler.planner.models import PlanOutput -from sqlbuild.spec.contracts.models import SourceEntry def prepare_build_schemas( diff --git a/src/sqlbuild/executor/pipeline/_helpers/settings.py b/src/sqlbuild/executor/pipeline/_helpers/settings.py index fb3fa386ca..f640d45d49 100644 --- a/src/sqlbuild/executor/pipeline/_helpers/settings.py +++ b/src/sqlbuild/executor/pipeline/_helpers/settings.py @@ -4,6 +4,7 @@ from dataclasses import replace +from sqlbuild._native import SettingsConfig from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter from sqlbuild.adapter.contract.types import TablePromotionMode from sqlbuild.executor.build.models import ( @@ -13,7 +14,6 @@ BuildRuntimeParams, ) from sqlbuild.executor.pipeline.models import ResolvedBuildInputs -from sqlbuild.spec.contracts.models import SettingsConfig def resolve_promotion_mode( diff --git a/src/sqlbuild/executor/pipeline/main/run.py b/src/sqlbuild/executor/pipeline/main/run.py index e2b61b7ec7..37c3a1fb96 100644 --- a/src/sqlbuild/executor/pipeline/main/run.py +++ b/src/sqlbuild/executor/pipeline/main/run.py @@ -6,7 +6,7 @@ from dataclasses import replace from typing import Any -from sqlbuild._native import BuildPhaseTimingTracker, CostContext +from sqlbuild._native import BuildPhaseTimingTracker, CostContext, SettingsConfig from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter from sqlbuild.compiler.planner.models import PlanOutput from sqlbuild.diagnostics.main.diagnostics_context import diagnostics_context @@ -65,7 +65,6 @@ from sqlbuild.runtime.observability.main.create_lifecycle_event import create_lifecycle_event from sqlbuild.runtime.observability.main.current_event_dispatcher import current_event_dispatcher from sqlbuild.runtime.observability.main.run_scope import run_scope -from sqlbuild.spec.contracts.models import SettingsConfig def run_build_pipeline( diff --git a/src/sqlbuild/executor/python_nodes/_helpers/fingerprinting.py b/src/sqlbuild/executor/python_nodes/_helpers/fingerprinting.py index 7e8e656d48..bb4aa6a85d 100644 --- a/src/sqlbuild/executor/python_nodes/_helpers/fingerprinting.py +++ b/src/sqlbuild/executor/python_nodes/_helpers/fingerprinting.py @@ -7,10 +7,8 @@ from datetime import UTC, datetime from typing import Any -from sqlbuild._native import write_fingerprint +from sqlbuild._native import Fingerprint, PythonNodeIdentity, write_fingerprint from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter -from sqlbuild.compiler.fingerprints.models import Fingerprint -from sqlbuild.compiler.python_nodes.models import PythonNodeIdentity def try_write_python_node_identity_fingerprint( diff --git a/src/sqlbuild/executor/python_nodes/_helpers/ingress_execution.py b/src/sqlbuild/executor/python_nodes/_helpers/ingress_execution.py index 3faa8e9f0e..ef2a5fed83 100644 --- a/src/sqlbuild/executor/python_nodes/_helpers/ingress_execution.py +++ b/src/sqlbuild/executor/python_nodes/_helpers/ingress_execution.py @@ -8,18 +8,19 @@ from functools import partial from typing import Any -from sqlbuild._native import CostContext +from sqlbuild._native import ( + CostContext, + DiscoveredPythonNode, + PythonNodeGraph, + PythonSqlRunLifecyclePlan, + SourceEntry, +) from sqlbuild.adapter.contract.classes.statement_recorder import StatementRecorder from sqlbuild.compiler.discovery.models import ( DiscoveredAssetFunction, DiscoveredLoaderFunction, DiscoveredTaskFunction, ) -from sqlbuild.compiler.python_nodes.models import ( - DiscoveredPythonNode, - PythonNodeGraph, - PythonSqlRunLifecyclePlan, -) from sqlbuild.compiler.python_nodes.types import PythonNodeKind, PythonNodeStatus, SkipMode from sqlbuild.errors.contracts.exceptions import ExecutorInputError from sqlbuild.executor.load.main._execute import execute_source_load @@ -57,7 +58,6 @@ current_execution_identity, ) from sqlbuild.runtime.observability.models import ExecutionIdentity -from sqlbuild.spec.contracts.models import SourceEntry def execute_ingress_python_loader_nodes( diff --git a/src/sqlbuild/executor/python_nodes/_helpers/lifecycle_nodes.py b/src/sqlbuild/executor/python_nodes/_helpers/lifecycle_nodes.py index 9a1654c004..a82fdd2210 100644 --- a/src/sqlbuild/executor/python_nodes/_helpers/lifecycle_nodes.py +++ b/src/sqlbuild/executor/python_nodes/_helpers/lifecycle_nodes.py @@ -2,11 +2,7 @@ from __future__ import annotations -from sqlbuild.compiler.python_nodes.models import ( - DiscoveredPythonNode, - PythonNodeGraph, - PythonSqlRunLifecyclePlan, -) +from sqlbuild._native import DiscoveredPythonNode, PythonNodeGraph, PythonSqlRunLifecyclePlan from sqlbuild.executor.scheduling.models import LifecycleExecutionNode diff --git a/src/sqlbuild/executor/python_nodes/_helpers/python_checks.py b/src/sqlbuild/executor/python_nodes/_helpers/python_checks.py index fe5a64ac54..ed48cde51b 100644 --- a/src/sqlbuild/executor/python_nodes/_helpers/python_checks.py +++ b/src/sqlbuild/executor/python_nodes/_helpers/python_checks.py @@ -6,15 +6,10 @@ from collections.abc import Mapping from typing import Any -from sqlbuild._native import CostContext +from sqlbuild._native import CostContext, DiscoveredPythonNode, PythonNodeGraph, PythonNodeIdentity from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter from sqlbuild.adapter.contract.classes.statement_recorder import StatementRecorder from sqlbuild.compiler.discovery.models import DiscoveredCheckFunction -from sqlbuild.compiler.python_nodes.models import ( - DiscoveredPythonNode, - PythonNodeGraph, - PythonNodeIdentity, -) from sqlbuild.compiler.python_nodes.types import PythonNodeKind, PythonNodeStatus from sqlbuild.errors.contracts.exceptions import ExecutorInputError from sqlbuild.executor.load.models import LoadExecutionResult diff --git a/src/sqlbuild/executor/python_nodes/classes/read_side_python_execution_tracker.py b/src/sqlbuild/executor/python_nodes/classes/read_side_python_execution_tracker.py index bed168a6ee..b85237fa24 100644 --- a/src/sqlbuild/executor/python_nodes/classes/read_side_python_execution_tracker.py +++ b/src/sqlbuild/executor/python_nodes/classes/read_side_python_execution_tracker.py @@ -5,9 +5,8 @@ from dataclasses import replace from typing import Any -from sqlbuild._native import CostContext +from sqlbuild._native import CostContext, DiscoveredPythonNode, PythonNodeGraph from sqlbuild.adapter.contract.classes.statement_recorder import StatementRecorder -from sqlbuild.compiler.python_nodes.models import DiscoveredPythonNode, PythonNodeGraph from sqlbuild.compiler.python_nodes.types import PythonNodeKind, PythonNodeStatus from sqlbuild.executor.node_results.main._direct_store import build_direct_node_result_store from sqlbuild.executor.python_nodes._helpers.fingerprinting import ( diff --git a/src/sqlbuild/executor/python_nodes/main/_checks.py b/src/sqlbuild/executor/python_nodes/main/_checks.py index 8068a7640c..6d1b0e7b9e 100644 --- a/src/sqlbuild/executor/python_nodes/main/_checks.py +++ b/src/sqlbuild/executor/python_nodes/main/_checks.py @@ -4,8 +4,8 @@ from collections.abc import Mapping +from sqlbuild._native import PythonNodeGraph from sqlbuild.compiler.discovery.models import DiscoveredCheckFunction -from sqlbuild.compiler.python_nodes.models import PythonNodeGraph from sqlbuild.executor.load.models import LoadExecutionResult from sqlbuild.executor.python_nodes._helpers.python_checks import execute_python_check_nodes from sqlbuild.executor.python_nodes.models import ( diff --git a/src/sqlbuild/executor/python_nodes/main/_fingerprinting.py b/src/sqlbuild/executor/python_nodes/main/_fingerprinting.py index 8ae8eb7a7e..33dbdf9b53 100644 --- a/src/sqlbuild/executor/python_nodes/main/_fingerprinting.py +++ b/src/sqlbuild/executor/python_nodes/main/_fingerprinting.py @@ -5,8 +5,8 @@ import logging from typing import Any +from sqlbuild._native import PythonNodeIdentity from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter -from sqlbuild.compiler.python_nodes.models import PythonNodeIdentity from sqlbuild.executor.python_nodes._helpers.fingerprinting import ( try_write_python_node_identity_fingerprint as _try_write_python_node_identity_fingerprint, ) diff --git a/src/sqlbuild/executor/python_nodes/main/_ingress.py b/src/sqlbuild/executor/python_nodes/main/_ingress.py index b213494c47..71322e2201 100644 --- a/src/sqlbuild/executor/python_nodes/main/_ingress.py +++ b/src/sqlbuild/executor/python_nodes/main/_ingress.py @@ -4,8 +4,8 @@ from collections.abc import Mapping +from sqlbuild._native import PythonNodeGraph, SourceEntry from sqlbuild.compiler.discovery.models import DiscoveredLoaderFunction -from sqlbuild.compiler.python_nodes.models import PythonNodeGraph from sqlbuild.executor.python_nodes._helpers.ingress_execution import ( execute_ingress_python_loader_nodes, ) @@ -14,7 +14,6 @@ PythonIngressLoaderExecutorResult, PythonNodeRuntime, ) -from sqlbuild.spec.contracts.models import SourceEntry def run_ingress_python_loader_nodes( diff --git a/src/sqlbuild/executor/python_nodes/main/read_side.py b/src/sqlbuild/executor/python_nodes/main/read_side.py index 46725c23af..cc5613834c 100644 --- a/src/sqlbuild/executor/python_nodes/main/read_side.py +++ b/src/sqlbuild/executor/python_nodes/main/read_side.py @@ -2,7 +2,7 @@ from __future__ import annotations -from sqlbuild.compiler.python_nodes.models import PythonNodeGraph +from sqlbuild._native import PythonNodeGraph from sqlbuild.executor.python_nodes.classes.read_side_python_execution_tracker import ( ReadSidePythonExecutionTracker, ) diff --git a/src/sqlbuild/executor/python_nodes/types.py b/src/sqlbuild/executor/python_nodes/types.py index 31c512f726..8c6c2ac47f 100644 --- a/src/sqlbuild/executor/python_nodes/types.py +++ b/src/sqlbuild/executor/python_nodes/types.py @@ -4,8 +4,8 @@ from typing import TYPE_CHECKING, Protocol +from sqlbuild._native import PythonNodeIdentity from sqlbuild.compiler.discovery.models import DiscoveredAssetFunction, DiscoveredTaskFunction -from sqlbuild.compiler.python_nodes.models import PythonNodeIdentity type ExecutablePythonNode = DiscoveredTaskFunction | DiscoveredAssetFunction diff --git a/src/sqlbuild/executor/run/_helpers/execution/final_audits.py b/src/sqlbuild/executor/run/_helpers/execution/final_audits.py index 52cff927a5..8393fc33fe 100644 --- a/src/sqlbuild/executor/run/_helpers/execution/final_audits.py +++ b/src/sqlbuild/executor/run/_helpers/execution/final_audits.py @@ -4,10 +4,10 @@ from typing import Any +from sqlbuild._native import Fingerprint, SourceEntry from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter from sqlbuild.compiler.auditing.types import AuditOutcome, AuditRunScope from sqlbuild.compiler.compile.models import CompiledRelationLocation -from sqlbuild.compiler.fingerprints.models import Fingerprint from sqlbuild.compiler.planner.models import AuditPlanEntry from sqlbuild.executor.auditing.main._execute import execute_audit from sqlbuild.executor.auditing.models import AuditExecutionResult @@ -16,7 +16,6 @@ reused_final_audit_results_by_binding_key, ) from sqlbuild.executor.run.models import FinalAuditRun, ModelMaterializationContext -from sqlbuild.spec.contracts.models import SourceEntry def run_final_model_audits( diff --git a/src/sqlbuild/executor/run/_helpers/execution/hooks.py b/src/sqlbuild/executor/run/_helpers/execution/hooks.py index e5cd6aa0b1..ac5d0c572e 100644 --- a/src/sqlbuild/executor/run/_helpers/execution/hooks.py +++ b/src/sqlbuild/executor/run/_helpers/execution/hooks.py @@ -5,7 +5,7 @@ from collections.abc import Mapping from typing import Any -from sqlbuild._native import CostContext, render_source_relation +from sqlbuild._native import CostContext, PythonNodeIdentity, SourceEntry, render_source_relation from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter from sqlbuild.adapter.contract.classes.statement_recorder import StatementRecorder from sqlbuild.adapter.relations.main.resolve_relation_location_qualified_name import ( @@ -19,7 +19,6 @@ ) from sqlbuild.compiler.fingerprints.constants import NODE_TYPE_HOOK from sqlbuild.compiler.python_nodes.main.identity import build_python_node_identity -from sqlbuild.compiler.python_nodes.models import PythonNodeIdentity from sqlbuild.compiler.python_nodes.types import SkipMode from sqlbuild.compiler.references.types import HardCodedRelationOwnerKind from sqlbuild.cost.models import CostResourceContext @@ -47,7 +46,6 @@ from sqlbuild.python_nodes.models import SqlResourceRef from sqlbuild.python_nodes.types import SqlResourceRefKind from sqlbuild.runtime.observability.classes.operation_lifecycle import OperationLifecycle -from sqlbuild.spec.contracts.models import SourceEntry def execute_hooks( diff --git a/src/sqlbuild/executor/run/_helpers/execution/staging.py b/src/sqlbuild/executor/run/_helpers/execution/staging.py index a2c668d98e..8b34bf380e 100644 --- a/src/sqlbuild/executor/run/_helpers/execution/staging.py +++ b/src/sqlbuild/executor/run/_helpers/execution/staging.py @@ -4,9 +4,9 @@ from typing import Any +from sqlbuild._native import Fingerprint from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter from sqlbuild.adapter.contract.classes.statement_recorder import StatementRecorder -from sqlbuild.compiler.fingerprints.models import Fingerprint from sqlbuild.executor.run.models import ModelMaterializationContext from sqlbuild.runtime.observability.classes.operation_lifecycle import ( OperationAttributes, diff --git a/src/sqlbuild/executor/run/_helpers/materializations/custom.py b/src/sqlbuild/executor/run/_helpers/materializations/custom.py index e3e6fbaa28..1e59e4f1e8 100644 --- a/src/sqlbuild/executor/run/_helpers/materializations/custom.py +++ b/src/sqlbuild/executor/run/_helpers/materializations/custom.py @@ -7,6 +7,7 @@ from dataclasses import replace from typing import Any, cast +from sqlbuild._native import SourceEntry from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter from sqlbuild.adapter.contract.classes.statement_recorder import StatementRecorder from sqlbuild.adapter.contract.models import ColumnInfo, RelationInfo @@ -46,7 +47,6 @@ from sqlbuild.executor.scheduling.types import ExecutionStatus from sqlbuild.provider.main.runtime import _empty_provider_container, invoke_with_providers from sqlbuild.runtime.observability.classes.operation_lifecycle import OperationLifecycle -from sqlbuild.spec.contracts.models import SourceEntry def execute_custom_entry( diff --git a/src/sqlbuild/executor/run/_helpers/materializations/incremental.py b/src/sqlbuild/executor/run/_helpers/materializations/incremental.py index cf4c517f67..7b0bbd7b04 100644 --- a/src/sqlbuild/executor/run/_helpers/materializations/incremental.py +++ b/src/sqlbuild/executor/run/_helpers/materializations/incremental.py @@ -5,12 +5,13 @@ from dataclasses import dataclass, replace from typing import Any -from sqlbuild._native import cursor_sentinel_to_token as sentinel_to_token from sqlbuild._native import ( + TableType, fit_auxiliary_relation_name, future_cursor_cap_warning, maximum_start_cap_warning, ) +from sqlbuild._native import cursor_sentinel_to_token as sentinel_to_token from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter from sqlbuild.adapter.contract.classes.statement_recorder import StatementRecorder from sqlbuild.adapter.contract.models import ColumnInfo @@ -69,7 +70,6 @@ from sqlbuild.runtime.observability.main.canonicalize_operation_adapter import ( canonicalize_operation_adapter, ) -from sqlbuild.spec.contracts.types import TableType _DEFAULT_ON_SCHEMA_CHANGE: OnSchemaChange = OnSchemaChange.APPEND_NEW_COLUMNS diff --git a/src/sqlbuild/executor/run/_helpers/materializations/microbatch.py b/src/sqlbuild/executor/run/_helpers/materializations/microbatch.py index 599fc8b238..1201893013 100644 --- a/src/sqlbuild/executor/run/_helpers/materializations/microbatch.py +++ b/src/sqlbuild/executor/run/_helpers/materializations/microbatch.py @@ -12,6 +12,12 @@ from typing import Any from sqlbuild._native import ( + AlignedInterval, + DateValue, + IntegerValue, + MicrobatchLimitAction, + TableType, + TimestampValue, aligned_timestamp_bounds, compute_query_hash, cursor_bound_display, @@ -59,7 +65,6 @@ MicrobatchStrategy, OnSchemaChange, ) -from sqlbuild.cursor_algebra.models import AlignedInterval, DateValue, IntegerValue, TimestampValue from sqlbuild.cursor_algebra.types import Bound, BoundSentinel, CursorScalar from sqlbuild.diagnostics.main.diagnostics_context import diagnostics_context from sqlbuild.diagnostics.main.log_debug_event import log_debug_event @@ -162,7 +167,6 @@ canonicalize_operation_adapter, ) from sqlbuild.runtime.observability.models import MicrobatchLifecycleContext -from sqlbuild.spec.contracts.types import MicrobatchLimitAction, TableType _DEFAULT_ON_SCHEMA_CHANGE: OnSchemaChange = OnSchemaChange.APPEND_NEW_COLUMNS _DEBUG_LOGGER: logging.Logger = logging.getLogger("sqlbuild.execution") diff --git a/src/sqlbuild/executor/run/_helpers/materializations/snapshot.py b/src/sqlbuild/executor/run/_helpers/materializations/snapshot.py index 9c15fb0b75..f2ca9f2636 100644 --- a/src/sqlbuild/executor/run/_helpers/materializations/snapshot.py +++ b/src/sqlbuild/executor/run/_helpers/materializations/snapshot.py @@ -4,7 +4,7 @@ from typing import Any -from sqlbuild._native import fit_auxiliary_relation_name +from sqlbuild._native import SnapshotsConfig, fit_auxiliary_relation_name from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter from sqlbuild.adapter.contract.classes.statement_recorder import StatementRecorder from sqlbuild.adapter.contract.models import ( @@ -78,7 +78,6 @@ from sqlbuild.runtime.observability.main.canonicalize_operation_adapter import ( canonicalize_operation_adapter, ) -from sqlbuild.spec.contracts.models import SnapshotsConfig _SCHEMA_CHANGE_STRICTNESS: dict[SnapshotSchemaChangePolicy, int] = { SnapshotSchemaChangePolicy.APPEND_NEW_COLUMNS: 0, diff --git a/src/sqlbuild/executor/run/_helpers/materializations/view.py b/src/sqlbuild/executor/run/_helpers/materializations/view.py index d26eb5257d..695d7b0691 100644 --- a/src/sqlbuild/executor/run/_helpers/materializations/view.py +++ b/src/sqlbuild/executor/run/_helpers/materializations/view.py @@ -5,6 +5,7 @@ from collections.abc import Mapping from typing import Any +from sqlbuild._native import SourceEntry from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter from sqlbuild.adapter.contract.classes.statement_recorder import StatementRecorder from sqlbuild.adapter.relations.main.resolve_relation_location_qualified_name import ( @@ -31,7 +32,6 @@ ) from sqlbuild.executor.run.types import ExecutionPhase, HookPhase from sqlbuild.executor.scheduling.types import ExecutionStatus -from sqlbuild.spec.contracts.models import SourceEntry def execute_view_entry( diff --git a/src/sqlbuild/executor/run/_helpers/reuse/fingerprint_metadata.py b/src/sqlbuild/executor/run/_helpers/reuse/fingerprint_metadata.py index d1130262d5..1f3a10a36c 100644 --- a/src/sqlbuild/executor/run/_helpers/reuse/fingerprint_metadata.py +++ b/src/sqlbuild/executor/run/_helpers/reuse/fingerprint_metadata.py @@ -7,8 +7,7 @@ from collections.abc import Mapping from typing import cast -from sqlbuild._native import build_audit_gate_identity -from sqlbuild.compiler.auditing.models import AuditGateIdentity, AuditIdentity +from sqlbuild._native import AuditGateIdentity, AuditIdentity, build_audit_gate_identity from sqlbuild.compiler.auditing.types import AuditOutcome, AuditSeverity from sqlbuild.compiler.fingerprints.constants import AUDIT_GATE_METADATA_KEY from sqlbuild.compiler.planner.constants import MIGRATION_FINGERPRINT_METADATA_KEY diff --git a/src/sqlbuild/executor/run/_helpers/reuse/fingerprinting.py b/src/sqlbuild/executor/run/_helpers/reuse/fingerprinting.py index 4950a2afcf..fa2713af38 100644 --- a/src/sqlbuild/executor/run/_helpers/reuse/fingerprinting.py +++ b/src/sqlbuild/executor/run/_helpers/reuse/fingerprinting.py @@ -6,10 +6,9 @@ from datetime import UTC, datetime from typing import Any -from sqlbuild._native import compute_query_hash, write_fingerprint +from sqlbuild._native import Fingerprint, compute_query_hash, write_fingerprint from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter from sqlbuild.compiler.fingerprints.constants import NODE_TYPE_MODEL -from sqlbuild.compiler.fingerprints.models import Fingerprint from sqlbuild.compiler.planner.models import AuditPlanEntry, ModelPlanEntry from sqlbuild.executor.auditing.models import AuditExecutionResult from sqlbuild.executor.run._helpers.reuse.fingerprint_metadata import ( diff --git a/src/sqlbuild/executor/run/_helpers/validation/contracts.py b/src/sqlbuild/executor/run/_helpers/validation/contracts.py index 0836cc0b4b..e18317ac5b 100644 --- a/src/sqlbuild/executor/run/_helpers/validation/contracts.py +++ b/src/sqlbuild/executor/run/_helpers/validation/contracts.py @@ -2,7 +2,7 @@ from __future__ import annotations -from sqlbuild._native import matching_dynamic_families +from sqlbuild._native import SchemaDynamicColumnFamily, matching_dynamic_families from sqlbuild.adapter.contract.models import ColumnInfo from sqlbuild.adapter.contract.types import TypeDialect from sqlbuild.adapter.type_system.main.types_equal import types_equal @@ -14,7 +14,6 @@ RUNTIME_CONTRACT_MISSING_DECLARATIONS_CODE, RUNTIME_CONTRACT_TYPE_MISMATCH_CODE, ) -from sqlbuild.spec.contracts.models import SchemaDynamicColumnFamily def validate_runtime_contract( diff --git a/src/sqlbuild/executor/run/_helpers/validation/cursor_bounds.py b/src/sqlbuild/executor/run/_helpers/validation/cursor_bounds.py index edd3b81662..135b18c79e 100644 --- a/src/sqlbuild/executor/run/_helpers/validation/cursor_bounds.py +++ b/src/sqlbuild/executor/run/_helpers/validation/cursor_bounds.py @@ -9,6 +9,11 @@ from typing import Any from sqlbuild._native import ( + AlignedInterval, + DateValue, + IntegerValue, + StartCursorsConfig, + TimestampValue, apply_maximum_start_policy, apply_typed_cursor_replay_policy, cursor_window_override_flags, @@ -51,19 +56,12 @@ MicrobatchStrategy, ) from sqlbuild.cursor_algebra.constants import GRAIN_ORDER -from sqlbuild.cursor_algebra.models import ( - AlignedInterval, - DateValue, - IntegerValue, - TimestampValue, -) from sqlbuild.cursor_algebra.types import BoundSentinel, CursorScalar from sqlbuild.errors.contracts.exceptions import ExecutorInputError from sqlbuild.executor.run.exceptions import EmptyCursorInputsError from sqlbuild.executor.run.models import RuntimeCursorInputRelation, RuntimeCursorSpec from sqlbuild.executor.run.types import RuntimeCursorWatermarkMode, WatermarkResolver from sqlbuild.spec.contracts.constants import ZERO_DAY_CURSOR_DURATION -from sqlbuild.spec.contracts.models import StartCursorsConfig def has_runtime_owned_cursor_watermarks( diff --git a/src/sqlbuild/executor/run/main/_execute.py b/src/sqlbuild/executor/run/main/_execute.py index f4a286b14f..a546af9a14 100644 --- a/src/sqlbuild/executor/run/main/_execute.py +++ b/src/sqlbuild/executor/run/main/_execute.py @@ -5,7 +5,7 @@ from dataclasses import replace from typing import Any -from sqlbuild._native import setting_help +from sqlbuild._native import Fingerprint, setting_help from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter from sqlbuild.adapter.contract.classes.statement_recorder import StatementRecorder from sqlbuild.adapter.contract.models import ColumnInfo @@ -16,7 +16,6 @@ STAGED_PROMOTION_MODE, TABLE_PROMOTION_MODE_SETTING_KEY, ) -from sqlbuild.compiler.fingerprints.models import Fingerprint from sqlbuild.compiler.planner.models import ModelPlanEntry from sqlbuild.diagnostics.main.diagnostics_context import diagnostics_context from sqlbuild.errors.contracts.exceptions import ExecutorInputError diff --git a/src/sqlbuild/executor/run/models.py b/src/sqlbuild/executor/run/models.py index 1ea931c07e..9ee0cf18f1 100644 --- a/src/sqlbuild/executor/run/models.py +++ b/src/sqlbuild/executor/run/models.py @@ -8,6 +8,15 @@ from datetime import datetime from typing import Any, cast +from sqlbuild._native import ( + DateValue, + FutureCursorsConfig, + IntegerValue, + MicrobatchLimitAction, + SourceEntry, + StartCursorsConfig, + TimestampValue, +) from sqlbuild._native import cursor_compare as compare from sqlbuild._native import cursor_parse as parse from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter @@ -26,7 +35,6 @@ from sqlbuild.compiler.planner.types import CursorGrain, CursorType from sqlbuild.compiler.python_nodes.types import SkipMode from sqlbuild.cursor_algebra.exceptions import CursorAlgebraError -from sqlbuild.cursor_algebra.models import DateValue, IntegerValue, TimestampValue from sqlbuild.cursor_algebra.types import BoundSentinel, CursorScalar from sqlbuild.errors.contracts.exceptions import ExecutorInputError from sqlbuild.executor.auditing.models import AuditExecutionResult @@ -47,8 +55,6 @@ from sqlbuild.microbatches.types import MicrobatchEventStore from sqlbuild.provider.main.runtime import ProviderContainer, _empty_provider_container from sqlbuild.python_nodes.models import SqlResourceRef -from sqlbuild.spec.contracts.models import FutureCursorsConfig, SourceEntry, StartCursorsConfig -from sqlbuild.spec.contracts.types import MicrobatchLimitAction @dataclass(frozen=True, kw_only=True, init=False) diff --git a/src/sqlbuild/executor/scenario/_helpers/local/execution.py b/src/sqlbuild/executor/scenario/_helpers/local/execution.py index 90a20fb144..7c8c4c7501 100644 --- a/src/sqlbuild/executor/scenario/_helpers/local/execution.py +++ b/src/sqlbuild/executor/scenario/_helpers/local/execution.py @@ -6,7 +6,7 @@ from pathlib import Path from typing import Any -from sqlbuild._native import error_code, error_help, error_message +from sqlbuild._native import SourceEntry, error_code, error_help, error_message from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter from sqlbuild.adapter.contract.classes.statement_recorder import StatementRecorder from sqlbuild.adapter.contract.types import TablePromotionMode @@ -69,7 +69,6 @@ from sqlbuild.runtime.observability.classes.resource_attempt_lifecycle import ( ResourceAttemptLifecycle, ) -from sqlbuild.spec.contracts.models import SourceEntry _LOCAL_STEP_FAILED_MESSAGE: str = "local scenario step failed" diff --git a/src/sqlbuild/executor/scenario/models.py b/src/sqlbuild/executor/scenario/models.py index a0fdb38e90..4f147ac77f 100644 --- a/src/sqlbuild/executor/scenario/models.py +++ b/src/sqlbuild/executor/scenario/models.py @@ -5,6 +5,7 @@ from dataclasses import dataclass, field from pathlib import Path +from sqlbuild._native import SqlLexicalSyntax from sqlbuild.adapter.contract.models import LifeCycleEvent from sqlbuild.adapter.contract.types import TablePromotionMode from sqlbuild.compiler.planner.models import ( @@ -13,7 +14,6 @@ ScenarioRelationMap, ) from sqlbuild.compiler.planner.types import MaterializationType, ScenarioArtifactKind -from sqlbuild.compiler.sql_analysis.models import SqlLexicalSyntax from sqlbuild.executor.build.models import FunctionExecutionResult, SeedExecutionResult from sqlbuild.executor.run.models import ModelExecutionResult from sqlbuild.executor.scenario.classes.prepared_scenario_schemas import PreparedScenarioSchemas diff --git a/src/sqlbuild/executor/seed/_helpers/fingerprinting.py b/src/sqlbuild/executor/seed/_helpers/fingerprinting.py index 7e6a2e752e..c72250718b 100644 --- a/src/sqlbuild/executor/seed/_helpers/fingerprinting.py +++ b/src/sqlbuild/executor/seed/_helpers/fingerprinting.py @@ -6,10 +6,9 @@ from datetime import UTC, datetime from typing import Any -from sqlbuild._native import write_fingerprint +from sqlbuild._native import Fingerprint, write_fingerprint from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter from sqlbuild.compiler.fingerprints.constants import NODE_TYPE_SEED -from sqlbuild.compiler.fingerprints.models import Fingerprint from sqlbuild.compiler.planner.models import SeedPlanEntry diff --git a/src/sqlbuild/executor/testing/models.py b/src/sqlbuild/executor/testing/models.py index 9935ec7411..e3ce3d1abf 100644 --- a/src/sqlbuild/executor/testing/models.py +++ b/src/sqlbuild/executor/testing/models.py @@ -5,9 +5,9 @@ from dataclasses import dataclass, field from pathlib import Path +from sqlbuild._native import SqlValue from sqlbuild.compiler.discovery.models import SqlTestParameterDeclaration from sqlbuild.executor.testing.types import SqlTestOutcome -from sqlbuild.sql_values.models import SqlValue @dataclass(frozen=True) diff --git a/src/sqlbuild/integrations/dbt/_helpers/profile/render.py b/src/sqlbuild/integrations/dbt/_helpers/profile/render.py index 4fe15fb4d3..902ac42703 100644 --- a/src/sqlbuild/integrations/dbt/_helpers/profile/render.py +++ b/src/sqlbuild/integrations/dbt/_helpers/profile/render.py @@ -8,8 +8,8 @@ from pathlib import Path from typing import Any, cast +from sqlbuild._native import ResolvedDbtProfileOutput, SelectedDbtProfileOutput from sqlbuild.integrations.dbt.exceptions import DbtProfileError -from sqlbuild.integrations.dbt.models import ResolvedDbtProfileOutput, SelectedDbtProfileOutput _DBT_FALSE_VALUES: frozenset[str] = frozenset({"false", "0", "no", "n", "off"}) _DBT_JINJA_MARKERS: tuple[str, ...] = ("{{", "{%", "{#") diff --git a/src/sqlbuild/integrations/dbt/models.py b/src/sqlbuild/integrations/dbt/models.py deleted file mode 100644 index 477de1ed86..0000000000 --- a/src/sqlbuild/integrations/dbt/models.py +++ /dev/null @@ -1,33 +0,0 @@ -"""dbt interop domain models.""" - -from __future__ import annotations - -from dataclasses import dataclass -from pathlib import Path - - -@dataclass(frozen=True) -class SelectedDbtProfileOutput: - """Selected dbt profile target output before rendering.""" - - profile_name: str - target_name: str - output: dict[str, object] - - -@dataclass(frozen=True) -class ResolvedDbtProfileOutput: - """Rendered dbt profile target output.""" - - project_dir: Path - profiles_dir: Path - profile_name: str - target_name: str - output: dict[str, object] - - @property - def adapter_type(self) -> str: - """Return the dbt adapter type from the rendered output.""" - - value: object | None = self.output.get("type") - return value if isinstance(value, str) else "" diff --git a/src/sqlbuild/integrations/dbt/types.py b/src/sqlbuild/integrations/dbt/types.py deleted file mode 100644 index dc4da7b53d..0000000000 --- a/src/sqlbuild/integrations/dbt/types.py +++ /dev/null @@ -1,12 +0,0 @@ -"""dbt integration type aliases.""" - -from __future__ import annotations - -from pathlib import Path -from typing import Protocol - - -class DbtInvoker(Protocol): - """Callable dbt process invocation contract.""" - - def __call__(self, *, argv: tuple[str, ...], cwd: Path | None) -> object: ... diff --git a/src/sqlbuild/integrations/ingestr/_helpers/command.py b/src/sqlbuild/integrations/ingestr/_helpers/command.py index e9d0d733a7..2c2a6f5531 100644 --- a/src/sqlbuild/integrations/ingestr/_helpers/command.py +++ b/src/sqlbuild/integrations/ingestr/_helpers/command.py @@ -6,12 +6,11 @@ from dataclasses import replace from typing import cast -from sqlbuild._native import expand_template_data +from sqlbuild._native import IntegrationLoaderConfig, SourceEntry, expand_template_data from sqlbuild.integrations.ingestr._helpers.destination import build_destination_uri from sqlbuild.integrations.ingestr.constants import INGESTR_INTEGRATION_KIND from sqlbuild.integrations.ingestr.exceptions import IngestrIntegrationError from sqlbuild.integrations.ingestr.models import IngestrSourceConfig -from sqlbuild.spec.contracts.models import IntegrationLoaderConfig, SourceEntry def resolve_ingestr_config( diff --git a/src/sqlbuild/integrations/ingestr/main/_loaders.py b/src/sqlbuild/integrations/ingestr/main/_loaders.py index 916dc95fff..a8c5688131 100644 --- a/src/sqlbuild/integrations/ingestr/main/_loaders.py +++ b/src/sqlbuild/integrations/ingestr/main/_loaders.py @@ -10,7 +10,6 @@ from sqlbuild.integrations.ingestr._helpers.runner import run_ingestr_command from sqlbuild.integrations.ingestr.exceptions import IngestrIntegrationError from sqlbuild.integrations.ingestr.models import IngestrCommandResult -from sqlbuild.spec.contracts.models import SourceEntry def ingestr_loader_function( diff --git a/src/sqlbuild/loaders.pyi b/src/sqlbuild/loaders.pyi index 977f5e3c38..4ca3937bce 100644 --- a/src/sqlbuild/loaders.pyi +++ b/src/sqlbuild/loaders.pyi @@ -1,9 +1,9 @@ from collections.abc import Callable, Sequence from typing import overload +from sqlbuild._native import SourceColumnEntry from sqlbuild.python_nodes.models import LoaderDefinition as LoaderDefinition from sqlbuild.python_nodes.types import LoaderColumnSpec -from sqlbuild.spec.contracts.models import SourceColumnEntry @overload def loader(function: Callable[..., object]) -> Callable[..., object]: ... diff --git a/src/sqlbuild/microbatches/_helpers/sql.py b/src/sqlbuild/microbatches/_helpers/sql.py index 7f8928e42c..c5c63fc6d6 100644 --- a/src/sqlbuild/microbatches/_helpers/sql.py +++ b/src/sqlbuild/microbatches/_helpers/sql.py @@ -4,7 +4,7 @@ from collections.abc import Callable -from sqlbuild._native import render_state_sql_literal +from sqlbuild._native import StateSqlValueType, render_state_sql_literal from sqlbuild.adapter.contract.types import FrameworkType from sqlbuild.microbatches.classes.event_codec import MicrobatchEventCodec from sqlbuild.microbatches.constants import ( @@ -15,7 +15,6 @@ ) from sqlbuild.microbatches.exceptions import MicrobatchStateError from sqlbuild.microbatches.models import MicrobatchEvent, MicrobatchScope -from sqlbuild.sql_values.types import StateSqlValueType def build_create_table_sql( diff --git a/src/sqlbuild/microbatches/constants.py b/src/sqlbuild/microbatches/constants.py index f302c98c0a..f0d2a3b396 100644 --- a/src/sqlbuild/microbatches/constants.py +++ b/src/sqlbuild/microbatches/constants.py @@ -1,6 +1,6 @@ """Stable constants for microbatch state storage and retries.""" -from sqlbuild.sql_values.types import StateSqlValueType +from sqlbuild._native import StateSqlValueType MICROBATCH_TABLE_NAME: str = "_sqlbuild_microbatches" DIRECT_MICROBATCH_SCOPE_KIND: str = "direct_logical" diff --git a/src/sqlbuild/python_nodes/_helpers/column_normalization.py b/src/sqlbuild/python_nodes/_helpers/column_normalization.py index 50548275b8..6e59ac75b6 100644 --- a/src/sqlbuild/python_nodes/_helpers/column_normalization.py +++ b/src/sqlbuild/python_nodes/_helpers/column_normalization.py @@ -2,8 +2,8 @@ from collections.abc import Sequence +from sqlbuild._native import SourceColumnEntry from sqlbuild.python_nodes.types import LoaderColumnSpec, PythonNodeColumnSpec -from sqlbuild.spec.contracts.models import SourceColumnEntry def normalize_columns( diff --git a/src/sqlbuild/python_nodes/_helpers/loader_authoring.py b/src/sqlbuild/python_nodes/_helpers/loader_authoring.py index 458112ffa8..a69d729e8f 100644 --- a/src/sqlbuild/python_nodes/_helpers/loader_authoring.py +++ b/src/sqlbuild/python_nodes/_helpers/loader_authoring.py @@ -4,12 +4,12 @@ from collections.abc import Callable, Sequence +from sqlbuild._native import SourceColumnEntry +from sqlbuild._native import SourceWriteStrategy as SourceWriteStrategy from sqlbuild.python_nodes.main._apply_loader import apply_loader from sqlbuild.python_nodes.main.read_loader_definition import read_loader_definition from sqlbuild.python_nodes.models import LoaderDefinition from sqlbuild.python_nodes.types import LoaderColumnSpec -from sqlbuild.spec.contracts.models import SourceColumnEntry -from sqlbuild.spec.contracts.types import SourceWriteStrategy as SourceWriteStrategy def loader( diff --git a/src/sqlbuild/python_nodes/_helpers/loader_normalization.py b/src/sqlbuild/python_nodes/_helpers/loader_normalization.py index 0dce329575..334eb4caec 100644 --- a/src/sqlbuild/python_nodes/_helpers/loader_normalization.py +++ b/src/sqlbuild/python_nodes/_helpers/loader_normalization.py @@ -2,7 +2,7 @@ from collections.abc import Sequence -from sqlbuild.spec.contracts.types import SourceWriteStrategy +from sqlbuild._native import SourceWriteStrategy def normalize_unique_key(value: str | Sequence[str] | None) -> tuple[str, ...]: diff --git a/src/sqlbuild/python_nodes/main/_apply_loader.py b/src/sqlbuild/python_nodes/main/_apply_loader.py index f2fc9f444f..67367a9a9e 100644 --- a/src/sqlbuild/python_nodes/main/_apply_loader.py +++ b/src/sqlbuild/python_nodes/main/_apply_loader.py @@ -5,6 +5,7 @@ from collections.abc import Callable, Sequence from typing import Any, cast +from sqlbuild._native import SourceColumnEntry from sqlbuild.python_nodes._helpers.attachment import attach_definition from sqlbuild.python_nodes._helpers.column_normalization import normalize_columns from sqlbuild.python_nodes._helpers.dependency_normalization import normalize_loader_dependencies @@ -15,7 +16,6 @@ ) from sqlbuild.python_nodes.models import LoaderDefinition from sqlbuild.python_nodes.types import LoaderColumnSpec -from sqlbuild.spec.contracts.models import SourceColumnEntry def apply_loader( diff --git a/src/sqlbuild/python_nodes/main/apply_asset.py b/src/sqlbuild/python_nodes/main/apply_asset.py index d077bbe786..708ea4220c 100644 --- a/src/sqlbuild/python_nodes/main/apply_asset.py +++ b/src/sqlbuild/python_nodes/main/apply_asset.py @@ -5,6 +5,7 @@ from collections.abc import Callable, Mapping, Sequence from typing import Any, cast +from sqlbuild._native import SourceColumnEntry from sqlbuild.python_nodes._helpers.attachment import attach_definition from sqlbuild.python_nodes._helpers.column_normalization import normalize_columns from sqlbuild.python_nodes._helpers.dependency_normalization import ( @@ -19,7 +20,6 @@ SqlResourceRef, ) from sqlbuild.python_nodes.types import ColumnLineageRefSpec, PythonNodeColumnSpec -from sqlbuild.spec.contracts.models import SourceColumnEntry def apply_asset( diff --git a/src/sqlbuild/python_nodes/models.py b/src/sqlbuild/python_nodes/models.py index f297bdd362..ac50102945 100644 --- a/src/sqlbuild/python_nodes/models.py +++ b/src/sqlbuild/python_nodes/models.py @@ -6,13 +6,11 @@ from collections.abc import Callable, Mapping from dataclasses import dataclass, field -from sqlbuild.compiler.auditing.models import MeasurementThresholds +from sqlbuild._native import MeasurementThresholds, SourceColumnEntry, SourceWriteStrategy from sqlbuild.compiler.auditing.types import AuditSeverity from sqlbuild.errors.contracts.exceptions import SharedInputError from sqlbuild.python_nodes.constants import RELATION_RENDERER from sqlbuild.python_nodes.types import PythonCheckSeverity, SqlResourceRefKind -from sqlbuild.spec.contracts.models import SourceColumnEntry -from sqlbuild.spec.contracts.types import SourceWriteStrategy @dataclass(frozen=True) diff --git a/src/sqlbuild/runtime/event_exporting/constants.py b/src/sqlbuild/runtime/event_exporting/constants.py index 227df104bd..8135309f9b 100644 --- a/src/sqlbuild/runtime/event_exporting/constants.py +++ b/src/sqlbuild/runtime/event_exporting/constants.py @@ -1,5 +1,6 @@ """Lifecycle-event sink declaration constants.""" +from sqlbuild._native import EventExportSeverity from sqlbuild.runtime.event_exporting.types import LifecycleEventKind from sqlbuild.spec.contracts.types import EventExportSeverity diff --git a/src/sqlbuild/runtime/event_exporting/main/lifecycle_event_sink.py b/src/sqlbuild/runtime/event_exporting/main/lifecycle_event_sink.py index 6b3503a88c..d3a8db86bd 100644 --- a/src/sqlbuild/runtime/event_exporting/main/lifecycle_event_sink.py +++ b/src/sqlbuild/runtime/event_exporting/main/lifecycle_event_sink.py @@ -6,6 +6,7 @@ from collections.abc import Callable, Iterable from typing import Any, cast +from sqlbuild._native import EventExportSeverity from sqlbuild.python_nodes.main.attach_definition import attach_definition from sqlbuild.runtime.event_exporting.constants import ( EVENT_EXPORT_KINDS, @@ -15,7 +16,6 @@ from sqlbuild.runtime.event_exporting.exceptions import EventExporterInputError from sqlbuild.runtime.event_exporting.models import LifecycleEventSinkDefinition from sqlbuild.runtime.event_exporting.types import LifecycleEventKind -from sqlbuild.spec.contracts.types import EventExportSeverity def lifecycle_event_sink( diff --git a/src/sqlbuild/spec/contracts/constants.py b/src/sqlbuild/spec/contracts/constants.py index d20a1ff03e..29539a90e6 100644 --- a/src/sqlbuild/spec/contracts/constants.py +++ b/src/sqlbuild/spec/contracts/constants.py @@ -2,7 +2,7 @@ from __future__ import annotations -from sqlbuild.spec.contracts.models import SeedCsvSettings +from sqlbuild._native import SeedCsvSettings DEFAULT_SEED_CSV_SETTINGS: SeedCsvSettings = SeedCsvSettings() ZERO_DAY_CURSOR_DURATION: str = "0d" diff --git a/src/sqlbuild/spec/contracts/models.py b/src/sqlbuild/spec/contracts/models.py deleted file mode 100644 index 84b803ef0d..0000000000 --- a/src/sqlbuild/spec/contracts/models.py +++ /dev/null @@ -1,634 +0,0 @@ -"""Structured project specification models.""" - -from __future__ import annotations - -from dataclasses import dataclass, field -from decimal import Decimal -from pathlib import Path -from typing import Literal - -from sqlbuild._native import CollectionRendering -from sqlbuild.compiler.auditing.models import MeasurementThresholds -from sqlbuild.compiler.auditing.types import AuditSeverity -from sqlbuild.cost.constants import DEFAULT_USD_PER_CREDIT -from sqlbuild.spec.contracts.types import ( - ColumnContractMode, - EventExportSeverity, - FutureCursorAction, - MicrobatchLimitAction, - MissingMigrationOriginPolicy, - RetentionDecreasePolicy, - SourceFreshnessStrategy, - SourceFreshnessValueKind, - SourceWriteStrategy, - TableType, - TableTypeDowngradePolicy, - TableTypeSource, - TimeTravelRetentionSource, - WarehouseGroup, -) - - -@dataclass(frozen=True) -class ClonePolicy: - """Environment clone policy.""" - - allow_as_clone_origin: bool = False - allow_as_clone_destination: bool = False - - -@dataclass(frozen=True) -class LocalClonePolicy: - """Local environment clone policy overrides.""" - - allow_as_clone_origin: bool | None = None - allow_as_clone_destination: bool | None = None - - -@dataclass(frozen=True) -class AuthoredTimeTravelRetention: - """One authored time-travel retention policy.""" - - desired_days: int | None = None - unmanaged: bool = False - - -@dataclass(frozen=True) -class MaterializationRetentionDefaults: - """Retention defaults for one built-in materialization.""" - - time_travel_retention: AuthoredTimeTravelRetention | None = None - table_type: TableType | None = None - - -@dataclass(frozen=True) -class MaterializationDefaultsConfig: - """Project retention defaults by materialization type.""" - - table: MaterializationRetentionDefaults = field( - default_factory=MaterializationRetentionDefaults - ) - incremental: MaterializationRetentionDefaults = field( - default_factory=MaterializationRetentionDefaults - ) - snapshot: MaterializationRetentionDefaults = field( - default_factory=MaterializationRetentionDefaults - ) - - -@dataclass(frozen=True) -class ResolvedTimeTravelRetention: - """Effective retention policy attached to a compiled model.""" - - desired_days: int | None = None - unmanaged: bool = True - source: TimeTravelRetentionSource | None = None - - -@dataclass(frozen=True) -class ResolvedTableType: - """Effective Snowflake table type attached to a compiled model.""" - - value: TableType = TableType.TRANSIENT - source: TableTypeSource = TableTypeSource.DEFAULT - declared: bool = False - - -@dataclass(frozen=True) -class ExecutionLimitsConfig: - """Optional target-scoped build execution limits.""" - - max_models: int | None = None - max_duration: str | None = None - max_duration_seconds: int | None = None - remediation: str | None = None - - -@dataclass(frozen=True) -class TargetDiffConfig: - """Optional target-scoped data-diff limits.""" - - max_full_rows: int | Literal["unlimited"] | None = None - - -@dataclass(frozen=True) -class TargetWarehousesConfig: - """Optional per-command-group default warehouses for one target.""" - - build: str | None = None - query: str | None = None - - def for_group(self, group: WarehouseGroup) -> str | None: - return self.build if group == WarehouseGroup.BUILD else self.query - - @property - def configured(self) -> bool: - return self.build is not None or self.query is not None - - -@dataclass(frozen=True) -class TargetConfig: - """One named target configuration.""" - - connection: dict[str, object] = field(default_factory=dict) - connection_name: str | None = None - vars: dict[str, str] = field(default_factory=dict) - database: str | None = None - schema: str | None = None - loader_schema: str | None = None - defer_sources_to: str | None = None - defer_clone_from: str | None = None - clone: ClonePolicy = field(default_factory=ClonePolicy) - compile_cache: bool | None = None - time_travel_retention: AuthoredTimeTravelRetention | None = None - time_travel_retention_by_materialization: dict[str, AuthoredTimeTravelRetention] = field( - default_factory=dict - ) - owns_time_travel_retention_namespace: bool = False - default_table_type: TableType | None = None - table_type_downgrade: TableTypeDowngradePolicy = TableTypeDowngradePolicy.REQUIRE_CONFIRMATION - time_travel_retention_decrease: RetentionDecreasePolicy = RetentionDecreasePolicy.DENY - missing_migration_origin: MissingMigrationOriginPolicy = MissingMigrationOriginPolicy.ALLOW - execution_limits: ExecutionLimitsConfig = field(default_factory=ExecutionLimitsConfig) - diff: TargetDiffConfig = field(default_factory=TargetDiffConfig) - warehouses: TargetWarehousesConfig = field(default_factory=TargetWarehousesConfig) - - -@dataclass(frozen=True) -class LocalTargetConfig: - """Local developer overrides for one named target.""" - - connection: dict[str, object] = field(default_factory=dict) - connection_name: str | None = None - vars: dict[str, str] = field(default_factory=dict) - database: str | None = None - schema: str | None = None - loader_schema: str | None = None - defer_sources_to: str | None = None - defer_clone_from: str | None = None - clone: LocalClonePolicy = field(default_factory=LocalClonePolicy) - compile_cache: bool | None = None - time_travel_retention: AuthoredTimeTravelRetention | None = None - time_travel_retention_by_materialization: dict[str, AuthoredTimeTravelRetention] | None = None - owns_time_travel_retention_namespace: bool | None = None - default_table_type: TableType | None = None - table_type_downgrade: TableTypeDowngradePolicy | None = None - time_travel_retention_decrease: RetentionDecreasePolicy | None = None - missing_migration_origin: MissingMigrationOriginPolicy | None = None - execution_limits: ExecutionLimitsConfig = field(default_factory=ExecutionLimitsConfig) - diff: TargetDiffConfig = field(default_factory=TargetDiffConfig) - warehouses: TargetWarehousesConfig = field(default_factory=TargetWarehousesConfig) - - -@dataclass(frozen=True) -class SettingsConfig: - """Global feature toggles.""" - - sql_analysis: bool = True - require_sql_analysis: bool = False - query_change_tracking: bool = True - column_contract_mode: ColumnContractMode = ColumnContractMode.IMPLICIT - concurrency: int = 1 - auto_load_sources: bool = True - microbatch_concurrency: bool = False - microbatch_unaccounted_partition_policy: str = "synthesize" - table_promotion_mode: str | None = None - default_audit_severity: AuditSeverity | None = None - default_audit_run_scope: str | None = None - - @property - def sql_validation(self) -> bool: - """Compatibility alias for the unified SQL analysis gate.""" - - return self.sql_analysis - - -@dataclass(frozen=True) -class ScopesConfig: - """Declaration scope policy configuration.""" - - enforce_placement: bool = True - - -@dataclass(frozen=True) -class ReferencesConfig: - """Explicit-reference enforcement policy configuration.""" - - enforce_explicit: bool = True - - -@dataclass(frozen=True) -class CostConfig: - """Snowflake compute estimate configuration.""" - - usd_per_credit: Decimal = DEFAULT_USD_PER_CREDIT - usd_per_credit_is_default: bool = True - - -@dataclass(frozen=True) -class ConstantsConfig: - """Project-wide constant rendering configuration.""" - - collection_rendering: CollectionRendering = CollectionRendering.VALUE_LIST - - -@dataclass(frozen=True) -class FutureCursorsConfig: - """Safety policy for automatically discovered future cursor watermarks.""" - - max_distance: str | None = None - action: FutureCursorAction = FutureCursorAction.ERROR - - -@dataclass(frozen=True) -class StartCursorsConfig: - """Safety policy for automatically discovered incremental starts.""" - - max_ahead: str | None = None - action: FutureCursorAction = FutureCursorAction.ERROR - - -@dataclass(frozen=True) -class CursorsConfig: - """Project-wide cursor safety configuration.""" - - future: FutureCursorsConfig = field(default_factory=FutureCursorsConfig) - start: StartCursorsConfig = field(default_factory=StartCursorsConfig) - - -@dataclass(frozen=True) -class MicrobatchLimitsConfig: - """Per-model microbatch planning safety limit.""" - - max_batches: int | None = None - action: MicrobatchLimitAction = MicrobatchLimitAction.ERROR - - -@dataclass(frozen=True) -class MicrobatchesConfig: - """Project-wide microbatch configuration.""" - - limits: MicrobatchLimitsConfig = field(default_factory=MicrobatchLimitsConfig) - - -@dataclass(frozen=True) -class DefaultsConfig: - """Project-wide resource defaults.""" - - materialized: str | None = None - database: str | None = None - schema: str | None = None - seed_database: str | None = None - seed_schema: str | None = None - function_database: str | None = None - function_schema: str | None = None - incremental_strategy: str | None = None - incremental_mode: str | None = None - microbatch_strategy: str | None = None - cursor_watermark_mode: str | None = None - merge_exclude_columns: tuple[str, ...] = field(default_factory=tuple) - full_refresh: bool | None = None - append_cursor_inclusive: object | None = None - cursor_start: object | None = None - cursor_end: object | None = None - cursor_start_max_ahead: str | None = None - cursor_start_max_action: str | None = None - cursor_future_max_distance: str | None = None - cursor_future_action: str | None = None - lookback: str | None = None - batch_size: str | int | None = None - batch_concurrency: int | None = None - max_microbatches: int | None = None - unaccounted_partition_policy: str | None = None - replay_on_change: str | None = None - row_diff_exclude_columns: tuple[str, ...] = field(default_factory=tuple) - row_diff_tolerances: dict[str, object] = field(default_factory=dict) - row_diff_sample_rows: int | None = None - row_diff_sample_seed: int | None = None - tags: tuple[str, ...] = field(default_factory=tuple) - contract: str | None = None - pre_hooks: object | None = None - post_hooks: object | None = None - - -@dataclass(frozen=True) -class DiffConfig: - """Project-wide data-diff configuration.""" - - query_artifact_ttl: str = "24h" - - -@dataclass(frozen=True) -class JanitorConfig: - """Janitor command defaults.""" - - enabled: bool = False - retention_days: int | None = None - archive_retention_days: int = 14 - direct_state_history_versions: int = 20 - delete_tracked_only: bool = True - exclude_patterns: tuple[str, ...] = field(default_factory=tuple) - - -@dataclass(frozen=True) -class MigrationsConfig: - """Model migration defaults.""" - - old_name_views: str | None = "30d" - - -@dataclass(frozen=True) -class SnapshotsConfig: - """Snapshot materialization safety defaults.""" - - current_state_full_refresh: str = "deny" - historical_full_refresh: str = "require_confirmation" - schema_change: str = "append_new_columns" - wildcard_check_schema_change: str = "require_confirmation" - - -@dataclass(frozen=True) -class ScenarioSnapshotLimitsConfig: - """Scenario snapshot capture safety limits.""" - - max_rows_per_relation: int | None = None - max_total_rows: int | None = None - max_bytes_per_relation: int | None = None - max_total_bytes: int | None = None - - -@dataclass(frozen=True) -class ScenarioConfig: - """Scenario command configuration.""" - - run_namespace: str | None = None - local_type_overrides: dict[str, dict[str, str]] = field(default_factory=dict) - snapshot_limits: ScenarioSnapshotLimitsConfig = field( - default_factory=ScenarioSnapshotLimitsConfig - ) - - -@dataclass(frozen=True) -class DbtConfig: - """dbt interop configuration.""" - - project_dir: str | None = None - profiles_dir: str | None = None - target: str | None = None - target_path: str | None = None - vars: dict[str, object] = field(default_factory=dict) - - -@dataclass(frozen=True) -class LocalDbtConfig: - """Local dbt interop configuration overrides.""" - - target: str | None = None - vars: dict[str, object] = field(default_factory=dict) - - -@dataclass(frozen=True) -class LifecycleEventSinkFilterConfig: - """Destination-neutral runtime narrowing for lifecycle-event sinks.""" - - event_kinds: frozenset[str] | None = None - min_severity: EventExportSeverity | None = None - - -@dataclass(frozen=True) -class LifecycleEventSinksConfig: - """Global and named lifecycle-event sink runtime filters and delivery settings.""" - - defaults: LifecycleEventSinkFilterConfig = field(default_factory=LifecycleEventSinkFilterConfig) - named: dict[str, LifecycleEventSinkFilterConfig] = field(default_factory=dict) - shutdown_timeout: str | None = None - shutdown_timeout_seconds: int | None = None - - -@dataclass(frozen=True) -class SinksConfig: - """Typed destination sink configuration.""" - - lifecycle: LifecycleEventSinksConfig = field(default_factory=LifecycleEventSinksConfig) - - -@dataclass(frozen=True) -class ProjectConfig: - """Shared project configuration loaded from sqlbuild_project.toml.""" - - name: str - adapter: str - default_target: str | None = None - connection: dict[str, object] = field(default_factory=dict) - connections: dict[str, dict[str, object]] = field(default_factory=dict) - settings: SettingsConfig = field(default_factory=SettingsConfig) - scopes: ScopesConfig = field(default_factory=ScopesConfig) - references: ReferencesConfig = field(default_factory=ReferencesConfig) - cost: CostConfig = field(default_factory=CostConfig) - constants: ConstantsConfig = field(default_factory=ConstantsConfig) - cursors: CursorsConfig = field(default_factory=CursorsConfig) - microbatches: MicrobatchesConfig = field(default_factory=MicrobatchesConfig) - defaults: DefaultsConfig = field(default_factory=DefaultsConfig) - diff: DiffConfig = field(default_factory=DiffConfig) - materialization_defaults: MaterializationDefaultsConfig = field( - default_factory=MaterializationDefaultsConfig - ) - path_defaults: dict[str, dict[str, object]] = field(default_factory=dict) - vars: dict[str, str] = field(default_factory=dict) - targets: dict[str, TargetConfig] = field(default_factory=dict) - janitor: JanitorConfig = field(default_factory=JanitorConfig) - migrations: MigrationsConfig = field(default_factory=MigrationsConfig) - snapshots: SnapshotsConfig = field(default_factory=SnapshotsConfig) - scenario: ScenarioConfig = field(default_factory=ScenarioConfig) - dbt: DbtConfig = field(default_factory=DbtConfig) - sinks: SinksConfig = field(default_factory=SinksConfig) - - -@dataclass(frozen=True) -class LocalConfig: - """Local developer overrides from sqlbuild_local.toml.""" - - target: str | None = None - adapter: str | None = None - connection: dict[str, object] = field(default_factory=dict) - connections: dict[str, dict[str, object]] = field(default_factory=dict) - targets: dict[str, LocalTargetConfig] = field(default_factory=dict) - settings: SettingsConfig = field(default_factory=SettingsConfig) - setting_overrides: frozenset[str] = field(default_factory=frozenset) - vars: dict[str, str] = field(default_factory=dict) - dbt: LocalDbtConfig = field(default_factory=LocalDbtConfig) - scenario: ScenarioConfig = field(default_factory=ScenarioConfig) - - -@dataclass(frozen=True) -class SourceLocation: - """An authored source location for compiler diagnostics.""" - - path: Path - line: int - column: int - end_line: int | None = None - end_column: int | None = None - - -@dataclass(frozen=True) -class SeedCsvSettings: - """CSV reader settings for a seed file.""" - - delimiter: str | None = None - quotechar: str | None = None - doublequote: bool | None = None - escapechar: str | None = None - skipinitialspace: bool | None = None - lineterminator: str | None = None - encoding: str | None = None - na_values: tuple[object, ...] | dict[str, tuple[object, ...]] | None = None - keep_default_na: bool | None = None - - -@dataclass(frozen=True) -class SchemaAuditInstance: - """One audit instance attached to model, column, or seed metadata.""" - - definition_name: str - arguments: dict[str, object] = field(default_factory=dict) - name: str | None = None - description: str | None = None - severity: AuditSeverity | None = None - run_scope: str | None = None - always_run: bool = False - thresholds: MeasurementThresholds | None = None - minimum_samples: int | None = None - evidence_limit: int | None = None - location: SourceLocation | None = field(default=None, compare=False, repr=False) - - -@dataclass(frozen=True) -class SchemaColumn: - """One declared model, seed, or source column entry.""" - - name: str - type: str | None = None - nullable: bool | None = None - description: str | None = None - meta: dict[str, object] = field(default_factory=dict) - audits: tuple[SchemaAuditInstance, ...] = field(default_factory=tuple) - location: SourceLocation | None = None - migrate_from: str | None = None - - -@dataclass(frozen=True) -class SchemaDynamicColumnFamily: - """One declared family of columns generated by a runtime-dynamic pivot.""" - - name: str - pivot_column: str - value_column: str - aggregate: str - type: str - name_pattern: str | None = None - - -@dataclass(frozen=True) -class SchemaModelEntry: - """One model metadata entry normalized from MODEL(...).""" - - name: str - model_schema: str | None = None - description: str | None = None - type_enforcement: bool | None = None - meta: dict[str, object] = field(default_factory=dict) - columns: tuple[SchemaColumn, ...] = field(default_factory=tuple) - dynamic_columns: tuple[SchemaDynamicColumnFamily, ...] = field(default_factory=tuple) - audits: tuple[SchemaAuditInstance, ...] = field(default_factory=tuple) - tags: tuple[str, ...] = field(default_factory=tuple) - - -@dataclass(frozen=True) -class SchemaSeedEntry: - """One seed metadata entry from seed YAML.""" - - name: str - description: str | None = None - database: str | None = None - schema: str | None = None - meta: dict[str, object] = field(default_factory=dict) - csv_settings: SeedCsvSettings = field(default_factory=SeedCsvSettings) - columns: tuple[SchemaColumn, ...] = field(default_factory=tuple) - tags: tuple[str, ...] = field(default_factory=tuple) - audits: tuple[SchemaAuditInstance, ...] = field(default_factory=tuple) - - -@dataclass(frozen=True) -class IntegrationLoaderConfig: - """Typed configuration for one declarative integration loader.""" - - kind: str - config: object - - -@dataclass(frozen=True) -class SourceFreshnessAgePolicy: - """Age-based source freshness warning/error thresholds.""" - - warn_after: str | None = None - error_after: str | None = None - - -@dataclass(frozen=True) -class SourceFreshnessConfig: - """Configured source freshness observation for planning.""" - - strategy: SourceFreshnessStrategy - value_kind: SourceFreshnessValueKind | None = None - column: str | None = None - query: str | None = None - filter: str | None = None - lag_tolerance: str | None = None - age_policy: SourceFreshnessAgePolicy | None = None - - -@dataclass(frozen=True) -class SourceColumnEntry: - """One source column entry from sources/*.yml.""" - - name: str - type: str | None = None - nullable: bool | None = None - description: str | None = None - meta: dict[str, object] = field(default_factory=dict) - audits: tuple[SchemaAuditInstance, ...] = field(default_factory=tuple) - - -@dataclass(frozen=True) -class SourceEntry: - """One source declaration from sources/*.yml.""" - - name: str - database: str | None = None - schema: str | None = None - table: str | None = None - loader: str | None = None - managed: bool = False - integration_loader: IntegrationLoaderConfig | None = None - freshness: SourceFreshnessConfig | None = None - write_strategy: SourceWriteStrategy | None = None - load_batch_size: int | None = None - cursor_column: str | None = None - unique_key: tuple[str, ...] = field(default_factory=tuple) - expression: str | None = None - description: str | None = None - type_enforcement: bool | None = None - contract: str | None = None - meta: dict[str, object] = field(default_factory=dict) - columns: tuple[SourceColumnEntry, ...] = field(default_factory=tuple) - audits: tuple[SchemaAuditInstance, ...] = field(default_factory=tuple) - - -@dataclass(frozen=True) -class LoaderDestinationParts: - """Parsed warehouse relation parts for a loader destination.""" - - database: str | None - schema: str | None - table: str diff --git a/src/sqlbuild/spec/contracts/types.py b/src/sqlbuild/spec/contracts/types.py deleted file mode 100644 index 38e32f06c2..0000000000 --- a/src/sqlbuild/spec/contracts/types.py +++ /dev/null @@ -1,111 +0,0 @@ -"""Project specification types.""" - -from __future__ import annotations - -from enum import StrEnum - - -class EventExportSeverity(StrEnum): - """Severity used to filter exported lifecycle events.""" - - DEBUG = "debug" - INFO = "info" - WARNING = "warning" - ERROR = "error" - - -class WarehouseGroup(StrEnum): - """Command group that selects a target's default warehouse.""" - - BUILD = "build" - QUERY = "query" - - -class SourceWriteStrategy(StrEnum): - APPEND = "append" - DELETE_INSERT = "delete_insert" - MERGE = "merge" - TABLE = "table" - - -class FutureCursorAction(StrEnum): - """Response when an effective cursor lies beyond the future horizon.""" - - CAP = "cap" - ERROR = "error" - - -class MicrobatchLimitAction(StrEnum): - """Response when one model plans more than the configured batch limit.""" - - CAP_FROM_END = "cap_from_end" - CAP_FROM_START = "cap_from_start" - ERROR = "error" - WARN = "warn" - - -class ColumnContractMode(StrEnum): - """Default contract behavior for model column declarations.""" - - IMPLICIT = "implicit" - EXPLICIT = "explicit" - - -class SourceFreshnessStrategy(StrEnum): - ADAPTER = "adapter" - COLUMN = "column" - SQL = "sql" - - -class SourceFreshnessValueKind(StrEnum): - TIMESTAMP = "timestamp" - INTEGER = "integer" - STRING = "string" - - -class TimeTravelRetentionSource(StrEnum): - """Authored layer that supplied the effective retention policy.""" - - TARGET = "target" - MATERIALIZATION = "materialization" - MODEL = "model" - - -class TableType(StrEnum): - """Snowflake managed table type.""" - - PERMANENT = "permanent" - TRANSIENT = "transient" - - -class TableTypeSource(StrEnum): - """Authored layer that supplied the effective table type.""" - - DEFAULT = "default" - TARGET = "target" - MATERIALIZATION = "materialization" - MODEL = "model" - - -class TableTypeDowngradePolicy(StrEnum): - """Permanent-to-transient conversion policy.""" - - DENY = "deny" - REQUIRE_CONFIRMATION = "require_confirmation" - ALLOW = "allow" - - -class MissingMigrationOriginPolicy(StrEnum): - """Policy for a declared migrate_from whose origin does not exist in a target.""" - - DENY = "deny" - REQUIRE_CONFIRMATION = "require_confirmation" - ALLOW = "allow" - - -class RetentionDecreasePolicy(StrEnum): - """Policy for lowering live time-travel retention.""" - - DENY = "deny" - REQUIRE_CONFIRMATION = "require_confirmation" - ALLOW = "allow" diff --git a/src/sqlbuild/sql_values/models.py b/src/sqlbuild/sql_values/models.py deleted file mode 100644 index e548e25395..0000000000 --- a/src/sqlbuild/sql_values/models.py +++ /dev/null @@ -1,37 +0,0 @@ -"""Immutable models for validated and authored typed SQL values.""" - -from __future__ import annotations - -from dataclasses import dataclass - -from sqlbuild.sql_values.types import SqlValueKind, SqlValuePayload - - -@dataclass(frozen=True) -class SqlLogicalType: - """Logical type inferred for a normalized SQL value.""" - - kind: SqlValueKind - element_type: SqlLogicalType | None = None - - @property - def display_name(self) -> str: - """Return a stable human-readable type name.""" - - if self.element_type is None: - return self.kind.value - return f"{self.kind.value}<{self.element_type.display_name}>" - - -@dataclass(frozen=True) -class SqlValue: - """One validated, canonical SQL value.""" - - logical_type: SqlLogicalType - value: SqlValuePayload - - @property - def kind(self) -> SqlValueKind: - """Return the outer logical kind.""" - - return self.logical_type.kind diff --git a/src/sqlbuild/sql_values/types.py b/src/sqlbuild/sql_values/types.py deleted file mode 100644 index 2e6643497a..0000000000 --- a/src/sqlbuild/sql_values/types.py +++ /dev/null @@ -1,36 +0,0 @@ -"""Immutable logical values shared by SQLBuild-authored SQL features.""" - -from __future__ import annotations - -from decimal import Decimal -from enum import StrEnum - - -class SqlValueKind(StrEnum): - """Closed logical kinds accepted by the typed SQL value boundary.""" - - STRING = "string" - INTEGER = "integer" - BOOLEAN = "boolean" - FLOAT = "float" - DECIMAL = "decimal" - NULL = "null" - LIST = "list" - SET = "set" - OBJECT = "object" - - -class StateSqlValueType(StrEnum): - """Closed physical value kinds used by warehouse state columns.""" - - STRING = "string" - INTEGER = "integer" - BOOLEAN = "boolean" - TIMESTAMP = "timestamp" - TEXT_TIMESTAMP = "text_timestamp" - DATE = "date" - JSON = "json" - - -type SqlScalar = str | int | bool | float | Decimal | None -type SqlValuePayload = SqlScalar | tuple[object, ...] diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/janitor/helpers.py b/tests/e2e/src/sqlbuild/cli/commands/main/janitor/helpers.py index f785980e47..dd575d49d8 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/janitor/helpers.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/janitor/helpers.py @@ -10,13 +10,13 @@ from types import MappingProxyType from sqlbuild._native import ( + Fingerprint, + SourceFreshnessRecord, SourceFreshnessRenderers, write_fingerprint, write_source_freshness_records, ) from sqlbuild.adapters.duckdb.classes.duckdb_adapter import DuckDbAdapter -from sqlbuild.compiler.fingerprints.models import Fingerprint -from sqlbuild.compiler.source_freshness.models import SourceFreshnessRecord from tests.e2e.src.sqlbuild.cli.commands.shared.helpers import ( execute_duckdb, prepare_inline_project, diff --git a/tests/integration/src/sqlbuild/adapters/bigquery/test_client.py b/tests/integration/src/sqlbuild/adapters/bigquery/test_client.py index f55f2a1e62..7a459fbc21 100644 --- a/tests/integration/src/sqlbuild/adapters/bigquery/test_client.py +++ b/tests/integration/src/sqlbuild/adapters/bigquery/test_client.py @@ -8,6 +8,9 @@ import pytest from sqlbuild._native import ( + Fingerprint, + FingerprintSet, + SourceFreshnessValueKind, normalize_source_freshness_data_version, read_latest_fingerprints, source_freshness_data_version_hash, @@ -28,9 +31,7 @@ from sqlbuild.adapter.contract.types import FunctionNullabilityRule from sqlbuild.adapters.bigquery.classes.bigquery_adapter import BigQueryAdapter from sqlbuild.compiler.fingerprints.constants import FINGERPRINT_TABLE_NAME -from sqlbuild.compiler.fingerprints.models import Fingerprint, FingerprintSet from sqlbuild.compiler.lineage.types import InferredNullability -from sqlbuild.spec.contracts.types import SourceFreshnessValueKind from tests.integration.src.sqlbuild.adapters.bigquery._test_types import ( BigQueryBuildFlowTestCase, BigQueryDeleteInsertCursorTestCase, diff --git a/tests/integration/src/sqlbuild/adapters/databricks/test_client.py b/tests/integration/src/sqlbuild/adapters/databricks/test_client.py index 13f1b813ed..8ffa7946b9 100644 --- a/tests/integration/src/sqlbuild/adapters/databricks/test_client.py +++ b/tests/integration/src/sqlbuild/adapters/databricks/test_client.py @@ -7,6 +7,9 @@ import pytest from sqlbuild._native import ( + Fingerprint, + FingerprintSet, + SourceFreshnessValueKind, normalize_source_freshness_data_version, read_latest_fingerprints, source_freshness_data_version_hash, @@ -27,9 +30,7 @@ from sqlbuild.adapter.contract.types import FunctionNullabilityRule from sqlbuild.adapters.databricks.classes.databricks_adapter import DatabricksAdapter from sqlbuild.compiler.fingerprints.constants import FINGERPRINT_TABLE_NAME -from sqlbuild.compiler.fingerprints.models import Fingerprint, FingerprintSet from sqlbuild.compiler.lineage.types import InferredNullability -from sqlbuild.spec.contracts.types import SourceFreshnessValueKind from tests.integration.src.sqlbuild.adapters.databricks._test_types import ( DatabricksBuildFlowTestCase, DatabricksExpressionNullabilityRuleTestCase, diff --git a/tests/integration/src/sqlbuild/adapters/duckdb/_test_types.py b/tests/integration/src/sqlbuild/adapters/duckdb/_test_types.py index abae01ff09..38119cc4da 100644 --- a/tests/integration/src/sqlbuild/adapters/duckdb/_test_types.py +++ b/tests/integration/src/sqlbuild/adapters/duckdb/_test_types.py @@ -1,5 +1,6 @@ from dataclasses import dataclass, field +from sqlbuild._native import SeedCsvSettings from sqlbuild.adapter.contract.models import ( ColumnInfo, CursorValue, @@ -9,7 +10,6 @@ SchemaDiffResult, ) from sqlbuild.compiler.lineage.types import InferredNullability -from sqlbuild.spec.contracts.models import SeedCsvSettings @dataclass(frozen=True) diff --git a/tests/integration/src/sqlbuild/adapters/duckdb/test_client.py b/tests/integration/src/sqlbuild/adapters/duckdb/test_client.py index 2462ff668c..606a918d2c 100644 --- a/tests/integration/src/sqlbuild/adapters/duckdb/test_client.py +++ b/tests/integration/src/sqlbuild/adapters/duckdb/test_client.py @@ -7,6 +7,7 @@ import duckdb import pytest +from sqlbuild._native import SeedCsvSettings from sqlbuild.adapter.contract.classes.statement_recorder import StatementRecorder from sqlbuild.adapter.contract.models import ( ColumnInfo, @@ -23,7 +24,6 @@ from sqlbuild.adapter.contract.types import FunctionNullabilityRule from sqlbuild.adapters.duckdb.classes.duckdb_adapter import DuckDbAdapter from sqlbuild.compiler.lineage.types import InferredNullability -from sqlbuild.spec.contracts.models import SeedCsvSettings from tests.integration.src.sqlbuild.adapters.duckdb._test_types import ( CompositeRowDiffSamplingTestCase, ConnectSettingsTestCase, diff --git a/tests/integration/src/sqlbuild/adapters/snapshot_sql_rendering/helpers.py b/tests/integration/src/sqlbuild/adapters/snapshot_sql_rendering/helpers.py index a138efe157..b7015b1a08 100644 --- a/tests/integration/src/sqlbuild/adapters/snapshot_sql_rendering/helpers.py +++ b/tests/integration/src/sqlbuild/adapters/snapshot_sql_rendering/helpers.py @@ -13,6 +13,7 @@ import duckdb +from sqlbuild._native import TableType from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter from sqlbuild.adapter.contract.models import SnapshotChangeTarget from sqlbuild.adapters.bigquery.classes.bigquery_adapter import BigQueryAdapter @@ -21,7 +22,6 @@ from sqlbuild.adapters.motherduck.classes.motherduck_adapter import MotherDuckAdapter from sqlbuild.adapters.postgres.classes.postgres_adapter import PostgresAdapter from sqlbuild.adapters.snowflake.classes.snowflake_adapter import SnowflakeAdapter -from sqlbuild.spec.contracts.types import TableType from tests.integration.src.sqlbuild.adapters.snapshot_sql_rendering._test_types import ( SnapshotExecutionTestCase, ) diff --git a/tests/integration/src/sqlbuild/adapters/snowflake/_test_types.py b/tests/integration/src/sqlbuild/adapters/snowflake/_test_types.py index 75e2928189..24169d45bf 100644 --- a/tests/integration/src/sqlbuild/adapters/snowflake/_test_types.py +++ b/tests/integration/src/sqlbuild/adapters/snowflake/_test_types.py @@ -1,6 +1,7 @@ from dataclasses import dataclass, field from datetime import datetime +from sqlbuild._native import TableType from sqlbuild.adapter.contract.models import ( ColumnInfo, QueryResult, @@ -10,7 +11,6 @@ ) from sqlbuild.compiler.lineage.types import InferredNullability from sqlbuild.cost.types import CostStatus -from sqlbuild.spec.contracts.types import TableType @dataclass(frozen=True) diff --git a/tests/integration/src/sqlbuild/adapters/snowflake/test_client.py b/tests/integration/src/sqlbuild/adapters/snowflake/test_client.py index 03070bec2b..5d5c601f5d 100644 --- a/tests/integration/src/sqlbuild/adapters/snowflake/test_client.py +++ b/tests/integration/src/sqlbuild/adapters/snowflake/test_client.py @@ -8,6 +8,7 @@ import pytest from sqlbuild._native import ( + SourceFreshnessValueKind, normalize_source_freshness_data_version, source_freshness_data_version_hash, ) @@ -26,7 +27,6 @@ from sqlbuild.adapter.contract.types import FunctionNullabilityRule from sqlbuild.adapters.snowflake.classes.snowflake_adapter import SnowflakeAdapter from sqlbuild.compiler.lineage.types import InferredNullability -from sqlbuild.spec.contracts.types import SourceFreshnessValueKind from tests.integration.src.sqlbuild.adapters.snowflake._test_types import ( SnowflakeBuildFlowTestCase, SnowflakeExpressionNullabilityRuleTestCase, diff --git a/tests/integration/src/sqlbuild/adapters/snowflake/test_table_type_conversion.py b/tests/integration/src/sqlbuild/adapters/snowflake/test_table_type_conversion.py index 467aaba1a1..9b609c35f3 100644 --- a/tests/integration/src/sqlbuild/adapters/snowflake/test_table_type_conversion.py +++ b/tests/integration/src/sqlbuild/adapters/snowflake/test_table_type_conversion.py @@ -6,13 +6,12 @@ import pytest -from sqlbuild._native import table_type_copy_name +from sqlbuild._native import TableType, table_type_copy_name from sqlbuild.adapter.contract.models import RelationInfo from sqlbuild.compiler.compile.models import CompiledRelationLocation from sqlbuild.compiler.planner.models import TableTypePlanEntry from sqlbuild.executor.build._helpers.retention import apply_table_type_conversion from sqlbuild.executor.clone.main._clone_relation_operation import clone_relation_by_names -from sqlbuild.spec.contracts.types import TableType from tests.integration.src.sqlbuild.adapters.snowflake._test_types import ( SnowflakeCloneTableTypeTestCase, SnowflakeTableTypeConversionTestCase, diff --git a/tests/integration/src/sqlbuild/cli/commands/main/test_load_command.py b/tests/integration/src/sqlbuild/cli/commands/main/test_load_command.py index cf4ca69fc0..faf3f957cf 100644 --- a/tests/integration/src/sqlbuild/cli/commands/main/test_load_command.py +++ b/tests/integration/src/sqlbuild/cli/commands/main/test_load_command.py @@ -20,6 +20,7 @@ LoadCommandRequest, PlanCommandRequest, ScenarioTestCommandRequest, + SourceEntry, TestCommandRequest, discover_project_inputs, format_load_execution_json, @@ -42,7 +43,6 @@ LoadExecutionResult, LoadRuntimeParams, ) -from sqlbuild.spec.contracts.models import SourceEntry from tests.integration.src.sqlbuild.cli.commands.main._test_types import ( BuildRunAutoLoadFailureTestCase, BuildRunAutoLoadFlagTestCase, diff --git a/tests/integration/src/sqlbuild/cli/commands/main/test_rules.py b/tests/integration/src/sqlbuild/cli/commands/main/test_rules.py index bd995a0a75..165d694d04 100644 --- a/tests/integration/src/sqlbuild/cli/commands/main/test_rules.py +++ b/tests/integration/src/sqlbuild/cli/commands/main/test_rules.py @@ -18,13 +18,13 @@ CustomHostObserver, DeclarationScopeBuild, LintRunResult, + SqlLexicalSyntax, active_lint_stop, rules_bulk_cache_path, ) from sqlbuild._native import run_cli as main from sqlbuild.compiler.compile.models import LoadedMacro from sqlbuild.compiler.discovery.models import DiscoveredMacroFile, DiscoveredProjectInputs -from sqlbuild.compiler.sql_analysis.models import SqlLexicalSyntax from tests.integration.src.sqlbuild.cli.commands.main._test_types import ( CustomHostFailureIntegrationTestCase, CustomHostInterruptIntegrationTestCase, diff --git a/tests/integration/src/sqlbuild/cli/output/test_measurement_integration_envelope.py b/tests/integration/src/sqlbuild/cli/output/test_measurement_integration_envelope.py index c1b903e346..aeb5db14ee 100644 --- a/tests/integration/src/sqlbuild/cli/output/test_measurement_integration_envelope.py +++ b/tests/integration/src/sqlbuild/cli/output/test_measurement_integration_envelope.py @@ -6,8 +6,9 @@ import pytest -from sqlbuild._native import IntegrationCheckResult, IntegrationResultEnvelope -from sqlbuild.compiler.auditing.models import ( +from sqlbuild._native import ( + IntegrationCheckResult, + IntegrationResultEnvelope, MeasurementThresholdBound, MeasurementThresholds, ) diff --git a/tests/integration/src/sqlbuild/compiler/analysis_session/helpers.py b/tests/integration/src/sqlbuild/compiler/analysis_session/helpers.py index 77b3164b53..9d5bf390b9 100644 --- a/tests/integration/src/sqlbuild/compiler/analysis_session/helpers.py +++ b/tests/integration/src/sqlbuild/compiler/analysis_session/helpers.py @@ -24,6 +24,7 @@ NativeModelAnalyses, NativeModelAnalysisRequest, NativePivotTables, + SchemaDynamicColumnFamily, analyze_native_model_sql, build_analysis_cache_context, build_compile_inputs, @@ -43,7 +44,6 @@ from sqlbuild.compiler.compile.models import CompactLineageFacts, InferredColumn from sqlbuild.compiler.compile.types import CompiledProjectView from sqlbuild.compiler.lineage.types import InferredNullability -from sqlbuild.spec.contracts.models import SchemaDynamicColumnFamily from tests.integration.src.sqlbuild.compiler.analysis_session._test_types import ( AdapterRuleCallbackTestCase, ) diff --git a/tests/integration/src/sqlbuild/compiler/attachments/helpers.py b/tests/integration/src/sqlbuild/compiler/attachments/helpers.py index 173909af0d..ac846e8dba 100644 --- a/tests/integration/src/sqlbuild/compiler/attachments/helpers.py +++ b/tests/integration/src/sqlbuild/compiler/attachments/helpers.py @@ -8,10 +8,14 @@ from typing import Any import sqlbuild._native as _native -from sqlbuild._native import SqlReferenceScan, find_macro_call_names, scan_sql_reference_calls +from sqlbuild._native import ( + SqlLexicalSyntax, + SqlReferenceScan, + find_macro_call_names, + scan_sql_reference_calls, +) from sqlbuild.compiler.compile.exceptions import CompileInputError from sqlbuild.compiler.compile.types import SqlTestMode -from sqlbuild.compiler.sql_analysis.models import SqlLexicalSyntax _SQL_PIECES: tuple[str, ...] = ( "SELECT * FROM t WHERE ", diff --git a/tests/integration/src/sqlbuild/compiler/auditing/main/_test_types.py b/tests/integration/src/sqlbuild/compiler/auditing/main/_test_types.py index ccfd8fdc5e..14697b5a0d 100644 --- a/tests/integration/src/sqlbuild/compiler/auditing/main/_test_types.py +++ b/tests/integration/src/sqlbuild/compiler/auditing/main/_test_types.py @@ -1,6 +1,6 @@ from dataclasses import dataclass -from sqlbuild.spec.contracts.models import SourceEntry +from sqlbuild._native import SourceEntry @dataclass(frozen=True) diff --git a/tests/integration/src/sqlbuild/compiler/auditing/main/test_rendering.py b/tests/integration/src/sqlbuild/compiler/auditing/main/test_rendering.py index 392ba22039..ebd276dc64 100644 --- a/tests/integration/src/sqlbuild/compiler/auditing/main/test_rendering.py +++ b/tests/integration/src/sqlbuild/compiler/auditing/main/test_rendering.py @@ -6,9 +6,8 @@ import pytest -from sqlbuild._native import render_audit_sql +from sqlbuild._native import SourceEntry, render_audit_sql from sqlbuild.compiler.compile.models import CompiledRelationLocation -from sqlbuild.spec.contracts.models import SourceEntry from tests.integration.src.sqlbuild.compiler.auditing.main._test_types import ( ExecuteRenderedAuditTestCase, ) diff --git a/tests/integration/src/sqlbuild/compiler/compile/_helpers/refs/test_native_references.py b/tests/integration/src/sqlbuild/compiler/compile/_helpers/refs/test_native_references.py index ac2167540c..9edef9e095 100644 --- a/tests/integration/src/sqlbuild/compiler/compile/_helpers/refs/test_native_references.py +++ b/tests/integration/src/sqlbuild/compiler/compile/_helpers/refs/test_native_references.py @@ -9,9 +9,8 @@ import pytest import sqlbuild._native as _native -from sqlbuild._native import extract_sql_references +from sqlbuild._native import SqlLexicalSyntax, extract_sql_references from sqlbuild.compiler.compile.exceptions import CompileInputError -from sqlbuild.compiler.sql_analysis.models import SqlLexicalSyntax from tests.integration.src.sqlbuild.compiler.compile._helpers.refs._test_types import ( CraftedReferenceTestCase, ReferenceExtractionTestCase, diff --git a/tests/integration/src/sqlbuild/compiler/compile/native_model_config/helpers.py b/tests/integration/src/sqlbuild/compiler/compile/native_model_config/helpers.py index 6b1afe12aa..51f29fb712 100644 --- a/tests/integration/src/sqlbuild/compiler/compile/native_model_config/helpers.py +++ b/tests/integration/src/sqlbuild/compiler/compile/native_model_config/helpers.py @@ -11,12 +11,12 @@ ModelValidationRequest, NativeModelConfigInputs, NativeModelConfigSession, + ProjectConfig, native_model_config_session, ) from sqlbuild.compiler.compile.models import CompileSqlReference from sqlbuild.compiler.discovery.models import DiscoveredSqlModelFile from sqlbuild.compiler.references.types import SqlReferenceKind -from sqlbuild.spec.contracts.models import ProjectConfig _ACCEPTED: str = "accepted" diff --git a/tests/integration/src/sqlbuild/compiler/compiled_project/helpers.py b/tests/integration/src/sqlbuild/compiler/compiled_project/helpers.py index d95f5c52d3..45aee5d974 100644 --- a/tests/integration/src/sqlbuild/compiler/compiled_project/helpers.py +++ b/tests/integration/src/sqlbuild/compiler/compiled_project/helpers.py @@ -19,6 +19,12 @@ CompileAdapterContext, CompileModelConfig, CompileProjectInputs, + LocalConfig, + MeasurementThresholdBound, + MeasurementThresholds, + ProjectConfig, + SourceColumnEntry, + SqlLexicalSyntax, assemble_project, build_compile_inputs, build_compiled_project, @@ -27,7 +33,6 @@ ) from sqlbuild.adapter.contract.models import ExpressionInferenceProfile from sqlbuild.adapters.duckdb.classes.duckdb_adapter import DuckDbAdapter -from sqlbuild.compiler.auditing.models import MeasurementThresholdBound, MeasurementThresholds from sqlbuild.compiler.compile import models as compile_models from sqlbuild.compiler.compile.models import CompileModelInput, CompilerDiagnostic from sqlbuild.compiler.compile.types import ( @@ -37,8 +42,6 @@ CompiledProjectView, ) from sqlbuild.compiler.discovery.models import DiscoveredProjectInputs, DiscoveredSqlModelFile -from sqlbuild.compiler.sql_analysis.models import SqlLexicalSyntax -from sqlbuild.spec.contracts.models import LocalConfig, ProjectConfig, SourceColumnEntry from tests.integration.src.sqlbuild.compiler.graph.helpers import write_files if TYPE_CHECKING: diff --git a/tests/integration/src/sqlbuild/compiler/contracts/helpers.py b/tests/integration/src/sqlbuild/compiler/contracts/helpers.py index f5d3da81f7..d392365a7e 100644 --- a/tests/integration/src/sqlbuild/compiler/contracts/helpers.py +++ b/tests/integration/src/sqlbuild/compiler/contracts/helpers.py @@ -13,9 +13,14 @@ import sqlbuild._native as native_module from sqlbuild._native import ( + ColumnContractMode, ContractValidationResult, DynamicColumnContractProof, DynamicColumnFamilyProof, + SchemaColumn, + SchemaDynamicColumnFamily, + SchemaModelEntry, + SourceLocation, build_project_graph, compiled_model_record, compiled_project_record, @@ -27,13 +32,6 @@ from sqlbuild.compiler.compile.types import CompiledModelView, CompiledProjectView from sqlbuild.compiler.lineage.types import InferredNullability from sqlbuild.compiler.planner.types import ContractPolicy, IncrementalMode, MaterializationType -from sqlbuild.spec.contracts.models import ( - SchemaColumn, - SchemaDynamicColumnFamily, - SchemaModelEntry, - SourceLocation, -) -from sqlbuild.spec.contracts.types import ColumnContractMode _PROJECT_TOML: str = ( 'name = "orders_contracts"\nadapter = "duckdb"\n\n[connection]\ndatabase = "orders.duckdb"\n' diff --git a/tests/integration/src/sqlbuild/compiler/fingerprints/main/_test_types.py b/tests/integration/src/sqlbuild/compiler/fingerprints/main/_test_types.py index 495329ab8c..ee54d48b01 100644 --- a/tests/integration/src/sqlbuild/compiler/fingerprints/main/_test_types.py +++ b/tests/integration/src/sqlbuild/compiler/fingerprints/main/_test_types.py @@ -1,6 +1,6 @@ from dataclasses import dataclass, field -from sqlbuild.compiler.fingerprints.models import Fingerprint +from sqlbuild._native import Fingerprint @dataclass(frozen=True) diff --git a/tests/integration/src/sqlbuild/compiler/fingerprints/main/helpers.py b/tests/integration/src/sqlbuild/compiler/fingerprints/main/helpers.py index 7402cfe234..09488d9cc3 100644 --- a/tests/integration/src/sqlbuild/compiler/fingerprints/main/helpers.py +++ b/tests/integration/src/sqlbuild/compiler/fingerprints/main/helpers.py @@ -6,9 +6,8 @@ from pathlib import Path from typing import Any -from sqlbuild._native import write_fingerprint +from sqlbuild._native import Fingerprint, write_fingerprint from sqlbuild.adapters.duckdb.classes.duckdb_adapter import DuckDbAdapter -from sqlbuild.compiler.fingerprints.models import Fingerprint def write_fingerprint_from_own_connection( diff --git a/tests/integration/src/sqlbuild/compiler/fingerprints/main/test_read_write.py b/tests/integration/src/sqlbuild/compiler/fingerprints/main/test_read_write.py index 7f9a6f1e29..60429d837e 100644 --- a/tests/integration/src/sqlbuild/compiler/fingerprints/main/test_read_write.py +++ b/tests/integration/src/sqlbuild/compiler/fingerprints/main/test_read_write.py @@ -8,6 +8,8 @@ import pytest from sqlbuild._native import ( + Fingerprint, + FingerprintSet, read_latest_fingerprints, select_latest_fingerprints, write_fingerprint, @@ -16,7 +18,6 @@ from sqlbuild.adapter.contract.types import FrameworkType from sqlbuild.adapters.duckdb.classes.duckdb_adapter import DuckDbAdapter from sqlbuild.compiler.fingerprints.constants import FINGERPRINT_TABLE_NAME -from sqlbuild.compiler.fingerprints.models import Fingerprint, FingerprintSet from tests.integration.src.sqlbuild.compiler.fingerprints.main._test_types import ( BatchWriteTestCase, InvalidDefinitionStorageTestCase, diff --git a/tests/integration/src/sqlbuild/compiler/model_config/helpers.py b/tests/integration/src/sqlbuild/compiler/model_config/helpers.py index c3001f2298..7fc4e4c664 100644 --- a/tests/integration/src/sqlbuild/compiler/model_config/helpers.py +++ b/tests/integration/src/sqlbuild/compiler/model_config/helpers.py @@ -5,7 +5,7 @@ import random import re from collections.abc import Callable, Iterator -from dataclasses import dataclass, replace +from dataclasses import dataclass, fields, replace from pathlib import Path from typing import TYPE_CHECKING, TypeGuard, cast @@ -14,6 +14,8 @@ import sqlbuild._native as _native from sqlbuild._native import ( CompileAdapterContext, + SchemaModelEntry, + SourceLocation, build_compile_inputs, discover_project_inputs, parse_audit_instances, @@ -22,9 +24,10 @@ from sqlbuild.adapters.duckdb.classes.duckdb_adapter import DuckDbAdapter from sqlbuild.compiler.compile.exceptions import CompileInputError from sqlbuild.compiler.discovery.models import DiscoveredProjectInputs, DiscoveredSqlModelFile -from sqlbuild.spec.contracts.models import SchemaModelEntry, SourceLocation if TYPE_CHECKING: + from _typeshed import DataclassInstance + from sqlbuild._native import CompileInputsView _TEXTS: tuple[object, ...] = ( @@ -361,9 +364,17 @@ def _attribute_orders(value: object) -> object: return _item_orders(_flattened(value)) +def _field_names(item: object) -> tuple[str, ...]: + return tuple(field.name for field in fields(cast("DataclassInstance", item))) + + def _item_orders(items: tuple[object, ...]) -> object: return tuple( - (type(item).__name__, tuple(vars(item)), _item_orders(getattr(item, "audits", ()))) + ( + type(item).__name__, + _field_names(item), + _item_orders(getattr(item, "audits", ())), + ) for item in items ) diff --git a/tests/integration/src/sqlbuild/compiler/pipeline/test_python_source_reads.py b/tests/integration/src/sqlbuild/compiler/pipeline/test_python_source_reads.py index 6503f5f1e2..bc1e2f795d 100644 --- a/tests/integration/src/sqlbuild/compiler/pipeline/test_python_source_reads.py +++ b/tests/integration/src/sqlbuild/compiler/pipeline/test_python_source_reads.py @@ -7,9 +7,9 @@ import pytest +from sqlbuild._native import SourceEntry from sqlbuild.adapters.duckdb.classes.duckdb_adapter import DuckDbAdapter from sqlbuild.compiler.pipeline.models import CompilePipelineResult -from sqlbuild.spec.contracts.models import SourceEntry from tests.integration.src.sqlbuild.compiler.pipeline._test_types import ( PythonSourceReadPlanTestCase, ) diff --git a/tests/integration/src/sqlbuild/compiler/planner/_helpers/_test_types.py b/tests/integration/src/sqlbuild/compiler/planner/_helpers/_test_types.py index fb54d3e2c9..1151b0c650 100644 --- a/tests/integration/src/sqlbuild/compiler/planner/_helpers/_test_types.py +++ b/tests/integration/src/sqlbuild/compiler/planner/_helpers/_test_types.py @@ -2,14 +2,14 @@ from sqlbuild._native import ( CursorSnapshotScope, + Fingerprint, + SourceEntry, ) from sqlbuild.compiler.compile.models import CompiledObjectKey, CompiledRelationLocation -from sqlbuild.compiler.fingerprints.models import Fingerprint from sqlbuild.compiler.planner.models import ( ModelCursorSnapshot, ) from sqlbuild.cursor_algebra.types import CursorScalar -from sqlbuild.spec.contracts.models import SourceEntry @dataclass(frozen=True) diff --git a/tests/integration/src/sqlbuild/compiler/planner/_helpers/helpers.py b/tests/integration/src/sqlbuild/compiler/planner/_helpers/helpers.py index 607dc40b84..ad82b737b5 100644 --- a/tests/integration/src/sqlbuild/compiler/planner/_helpers/helpers.py +++ b/tests/integration/src/sqlbuild/compiler/planner/_helpers/helpers.py @@ -15,6 +15,10 @@ CompileModelConfig, DiscoveredSchemaFile, DiscoveredSeedFile, + SchemaColumn, + SchemaModelEntry, + SchemaSeedEntry, + SourceEntry, WarehouseSnapshot, compiled_model_record, gather_warehouse_snapshot, @@ -39,12 +43,6 @@ ) from sqlbuild.compiler.discovery.models import DiscoveredSourceFile from sqlbuild.compiler.references.types import SqlReferenceKind -from sqlbuild.spec.contracts.models import ( - SchemaColumn, - SchemaModelEntry, - SchemaSeedEntry, - SourceEntry, -) class RecordingDuckDbAdapter(DuckDbAdapter): diff --git a/tests/integration/src/sqlbuild/compiler/planner/_helpers/resolve/_test_types.py b/tests/integration/src/sqlbuild/compiler/planner/_helpers/resolve/_test_types.py index 62d9cf8e2d..057c8e071b 100644 --- a/tests/integration/src/sqlbuild/compiler/planner/_helpers/resolve/_test_types.py +++ b/tests/integration/src/sqlbuild/compiler/planner/_helpers/resolve/_test_types.py @@ -5,8 +5,8 @@ from dataclasses import dataclass from datetime import datetime +from sqlbuild._native import SourceEntry from sqlbuild.adapter.contract.models import ColumnInfo, RelationInfo, field -from sqlbuild.spec.contracts.models import SourceEntry @dataclass(frozen=True) diff --git a/tests/integration/src/sqlbuild/compiler/planner/_helpers/resolve/helpers.py b/tests/integration/src/sqlbuild/compiler/planner/_helpers/resolve/helpers.py index ed8184d490..4bbb5d6290 100644 --- a/tests/integration/src/sqlbuild/compiler/planner/_helpers/resolve/helpers.py +++ b/tests/integration/src/sqlbuild/compiler/planner/_helpers/resolve/helpers.py @@ -13,6 +13,7 @@ CompileModelConfig, CursorOverridePair, ModelPlanContext, + SourceEntry, WarehouseSnapshot, assemble_project, build_compile_inputs, @@ -37,7 +38,6 @@ ) from sqlbuild.compiler.planner.types import BackfillAction from sqlbuild.compiler.references.types import SqlReferenceKind -from sqlbuild.spec.contracts.models import SourceEntry class OrchestrationOverrideAdapter(DuckDbAdapter): diff --git a/tests/integration/src/sqlbuild/compiler/planner/_helpers/resolve/test_adapter_overrides.py b/tests/integration/src/sqlbuild/compiler/planner/_helpers/resolve/test_adapter_overrides.py index 1b1ca53e1b..910416c611 100644 --- a/tests/integration/src/sqlbuild/compiler/planner/_helpers/resolve/test_adapter_overrides.py +++ b/tests/integration/src/sqlbuild/compiler/planner/_helpers/resolve/test_adapter_overrides.py @@ -4,9 +4,8 @@ import pytest -from sqlbuild._native import resolve_source_references +from sqlbuild._native import SourceColumnEntry, SourceEntry, resolve_source_references from sqlbuild.adapter.contract.models import ColumnInfo -from sqlbuild.spec.contracts.models import SourceColumnEntry, SourceEntry from tests.integration.src.sqlbuild.compiler.planner._helpers.resolve._test_types import ( AdapterOverrideTestCase, ) diff --git a/tests/integration/src/sqlbuild/compiler/planner/_helpers/resolve/test_resolve.py b/tests/integration/src/sqlbuild/compiler/planner/_helpers/resolve/test_resolve.py index 361406c3c8..4d3a358a46 100644 --- a/tests/integration/src/sqlbuild/compiler/planner/_helpers/resolve/test_resolve.py +++ b/tests/integration/src/sqlbuild/compiler/planner/_helpers/resolve/test_resolve.py @@ -7,6 +7,8 @@ import pytest from sqlbuild._native import ( + SourceColumnEntry, + SourceEntry, WarehouseSnapshot, ) from sqlbuild.adapter.contract.models import ColumnInfo @@ -19,7 +21,6 @@ ModelCursorSnapshot, ) from sqlbuild.compiler.planner.types import CursorType -from sqlbuild.spec.contracts.models import SourceColumnEntry, SourceEntry from tests.integration.src.sqlbuild.compiler.planner._helpers.resolve._test_types import ( ResolveAndExecuteTestCase, ResolveBoundedOverrideTestCase, diff --git a/tests/integration/src/sqlbuild/compiler/planner/_helpers/test_audit_entry.py b/tests/integration/src/sqlbuild/compiler/planner/_helpers/test_audit_entry.py index 07b92d16ff..db0b98771e 100644 --- a/tests/integration/src/sqlbuild/compiler/planner/_helpers/test_audit_entry.py +++ b/tests/integration/src/sqlbuild/compiler/planner/_helpers/test_audit_entry.py @@ -5,7 +5,7 @@ import pytest -from sqlbuild._native import DiscoveredAuditBlock, DiscoveredAuditFile, plan_audit +from sqlbuild._native import DiscoveredAuditBlock, DiscoveredAuditFile, SourceEntry, plan_audit from sqlbuild.adapters.duckdb.classes.duckdb_adapter import DuckDbAdapter from sqlbuild.compiler.compile.models import ( CompiledAudit, @@ -14,7 +14,6 @@ ) from sqlbuild.compiler.compile.types import CompiledAuditView, CompiledResourceType from sqlbuild.compiler.planner.models import AuditPlanEntry -from sqlbuild.spec.contracts.models import SourceEntry from tests.integration.src.sqlbuild.compiler.planner._helpers._test_types import ( ExecuteAuditTestCase, ) diff --git a/tests/integration/src/sqlbuild/compiler/planner/_helpers/test_warehouse_snapshot.py b/tests/integration/src/sqlbuild/compiler/planner/_helpers/test_warehouse_snapshot.py index e9d59dc840..9e975e0e21 100644 --- a/tests/integration/src/sqlbuild/compiler/planner/_helpers/test_warehouse_snapshot.py +++ b/tests/integration/src/sqlbuild/compiler/planner/_helpers/test_warehouse_snapshot.py @@ -8,6 +8,10 @@ from sqlbuild._native import ( CursorSnapshotScope, + Fingerprint, + SchemaColumn, + SchemaModelEntry, + TimestampValue, WarehouseSnapshot, compiled_model_record, gather_source_columns, @@ -26,14 +30,11 @@ CompiledProjectView, CompiledResourceType, ) -from sqlbuild.compiler.fingerprints.models import Fingerprint from sqlbuild.compiler.planner.exceptions import PlannerInputError from sqlbuild.compiler.planner.models import ( CursorOverrides, ModelCursorSnapshot, ) -from sqlbuild.cursor_algebra.models import TimestampValue -from sqlbuild.spec.contracts.models import SchemaColumn, SchemaModelEntry from tests.integration.src.sqlbuild.compiler.planner._helpers._test_types import ( GatherCursorSnapshotTestCase, GatherEmptySnapshotTestCase, diff --git a/tests/integration/src/sqlbuild/compiler/planner/main/helpers.py b/tests/integration/src/sqlbuild/compiler/planner/main/helpers.py index 008c274148..8ec57443cb 100644 --- a/tests/integration/src/sqlbuild/compiler/planner/main/helpers.py +++ b/tests/integration/src/sqlbuild/compiler/planner/main/helpers.py @@ -21,10 +21,14 @@ DeferralInputs, DiscoveredSchemaFile, DiscoveredSeedFile, + Fingerprint, FunctionArgument, PlannerOverrides, PlannerPolicies, PlannerSelection, + SchemaSeedEntry, + SettingsConfig, + SourceEntry, build_compiled_function_fingerprint_sql, build_execution_plan, compiled_model_record, @@ -53,16 +57,10 @@ FunctionLanguage, ) from sqlbuild.compiler.discovery.models import DiscoveredProjectInputs, DiscoveredSourceFile -from sqlbuild.compiler.fingerprints.models import Fingerprint from sqlbuild.compiler.pipeline.models import CompilePipelineResult from sqlbuild.compiler.planner.models import PlanOutput from sqlbuild.compiler.references.types import SqlReferenceKind from sqlbuild.runtime.contracts.models import ConnectionHooks -from sqlbuild.spec.contracts.models import ( - SchemaSeedEntry, - SettingsConfig, - SourceEntry, -) from tests.integration.src.sqlbuild.compiler.planner.main._test_types import ( BuildExecutionPlanTestCase, FormatPlanIntegrationTestCase, diff --git a/tests/integration/src/sqlbuild/compiler/planner/main/test_build_execution_plan.py b/tests/integration/src/sqlbuild/compiler/planner/main/test_build_execution_plan.py index 8697f8058f..1c8de80ebd 100644 --- a/tests/integration/src/sqlbuild/compiler/planner/main/test_build_execution_plan.py +++ b/tests/integration/src/sqlbuild/compiler/planner/main/test_build_execution_plan.py @@ -9,7 +9,16 @@ import sqlbuild._native as identity_planning from sqlbuild._native import ( + CursorsConfig, + FutureCursorAction, + FutureCursorsConfig, + LocalConfig, PlanWarning, + ProjectConfig, + ResolvedTableType, + TableType, + TableTypeSource, + TargetConfig, WarningSeverity, compiled_model_record, format_plan, @@ -29,15 +38,6 @@ PlanAction, PlanReason, ) -from sqlbuild.spec.contracts.models import ( - CursorsConfig, - FutureCursorsConfig, - LocalConfig, - ProjectConfig, - ResolvedTableType, - TargetConfig, -) -from sqlbuild.spec.contracts.types import FutureCursorAction, TableType, TableTypeSource from tests.integration.src.sqlbuild.compiler.planner.main._test_types import ( BuildExecutionPlanTestCase, FormatPlanIntegrationTestCase, diff --git a/tests/integration/src/sqlbuild/compiler/scopes/helpers.py b/tests/integration/src/sqlbuild/compiler/scopes/helpers.py index 42a458f532..09cb40be63 100644 --- a/tests/integration/src/sqlbuild/compiler/scopes/helpers.py +++ b/tests/integration/src/sqlbuild/compiler/scopes/helpers.py @@ -26,6 +26,7 @@ ScopeIndex, ScopeKind, ScopeLookup, + SqlLexicalSyntax, VisibilityReason, VisibilityRecord, build_declaration_scope, @@ -43,7 +44,6 @@ DiscoveredProjectInputs, EnumDeclaration, ) -from sqlbuild.compiler.sql_analysis.models import SqlLexicalSyntax from tests.integration.src.sqlbuild.compiler.helpers import GoldenEntry, golden_entry CURRENT_PATH_COMPONENT: str = "." diff --git a/tests/integration/src/sqlbuild/compiler/source_freshness/main/_test_types.py b/tests/integration/src/sqlbuild/compiler/source_freshness/main/_test_types.py index 488db3daf8..b2dc24b6db 100644 --- a/tests/integration/src/sqlbuild/compiler/source_freshness/main/_test_types.py +++ b/tests/integration/src/sqlbuild/compiler/source_freshness/main/_test_types.py @@ -1,7 +1,6 @@ from dataclasses import dataclass -from sqlbuild._native import SourceFreshnessIdentity -from sqlbuild.compiler.source_freshness.models import SourceFreshnessRecord +from sqlbuild._native import SourceFreshnessIdentity, SourceFreshnessRecord @dataclass(frozen=True) diff --git a/tests/integration/src/sqlbuild/compiler/source_freshness/main/test_read_write.py b/tests/integration/src/sqlbuild/compiler/source_freshness/main/test_read_write.py index 73e5d059a0..833bc8a83e 100644 --- a/tests/integration/src/sqlbuild/compiler/source_freshness/main/test_read_write.py +++ b/tests/integration/src/sqlbuild/compiler/source_freshness/main/test_read_write.py @@ -8,6 +8,7 @@ from sqlbuild._native import ( SourceFreshnessIdentity, + SourceFreshnessRecord, SourceFreshnessRenderers, SourceFreshnessSet, read_latest_source_freshness, @@ -16,7 +17,6 @@ from sqlbuild.adapter.contract.types import FrameworkType from sqlbuild.adapters.duckdb.classes.duckdb_adapter import DuckDbAdapter from sqlbuild.compiler.source_freshness.constants import SOURCE_FRESHNESS_TABLE_NAME -from sqlbuild.compiler.source_freshness.models import SourceFreshnessRecord from tests.integration.src.sqlbuild.compiler.source_freshness.main._test_types import ( SourceFreshnessLatestResolutionTestCase, SourceFreshnessPruneHistoryTestCase, diff --git a/tests/integration/src/sqlbuild/executor/build/_test_types.py b/tests/integration/src/sqlbuild/executor/build/_test_types.py index 8e9ab894a8..b12e53944a 100644 --- a/tests/integration/src/sqlbuild/executor/build/_test_types.py +++ b/tests/integration/src/sqlbuild/executor/build/_test_types.py @@ -1,8 +1,8 @@ from dataclasses import dataclass, field +from sqlbuild._native import MicrobatchLimitAction from sqlbuild.executor.build.types import BuildStatus from sqlbuild.executor.scheduling.types import ExecutionStatus -from sqlbuild.spec.contracts.types import MicrobatchLimitAction @dataclass(frozen=True) diff --git a/tests/integration/src/sqlbuild/executor/build/helpers.py b/tests/integration/src/sqlbuild/executor/build/helpers.py index 3d0f0bc1d9..a98e098f01 100644 --- a/tests/integration/src/sqlbuild/executor/build/helpers.py +++ b/tests/integration/src/sqlbuild/executor/build/helpers.py @@ -10,7 +10,12 @@ import duckdb -from sqlbuild._native import CompilePipelineOptions, discover_project_inputs, run_compile_pipeline +from sqlbuild._native import ( + CompilePipelineOptions, + MicrobatchLimitAction, + discover_project_inputs, + run_compile_pipeline, +) from sqlbuild._native import run_cli as main from sqlbuild.adapter.contract.types import TablePromotionMode from sqlbuild.adapters.duckdb.classes.duckdb_adapter import DuckDbAdapter @@ -23,7 +28,6 @@ BuildRuntimeParams, ) from sqlbuild.executor.scheduling.types import ExecutionStatus -from sqlbuild.spec.contracts.types import MicrobatchLimitAction from tests.integration.src.sqlbuild.executor.build._test_types import ( BuildExecutionTestCase, ) diff --git a/tests/integration/src/sqlbuild/executor/build/test_capped_microbatch_dependencies.py b/tests/integration/src/sqlbuild/executor/build/test_capped_microbatch_dependencies.py index 5a2a93a8e8..a0e4779470 100644 --- a/tests/integration/src/sqlbuild/executor/build/test_capped_microbatch_dependencies.py +++ b/tests/integration/src/sqlbuild/executor/build/test_capped_microbatch_dependencies.py @@ -4,10 +4,10 @@ import pytest +from sqlbuild._native import MicrobatchLimitAction from sqlbuild.adapters.duckdb.classes.duckdb_adapter import DuckDbAdapter from sqlbuild.compiler.compile.exceptions import CompileInputError from sqlbuild.compiler.pipeline.models import CompilePipelineResult -from sqlbuild.spec.contracts.types import MicrobatchLimitAction from tests.integration.src.sqlbuild.executor.build._test_types import ( CappedFilterConsumerTestCase, CappedIntermediateFilterTestCase, diff --git a/tests/integration/src/sqlbuild/executor/run/incremental/_test_types.py b/tests/integration/src/sqlbuild/executor/run/incremental/_test_types.py index baef1aa160..d0d2f7fd06 100644 --- a/tests/integration/src/sqlbuild/executor/run/incremental/_test_types.py +++ b/tests/integration/src/sqlbuild/executor/run/incremental/_test_types.py @@ -1,11 +1,11 @@ from dataclasses import dataclass, field from datetime import datetime +from sqlbuild._native import FutureCursorsConfig from sqlbuild.compiler.auditing.types import AuditRunScope from sqlbuild.compiler.planner.types import OnSchemaChange from sqlbuild.executor.run.types import ExecutionPhase from sqlbuild.executor.scheduling.types import ExecutionStatus -from sqlbuild.spec.contracts.models import FutureCursorsConfig @dataclass(frozen=True) diff --git a/tests/integration/src/sqlbuild/executor/run/incremental/test_incremental_execution.py b/tests/integration/src/sqlbuild/executor/run/incremental/test_incremental_execution.py index 56b4063e9f..5bd8539c16 100644 --- a/tests/integration/src/sqlbuild/executor/run/incremental/test_incremental_execution.py +++ b/tests/integration/src/sqlbuild/executor/run/incremental/test_incremental_execution.py @@ -8,6 +8,7 @@ import pytest +from sqlbuild._native import FutureCursorAction, FutureCursorsConfig from sqlbuild.adapters.duckdb.classes.duckdb_adapter import DuckDbAdapter from sqlbuild.compiler.auditing.types import AuditRunScope from sqlbuild.compiler.discovery.models import DiscoveredHookFunction, PythonHookEntry, SqlHookEntry @@ -15,8 +16,6 @@ from sqlbuild.compiler.planner.types import OnSchemaChange from sqlbuild.executor.run.models import ModelExecutionResult from sqlbuild.executor.run.types import ExecutionPhase -from sqlbuild.spec.contracts.models import FutureCursorsConfig -from sqlbuild.spec.contracts.types import FutureCursorAction from tests.integration.src.sqlbuild.executor.run.incremental._test_types import ( IncrementalFailureTestCase, IncrementalSuccessTestCase, diff --git a/tests/integration/src/sqlbuild/executor/run/microbatch/_test_types.py b/tests/integration/src/sqlbuild/executor/run/microbatch/_test_types.py index 5135161793..0e337020e9 100644 --- a/tests/integration/src/sqlbuild/executor/run/microbatch/_test_types.py +++ b/tests/integration/src/sqlbuild/executor/run/microbatch/_test_types.py @@ -2,12 +2,11 @@ from dataclasses import dataclass, field from datetime import datetime +from sqlbuild._native import FutureCursorsConfig, MicrobatchLimitAction from sqlbuild.compiler.auditing.types import AuditRunScope from sqlbuild.compiler.planner.types import OnSchemaChange from sqlbuild.executor.run.types import ExecutionPhase from sqlbuild.executor.scheduling.types import ExecutionStatus -from sqlbuild.spec.contracts.models import FutureCursorsConfig -from sqlbuild.spec.contracts.types import MicrobatchLimitAction @dataclass(frozen=True) diff --git a/tests/integration/src/sqlbuild/executor/run/microbatch/test_microbatch_execution.py b/tests/integration/src/sqlbuild/executor/run/microbatch/test_microbatch_execution.py index e2961d27d3..51a27ff1ba 100644 --- a/tests/integration/src/sqlbuild/executor/run/microbatch/test_microbatch_execution.py +++ b/tests/integration/src/sqlbuild/executor/run/microbatch/test_microbatch_execution.py @@ -10,6 +10,7 @@ import pytest +from sqlbuild._native import FutureCursorAction, FutureCursorsConfig, MicrobatchLimitAction from sqlbuild.adapters.duckdb.classes.duckdb_adapter import DuckDbAdapter from sqlbuild.compiler.auditing.types import AuditRunScope from sqlbuild.compiler.discovery.models import DiscoveredHookFunction, PythonHookEntry, SqlHookEntry @@ -21,8 +22,6 @@ from sqlbuild.executor.run._helpers.materializations import microbatch as microbatch_module from sqlbuild.executor.run.models import BatchWindow, ModelExecutionResult from sqlbuild.executor.run.types import ExecutionPhase -from sqlbuild.spec.contracts.models import FutureCursorsConfig -from sqlbuild.spec.contracts.types import FutureCursorAction, MicrobatchLimitAction from tests.integration.src.sqlbuild.executor.run.microbatch._test_types import ( MicrobatchFailureTestCase, MicrobatchSuccessTestCase, diff --git a/tests/integration/src/sqlbuild/executor/run/microbatch/test_microbatch_observability.py b/tests/integration/src/sqlbuild/executor/run/microbatch/test_microbatch_observability.py index afc61aa22c..ec6c1320c7 100644 --- a/tests/integration/src/sqlbuild/executor/run/microbatch/test_microbatch_observability.py +++ b/tests/integration/src/sqlbuild/executor/run/microbatch/test_microbatch_observability.py @@ -7,6 +7,7 @@ import pytest +from sqlbuild._native import MicrobatchLimitAction from sqlbuild.adapters.duckdb.classes.duckdb_adapter import DuckDbAdapter from sqlbuild.executor.run._helpers.materializations import microbatch as microbatch_module from sqlbuild.executor.run.models import BatchWindow @@ -18,7 +19,6 @@ resource_attempt_scope, run_scope, ) -from sqlbuild.spec.contracts.types import MicrobatchLimitAction from tests.integration.src.sqlbuild.executor.run.microbatch._test_types import ( MicrobatchFailureTestCase, MicrobatchSuccessTestCase, diff --git a/tests/integration/src/sqlbuild/executor/run/microbatch/test_microbatch_reconciliation.py b/tests/integration/src/sqlbuild/executor/run/microbatch/test_microbatch_reconciliation.py index e779e11b4a..5fe88de068 100644 --- a/tests/integration/src/sqlbuild/executor/run/microbatch/test_microbatch_reconciliation.py +++ b/tests/integration/src/sqlbuild/executor/run/microbatch/test_microbatch_reconciliation.py @@ -10,6 +10,8 @@ from sqlbuild._native import ( BackfillResult, CursorWatermarkMode, + IntegerValue, + MicrobatchLimitAction, compute_cursor_bounds, ) from sqlbuild._native import cursor_sentinel_to_token as sentinel_to_token @@ -23,7 +25,6 @@ from sqlbuild.compiler.planner.types import ( BackfillAction, ) -from sqlbuild.cursor_algebra.models import IntegerValue from sqlbuild.executor.run._helpers.materializations.microbatch import ( _clamp_intervals_to_model_domain, _count_unaccounted_intervals, @@ -50,7 +51,6 @@ MicrobatchScope, MicrobatchWriteResult, ) -from sqlbuild.spec.contracts.types import MicrobatchLimitAction from tests.integration.src.sqlbuild.executor.run.microbatch._test_types import ( MicrobatchBehaviorTestCase, MicrobatchReconciliationChunkTestCase, diff --git a/tests/unit/scripts/compiler_differential/_helpers/coverage/test_discovery.py b/tests/unit/scripts/compiler_differential/_helpers/coverage/test_discovery.py index ba728bf872..42ce02e13a 100644 --- a/tests/unit/scripts/compiler_differential/_helpers/coverage/test_discovery.py +++ b/tests/unit/scripts/compiler_differential/_helpers/coverage/test_discovery.py @@ -15,9 +15,8 @@ required_discovery_kinds, ) from scripts.compiler_differential.constants import DISCOVERY_NON_COLLECTION_FIELDS -from sqlbuild._native import render_stage_capture +from sqlbuild._native import LocalConfig, ProjectConfig, render_stage_capture from sqlbuild.compiler.discovery.models import DiscoveredProjectInputs -from sqlbuild.spec.contracts.models import LocalConfig, ProjectConfig from tests.unit.scripts.compiler_differential._helpers.coverage._test_types import ( AuthoredByteKindsTestCase, CaptureProblemsTestCase, diff --git a/tests/unit/scripts/compiler_differential/_helpers/coverage/test_render.py b/tests/unit/scripts/compiler_differential/_helpers/coverage/test_render.py index 4b51f34890..a9bfb3280d 100644 --- a/tests/unit/scripts/compiler_differential/_helpers/coverage/test_render.py +++ b/tests/unit/scripts/compiler_differential/_helpers/coverage/test_render.py @@ -15,10 +15,14 @@ required_render_kinds, ) from scripts.compiler_differential.constants import RENDER_NON_COLLECTION_FIELDS -from sqlbuild._native import CompileProjectInputs, render_stage_capture +from sqlbuild._native import ( + CompileProjectInputs, + LocalConfig, + ProjectConfig, + SqlLexicalSyntax, + render_stage_capture, +) from sqlbuild.compiler.discovery.models import DiscoveredProjectInputs -from sqlbuild.compiler.sql_analysis.models import SqlLexicalSyntax -from sqlbuild.spec.contracts.models import LocalConfig, ProjectConfig from tests.unit.scripts.compiler_differential._helpers.coverage._test_types import ( CaptureProblemsTestCase, RenderCaptureKindsTestCase, diff --git a/tests/unit/src/sqlbuild/adapter/contract/_helpers/_test_types.py b/tests/unit/src/sqlbuild/adapter/contract/_helpers/_test_types.py index 096b53fc6b..4015954770 100644 --- a/tests/unit/src/sqlbuild/adapter/contract/_helpers/_test_types.py +++ b/tests/unit/src/sqlbuild/adapter/contract/_helpers/_test_types.py @@ -1,6 +1,6 @@ from dataclasses import dataclass -from sqlbuild.spec.contracts.models import SeedCsvSettings +from sqlbuild._native import SeedCsvSettings @dataclass(frozen=True) diff --git a/tests/unit/src/sqlbuild/adapter/contract/_helpers/test_seed_csv.py b/tests/unit/src/sqlbuild/adapter/contract/_helpers/test_seed_csv.py index 8fe449ef20..abd0e1226e 100644 --- a/tests/unit/src/sqlbuild/adapter/contract/_helpers/test_seed_csv.py +++ b/tests/unit/src/sqlbuild/adapter/contract/_helpers/test_seed_csv.py @@ -4,8 +4,8 @@ import pytest +from sqlbuild._native import SeedCsvSettings from sqlbuild.adapter.contract.main.normalize_seed_csv_value import normalize_seed_csv_value -from sqlbuild.spec.contracts.models import SeedCsvSettings from tests.unit.src.sqlbuild.adapter.contract._helpers._test_types import ( NormalizeSeedCsvValueTestCase, ) diff --git a/tests/unit/src/sqlbuild/adapter/contract/typed_sql_rendering/helpers.py b/tests/unit/src/sqlbuild/adapter/contract/typed_sql_rendering/helpers.py index bb8a7c17b8..bc0009d664 100644 --- a/tests/unit/src/sqlbuild/adapter/contract/typed_sql_rendering/helpers.py +++ b/tests/unit/src/sqlbuild/adapter/contract/typed_sql_rendering/helpers.py @@ -1,7 +1,6 @@ from __future__ import annotations -from sqlbuild._native import normalize_sql_value -from sqlbuild.sql_values.models import SqlValue +from sqlbuild._native import SqlValue, normalize_sql_value def typed_sql_value(raw_value: object) -> SqlValue: diff --git a/tests/unit/src/sqlbuild/adapter/contract/typed_sql_rendering/test_typed_sql_rendering.py b/tests/unit/src/sqlbuild/adapter/contract/typed_sql_rendering/test_typed_sql_rendering.py index 246ce97278..7a65929f2e 100644 --- a/tests/unit/src/sqlbuild/adapter/contract/typed_sql_rendering/test_typed_sql_rendering.py +++ b/tests/unit/src/sqlbuild/adapter/contract/typed_sql_rendering/test_typed_sql_rendering.py @@ -4,6 +4,7 @@ import pytest +from sqlbuild._native import SqlValue from sqlbuild.adapter.contract.classes.strict_adapter import StrictAdapter from sqlbuild.adapter.contract.exceptions import AdapterUserError from sqlbuild.adapters.bigquery.classes.bigquery_adapter import BigQueryAdapter @@ -13,7 +14,6 @@ from sqlbuild.adapters.postgres.classes.postgres_adapter import PostgresAdapter from sqlbuild.adapters.snowflake.classes.snowflake_adapter import SnowflakeAdapter from sqlbuild.adapters.sqlserver.classes.sqlserver_adapter import SqlServerAdapter -from sqlbuild.sql_values.models import SqlValue from tests.unit.src.sqlbuild.adapter.contract.typed_sql_rendering._test_types import ( AdapterTypedSqlRenderingTestCase, InvalidTypedArrayTestCase, diff --git a/tests/unit/src/sqlbuild/adapter/state_sql/test_source_freshness.py b/tests/unit/src/sqlbuild/adapter/state_sql/test_source_freshness.py index 9bf68b82c8..a525402edd 100644 --- a/tests/unit/src/sqlbuild/adapter/state_sql/test_source_freshness.py +++ b/tests/unit/src/sqlbuild/adapter/state_sql/test_source_freshness.py @@ -6,11 +6,11 @@ import pytest +from sqlbuild._native import SourceFreshnessRecord from sqlbuild.adapter.state_sql._helpers.source_freshness import ( render_insert_source_freshness_records_sql, ) from sqlbuild.adapters.duckdb.classes.duckdb_adapter import DuckDbAdapter -from sqlbuild.compiler.source_freshness.models import SourceFreshnessRecord from tests.unit.src.sqlbuild.adapter.state_sql._test_types import StateSqlGoldenTestCase diff --git a/tests/unit/src/sqlbuild/cli/commands/_helpers/build_planning/_test_types.py b/tests/unit/src/sqlbuild/cli/commands/_helpers/build_planning/_test_types.py index 19d4a9294c..56e3fef96d 100644 --- a/tests/unit/src/sqlbuild/cli/commands/_helpers/build_planning/_test_types.py +++ b/tests/unit/src/sqlbuild/cli/commands/_helpers/build_planning/_test_types.py @@ -1,7 +1,7 @@ from dataclasses import dataclass +from sqlbuild._native import SnapshotsConfig from sqlbuild.compiler.planner.models import PlanOutput -from sqlbuild.spec.contracts.models import SnapshotsConfig @dataclass(frozen=True) diff --git a/tests/unit/src/sqlbuild/cli/commands/_helpers/build_planning/test_execution_limits.py b/tests/unit/src/sqlbuild/cli/commands/_helpers/build_planning/test_execution_limits.py index 9295e1c946..fb722bc451 100644 --- a/tests/unit/src/sqlbuild/cli/commands/_helpers/build_planning/test_execution_limits.py +++ b/tests/unit/src/sqlbuild/cli/commands/_helpers/build_planning/test_execution_limits.py @@ -1,7 +1,6 @@ import pytest -from sqlbuild._native import CliUserError, enforce_model_execution_limit -from sqlbuild.spec.contracts.models import ExecutionLimitsConfig +from sqlbuild._native import CliUserError, ExecutionLimitsConfig, enforce_model_execution_limit from tests.unit.src.sqlbuild.cli.commands._helpers.build_planning._test_types import ( ModelExecutionLimitTestCase, ) diff --git a/tests/unit/src/sqlbuild/cli/commands/_helpers/build_planning/test_full_refresh.py b/tests/unit/src/sqlbuild/cli/commands/_helpers/build_planning/test_full_refresh.py index cb653ebbde..caf2762ff8 100644 --- a/tests/unit/src/sqlbuild/cli/commands/_helpers/build_planning/test_full_refresh.py +++ b/tests/unit/src/sqlbuild/cli/commands/_helpers/build_planning/test_full_refresh.py @@ -6,9 +6,13 @@ import pytest -from sqlbuild._native import CliUserError, enforce_plan_safety_policies +from sqlbuild._native import ( + CliUserError, + ExecutionLimitsConfig, + SnapshotsConfig, + enforce_plan_safety_policies, +) from sqlbuild.compiler.planner.models import PlanOutput -from sqlbuild.spec.contracts.models import ExecutionLimitsConfig, SnapshotsConfig from tests.unit.src.sqlbuild.cli.commands._helpers.build_planning._test_types import ( SnapshotFullRefreshPolicyTestCase, ) diff --git a/tests/unit/src/sqlbuild/cli/commands/_helpers/build_planning/test_plan_policies.py b/tests/unit/src/sqlbuild/cli/commands/_helpers/build_planning/test_plan_policies.py index 5dfa1f0206..b20ce4da2b 100644 --- a/tests/unit/src/sqlbuild/cli/commands/_helpers/build_planning/test_plan_policies.py +++ b/tests/unit/src/sqlbuild/cli/commands/_helpers/build_planning/test_plan_policies.py @@ -6,11 +6,12 @@ from sqlbuild._native import ( CliUserError, + ExecutionLimitsConfig, + SnapshotsConfig, dbt_interop_plan_safety_gates, enforce_plan_safety_policies, ) from sqlbuild.compiler.planner.models import PlanOutput -from sqlbuild.spec.contracts.models import ExecutionLimitsConfig, SnapshotsConfig from tests.unit.src.sqlbuild.cli.commands._helpers.build_planning._test_types import ( DbtPlanSafetyPolicyTestCase, ) diff --git a/tests/unit/src/sqlbuild/cli/commands/_helpers/build_planning/test_retention_decrease.py b/tests/unit/src/sqlbuild/cli/commands/_helpers/build_planning/test_retention_decrease.py index 1355aec214..35a1638a18 100644 --- a/tests/unit/src/sqlbuild/cli/commands/_helpers/build_planning/test_retention_decrease.py +++ b/tests/unit/src/sqlbuild/cli/commands/_helpers/build_planning/test_retention_decrease.py @@ -4,10 +4,14 @@ import pytest -from sqlbuild._native import CliUserError, enforce_plan_safety_policies +from sqlbuild._native import ( + CliUserError, + ExecutionLimitsConfig, + SnapshotsConfig, + enforce_plan_safety_policies, +) from sqlbuild.compiler.planner.models import PlanOutput from sqlbuild.compiler.planner.types import RetentionDirection -from sqlbuild.spec.contracts.models import ExecutionLimitsConfig, SnapshotsConfig from tests.unit.src.sqlbuild.cli.commands._helpers.build_planning._test_types import ( RetentionDecreasePolicyTestCase, ) diff --git a/tests/unit/src/sqlbuild/cli/commands/_helpers/build_planning/test_table_type.py b/tests/unit/src/sqlbuild/cli/commands/_helpers/build_planning/test_table_type.py index cb78756d38..0403cc2edb 100644 --- a/tests/unit/src/sqlbuild/cli/commands/_helpers/build_planning/test_table_type.py +++ b/tests/unit/src/sqlbuild/cli/commands/_helpers/build_planning/test_table_type.py @@ -4,9 +4,13 @@ import pytest -from sqlbuild._native import CliUserError, enforce_plan_safety_policies +from sqlbuild._native import ( + CliUserError, + ExecutionLimitsConfig, + SnapshotsConfig, + enforce_plan_safety_policies, +) from sqlbuild.compiler.planner.models import PlanOutput -from sqlbuild.spec.contracts.models import ExecutionLimitsConfig, SnapshotsConfig from tests.unit.src.sqlbuild.cli.commands._helpers.build_planning._test_types import ( TableTypeDowngradePolicyTestCase, ) diff --git a/tests/unit/src/sqlbuild/cli/commands/_helpers/check/test_planning.py b/tests/unit/src/sqlbuild/cli/commands/_helpers/check/test_planning.py index 394064a12b..ea74610ce7 100644 --- a/tests/unit/src/sqlbuild/cli/commands/_helpers/check/test_planning.py +++ b/tests/unit/src/sqlbuild/cli/commands/_helpers/check/test_planning.py @@ -6,18 +6,21 @@ import pytest -from sqlbuild._native import CliUserError, run_check_read_side_dependencies +from sqlbuild._native import ( + CliUserError, + PythonSqlRunLifecyclePlan, + SourceEntry, + run_check_read_side_dependencies, +) from sqlbuild.adapter.contract.models import RelationLookup from sqlbuild.adapter.relations.main import relation_lookup as relation_lookup_module from sqlbuild.compiler.compile.models import CompiledProject, CompiledRelationLocation from sqlbuild.compiler.pipeline.models import CompilePipelineResult from sqlbuild.compiler.planner.models import PlanOutput -from sqlbuild.compiler.python_nodes.models import PythonSqlRunLifecyclePlan from sqlbuild.executor.python_nodes.main import read_side as read_side_module from sqlbuild.executor.python_nodes.models import PythonNodeExecutionResult from sqlbuild.python_nodes.models import SqlResourceRef from sqlbuild.python_nodes.types import SqlResourceRefKind -from sqlbuild.spec.contracts.models import SourceEntry from tests.unit.src.sqlbuild.cli.commands._helpers.check._test_types import ( CheckPlanningTestCase, ) diff --git a/tests/unit/src/sqlbuild/cli/commands/_helpers/cost/helpers.py b/tests/unit/src/sqlbuild/cli/commands/_helpers/cost/helpers.py index 4f758851b9..1649470087 100644 --- a/tests/unit/src/sqlbuild/cli/commands/_helpers/cost/helpers.py +++ b/tests/unit/src/sqlbuild/cli/commands/_helpers/cost/helpers.py @@ -3,8 +3,7 @@ from pathlib import Path from typing import Any -from sqlbuild._native import BuildCostFinalization -from sqlbuild.spec.contracts.models import CostConfig +from sqlbuild._native import BuildCostFinalization, CostConfig def build_cost_finalization( diff --git a/tests/unit/src/sqlbuild/cli/commands/_helpers/freshness/_test_types.py b/tests/unit/src/sqlbuild/cli/commands/_helpers/freshness/_test_types.py index c69eeaf2d2..a284893347 100644 --- a/tests/unit/src/sqlbuild/cli/commands/_helpers/freshness/_test_types.py +++ b/tests/unit/src/sqlbuild/cli/commands/_helpers/freshness/_test_types.py @@ -3,8 +3,7 @@ from dataclasses import dataclass, field from datetime import UTC, datetime -from sqlbuild._native import SourceFreshnessIdentity -from sqlbuild.compiler.source_freshness.models import SourceFreshnessRecord +from sqlbuild._native import SourceFreshnessIdentity, SourceFreshnessRecord @dataclass(frozen=True) diff --git a/tests/unit/src/sqlbuild/cli/commands/_helpers/freshness/helpers.py b/tests/unit/src/sqlbuild/cli/commands/_helpers/freshness/helpers.py index 328765c511..592a1ef620 100644 --- a/tests/unit/src/sqlbuild/cli/commands/_helpers/freshness/helpers.py +++ b/tests/unit/src/sqlbuild/cli/commands/_helpers/freshness/helpers.py @@ -2,19 +2,20 @@ from datetime import UTC, datetime -from sqlbuild._native import source_freshness_data_version_hash +from sqlbuild._native import ( + SourceEntry, + SourceFreshnessAgePolicy, + SourceFreshnessConfig, + SourceFreshnessRecord, + SourceFreshnessStrategy, + SourceFreshnessValueKind, + source_freshness_data_version_hash, +) from sqlbuild.adapter.contract.models import ( QueryResult, TableFreshnessMetadata, TableFreshnessRequest, ) -from sqlbuild.compiler.source_freshness.models import SourceFreshnessRecord -from sqlbuild.spec.contracts.models import ( - SourceEntry, - SourceFreshnessAgePolicy, - SourceFreshnessConfig, -) -from sqlbuild.spec.contracts.types import SourceFreshnessStrategy, SourceFreshnessValueKind class FreshnessRecordingAdapter: diff --git a/tests/unit/src/sqlbuild/cli/commands/_helpers/helpers.py b/tests/unit/src/sqlbuild/cli/commands/_helpers/helpers.py index bf07ffd0a2..901ac2b132 100644 --- a/tests/unit/src/sqlbuild/cli/commands/_helpers/helpers.py +++ b/tests/unit/src/sqlbuild/cli/commands/_helpers/helpers.py @@ -6,6 +6,7 @@ from sqlbuild._native import ( CostContext, SourceFreshnessIdentity, + SourceFreshnessRecord, build_source_freshness_create_table_sql, ) from sqlbuild.adapter.contract.types import FrameworkType @@ -17,7 +18,6 @@ from sqlbuild.compiler.compile.types import CompiledResourceType from sqlbuild.compiler.planner.models import ChainStep, ModelPlanEntry, SqlTestPlanEntry from sqlbuild.compiler.planner.types import MaterializationType, PlanAction, PlanReason -from sqlbuild.compiler.source_freshness.models import SourceFreshnessRecord from sqlbuild.cost.models import CostResourceContext diff --git a/tests/unit/src/sqlbuild/cli/commands/_helpers/load_selection/helpers.py b/tests/unit/src/sqlbuild/cli/commands/_helpers/load_selection/helpers.py index f16717b056..51c473ce3b 100644 --- a/tests/unit/src/sqlbuild/cli/commands/_helpers/load_selection/helpers.py +++ b/tests/unit/src/sqlbuild/cli/commands/_helpers/load_selection/helpers.py @@ -4,17 +4,12 @@ from pathlib import Path +from sqlbuild._native import IntegrationLoaderConfig, LocalConfig, ProjectConfig, SourceEntry from sqlbuild.compiler.discovery.models import ( DiscoveredLoaderFunction, DiscoveredProjectInputs, DiscoveredSourceFile, ) -from sqlbuild.spec.contracts.models import ( - IntegrationLoaderConfig, - LocalConfig, - ProjectConfig, - SourceEntry, -) def fetch_orders(_ctx: object) -> list[dict[str, object]]: diff --git a/tests/unit/src/sqlbuild/cli/commands/_helpers/load_selection/test_selection.py b/tests/unit/src/sqlbuild/cli/commands/_helpers/load_selection/test_selection.py index b738f4483d..3cb53077ac 100644 --- a/tests/unit/src/sqlbuild/cli/commands/_helpers/load_selection/test_selection.py +++ b/tests/unit/src/sqlbuild/cli/commands/_helpers/load_selection/test_selection.py @@ -7,12 +7,15 @@ import pytest from sqlbuild._native import ( + LocalConfig, + ProjectConfig, + SourceEntry, + TargetConfig, resolve_loader_defaults, select_load_entries, select_load_reference_entries, ) from sqlbuild.compiler.discovery.models import DiscoveredLoaderFunction, DiscoveredProjectInputs -from sqlbuild.spec.contracts.models import LocalConfig, ProjectConfig, SourceEntry, TargetConfig from tests.unit.src.sqlbuild.cli.commands._helpers.load_selection._test_types import ( LoaderDefaultResolutionTestCase, LoaderDestinationSelectionTestCase, diff --git a/tests/unit/src/sqlbuild/cli/commands/_helpers/runtime/_test_types.py b/tests/unit/src/sqlbuild/cli/commands/_helpers/runtime/_test_types.py index f324319fe9..64497ac254 100644 --- a/tests/unit/src/sqlbuild/cli/commands/_helpers/runtime/_test_types.py +++ b/tests/unit/src/sqlbuild/cli/commands/_helpers/runtime/_test_types.py @@ -1,13 +1,13 @@ from dataclasses import dataclass, field -from sqlbuild._native import CliCommand -from sqlbuild.spec.contracts.models import ( +from sqlbuild._native import ( + CliCommand, LocalConfig, LocalTargetConfig, ProjectConfig, TargetWarehousesConfig, + WarehouseGroup, ) -from sqlbuild.spec.contracts.types import WarehouseGroup @dataclass(frozen=True) diff --git a/tests/unit/src/sqlbuild/cli/commands/_helpers/runtime/helpers.py b/tests/unit/src/sqlbuild/cli/commands/_helpers/runtime/helpers.py index bd00e4aa67..29760e4191 100644 --- a/tests/unit/src/sqlbuild/cli/commands/_helpers/runtime/helpers.py +++ b/tests/unit/src/sqlbuild/cli/commands/_helpers/runtime/helpers.py @@ -4,13 +4,8 @@ from pathlib import Path +from sqlbuild._native import LocalConfig, ProjectConfig, TargetConfig, TargetWarehousesConfig from sqlbuild.compiler.discovery.models import DiscoveredProjectInputs -from sqlbuild.spec.contracts.models import ( - LocalConfig, - ProjectConfig, - TargetConfig, - TargetWarehousesConfig, -) from tests.unit.src.sqlbuild.cli.commands._helpers.runtime._test_types import ( CommandWarehouseResolutionTestCase, ) diff --git a/tests/unit/src/sqlbuild/cli/commands/_helpers/runtime/test_connection.py b/tests/unit/src/sqlbuild/cli/commands/_helpers/runtime/test_connection.py index 4f2d58fa6b..cb0a7c8366 100644 --- a/tests/unit/src/sqlbuild/cli/commands/_helpers/runtime/test_connection.py +++ b/tests/unit/src/sqlbuild/cli/commands/_helpers/runtime/test_connection.py @@ -6,8 +6,12 @@ from sqlbuild import _native as native from sqlbuild._native import ( + LocalConfig, + LocalTargetConfig, NormalizedDbtProfileConnection, + ProjectConfig, ProjectConfigError, + TargetConfig, resolve_connection_config, resolve_project_connection_config, resolve_target_connection_config, @@ -15,12 +19,6 @@ from sqlbuild.adapter.contract.types import BuiltinAdapter from sqlbuild.compiler.compile.exceptions import CompileInputError from sqlbuild.compiler.discovery.models import DiscoveredProjectInputs -from sqlbuild.spec.contracts.models import ( - LocalConfig, - LocalTargetConfig, - ProjectConfig, - TargetConfig, -) from tests.unit.src.sqlbuild.cli.commands._helpers.runtime._test_types import ( ExplicitMemoryConnectionTestCase, MissingConnectionTestCase, diff --git a/tests/unit/src/sqlbuild/cli/commands/_helpers/runtime/test_warehouses.py b/tests/unit/src/sqlbuild/cli/commands/_helpers/runtime/test_warehouses.py index e732a3ad32..917bcc0c76 100644 --- a/tests/unit/src/sqlbuild/cli/commands/_helpers/runtime/test_warehouses.py +++ b/tests/unit/src/sqlbuild/cli/commands/_helpers/runtime/test_warehouses.py @@ -9,8 +9,11 @@ from sqlbuild._native import ( CliCommand, CliUserError, + LocalTargetConfig, ParsedCliInvocation, ResolvedWarehouse, + TargetWarehousesConfig, + WarehouseGroup, command_warehouse_scope, current_command_warehouse_scope, describe_connection_warehouse, @@ -18,8 +21,6 @@ resolve_project_connection_config, ) from sqlbuild.compiler.discovery.models import DiscoveredProjectInputs -from sqlbuild.spec.contracts.models import LocalTargetConfig, TargetWarehousesConfig -from sqlbuild.spec.contracts.types import WarehouseGroup from tests.unit.src.sqlbuild.cli.commands._helpers.runtime._test_types import ( CommandWarehouseErrorTestCase, CommandWarehouseGroupTestCase, diff --git a/tests/unit/src/sqlbuild/cli/commands/main/compile/helpers.py b/tests/unit/src/sqlbuild/cli/commands/main/compile/helpers.py index 72c54c82d1..b1a45adaab 100644 --- a/tests/unit/src/sqlbuild/cli/commands/main/compile/helpers.py +++ b/tests/unit/src/sqlbuild/cli/commands/main/compile/helpers.py @@ -10,6 +10,7 @@ CompileModelConfig, FunctionArgument, PreparedCompileArtifacts, + SeedCsvSettings, WrittenTarget, project_with, ) @@ -51,7 +52,6 @@ from sqlbuild.executor.build.types import BuildStatus from sqlbuild.executor.run.models import ModelExecutionResult from sqlbuild.executor.scheduling.types import ExecutionStatus -from sqlbuild.spec.contracts.models import SeedCsvSettings from tests.unit.src.sqlbuild.cli.commands.main.dag.helpers import ( prepare_python_dag_project, ) diff --git a/tests/unit/src/sqlbuild/cli/commands/main/compile/test_adapters.py b/tests/unit/src/sqlbuild/cli/commands/main/compile/test_adapters.py index d3ea7889a5..331c59ac23 100644 --- a/tests/unit/src/sqlbuild/cli/commands/main/compile/test_adapters.py +++ b/tests/unit/src/sqlbuild/cli/commands/main/compile/test_adapters.py @@ -5,9 +5,14 @@ import pytest -from sqlbuild._native import CliUserError, resolve_adapter, resolve_effective_adapter_name +from sqlbuild._native import ( + CliUserError, + LocalConfig, + ProjectConfig, + resolve_adapter, + resolve_effective_adapter_name, +) from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter -from sqlbuild.spec.contracts.models import LocalConfig, ProjectConfig from tests.unit.src.sqlbuild.cli.commands.main.compile._test_types import ( ExpectedMessageTestCase, ResolveAdapterErrorTestCase, diff --git a/tests/unit/src/sqlbuild/cli/commands/main/execution/helpers.py b/tests/unit/src/sqlbuild/cli/commands/main/execution/helpers.py index 50ff8083a9..ded745f743 100644 --- a/tests/unit/src/sqlbuild/cli/commands/main/execution/helpers.py +++ b/tests/unit/src/sqlbuild/cli/commands/main/execution/helpers.py @@ -10,13 +10,13 @@ import pytest from sqlbuild import _native as native +from sqlbuild._native import LocalConfig, ProjectConfig from sqlbuild.adapter.discovery.main import resolve_adapter as adapters_module from sqlbuild.compiler.compile.models import CompiledObjectKey, CompiledSqlScenario from sqlbuild.compiler.compile.types import CompiledResourceType from sqlbuild.compiler.discovery.models import DiscoveredProjectInputs, DiscoveredSqlScenarioFile from sqlbuild.diagnostics.main import process_resource_reporting as resource_reporting_module from sqlbuild.executor.pipeline.main import run as executor_run_module -from sqlbuild.spec.contracts.models import LocalConfig, ProjectConfig from tests.unit.src.sqlbuild.presentation.helpers import use_plain_terminal diff --git a/tests/unit/src/sqlbuild/cli/output/_helpers/test_execution_result_document.py b/tests/unit/src/sqlbuild/cli/output/_helpers/test_execution_result_document.py index 84ae015d09..f4e0ad4741 100644 --- a/tests/unit/src/sqlbuild/cli/output/_helpers/test_execution_result_document.py +++ b/tests/unit/src/sqlbuild/cli/output/_helpers/test_execution_result_document.py @@ -8,13 +8,18 @@ import pytest -from sqlbuild._native import format_audit_execution_json, format_parameterized_test_label -from sqlbuild.adapter.contract.models import LifeCycleEvent -from sqlbuild.adapter.contract.types import LifeCycleEventKind -from sqlbuild.compiler.auditing.models import ( +from sqlbuild._native import ( + FutureCursorAction, MeasurementThresholdBound, MeasurementThresholds, + SqlLogicalType, + SqlValue, + SqlValueKind, + format_audit_execution_json, + format_parameterized_test_label, ) +from sqlbuild.adapter.contract.models import LifeCycleEvent +from sqlbuild.adapter.contract.types import LifeCycleEventKind from sqlbuild.compiler.auditing.types import ( AuditAttachmentKind, AuditEvaluationMode, @@ -43,9 +48,6 @@ StepResult, ) from sqlbuild.executor.testing.types import SqlTestOutcome -from sqlbuild.spec.contracts.types import FutureCursorAction -from sqlbuild.sql_values.models import SqlLogicalType, SqlValue -from sqlbuild.sql_values.types import SqlValueKind from tests.unit.src.sqlbuild.cli.output._helpers._test_types import ( AuditExecutionProtocolTestCase, FailedModelSqlOutputTestCase, diff --git a/tests/unit/src/sqlbuild/cli/output/_helpers/test_integration_result_measurements.py b/tests/unit/src/sqlbuild/cli/output/_helpers/test_integration_result_measurements.py index 9502a17e50..07a291f365 100644 --- a/tests/unit/src/sqlbuild/cli/output/_helpers/test_integration_result_measurements.py +++ b/tests/unit/src/sqlbuild/cli/output/_helpers/test_integration_result_measurements.py @@ -6,8 +6,8 @@ import pytest -from sqlbuild._native import IntegrationCheckResult -from sqlbuild.compiler.auditing.models import ( +from sqlbuild._native import ( + IntegrationCheckResult, MeasurementThresholdBound, MeasurementThresholds, ) diff --git a/tests/unit/src/sqlbuild/cli/output/classes/test_terminal_event_index.py b/tests/unit/src/sqlbuild/cli/output/classes/test_terminal_event_index.py index 6177791a4d..5f1c6f1628 100644 --- a/tests/unit/src/sqlbuild/cli/output/classes/test_terminal_event_index.py +++ b/tests/unit/src/sqlbuild/cli/output/classes/test_terminal_event_index.py @@ -13,8 +13,10 @@ from sqlbuild._native import ( ExecutionEventWriter, + FutureCursorAction, IntegrationAssetResult, IntegrationResultEnvelope, + MicrobatchLimitAction, TerminalEventClaim, TerminalEventIndex, current_terminal_event_index, @@ -25,7 +27,6 @@ from sqlbuild.executor.scheduling.types import ExecutionStatus from sqlbuild.runtime.observability.exceptions import ObservabilityValidationError from sqlbuild.runtime.observability.models import LifecycleEvent -from sqlbuild.spec.contracts.types import FutureCursorAction, MicrobatchLimitAction from tests.unit.src.sqlbuild.cli.output.classes._test_types import ( CheckSeverityDecodingTestCase, EnvelopeFieldValidationTestCase, diff --git a/tests/unit/src/sqlbuild/cli/output/main/build_execution_json/test_execution_protocol_v1.py b/tests/unit/src/sqlbuild/cli/output/main/build_execution_json/test_execution_protocol_v1.py index a193b0dd5b..1d90242727 100644 --- a/tests/unit/src/sqlbuild/cli/output/main/build_execution_json/test_execution_protocol_v1.py +++ b/tests/unit/src/sqlbuild/cli/output/main/build_execution_json/test_execution_protocol_v1.py @@ -6,7 +6,7 @@ import pytest -from sqlbuild._native import format_build_execution_json +from sqlbuild._native import MicrobatchLimitAction, format_build_execution_json from sqlbuild.compiler.auditing.types import AuditOutcome from sqlbuild.compiler.planner.types import PlanReason from sqlbuild.compiler.python_nodes.types import PythonNodeKind, PythonNodeStatus @@ -15,7 +15,6 @@ from sqlbuild.executor.python_nodes.models import PythonNodeExecutionResult from sqlbuild.executor.run.models import ModelExecutionResult from sqlbuild.executor.scheduling.types import ExecutionStatus -from sqlbuild.spec.contracts.types import MicrobatchLimitAction from tests.unit.src.sqlbuild.cli.commands.shared._helpers.helpers import build_audit_result from tests.unit.src.sqlbuild.cli.output.main.build_execution_json._test_types import ( ExecutionJsonCostTestCase, diff --git a/tests/unit/src/sqlbuild/cli/output/main/plan/helpers.py b/tests/unit/src/sqlbuild/cli/output/main/plan/helpers.py index 11716681e2..c9699a1b5c 100644 --- a/tests/unit/src/sqlbuild/cli/output/main/plan/helpers.py +++ b/tests/unit/src/sqlbuild/cli/output/main/plan/helpers.py @@ -7,10 +7,13 @@ from sqlbuild._native import ( BackfillResult, ColumnRenameHint, + MicrobatchLimitAction, PlanProviderUsage, PlanWarning, SchemaChangeKind, SchemaColumnSource, + SeedCsvSettings, + SourceWriteStrategy, WarningSeverity, ) from sqlbuild.compiler.compile.models import ( @@ -36,8 +39,6 @@ PlanAction, PlanReason, ) -from sqlbuild.spec.contracts.models import SeedCsvSettings -from sqlbuild.spec.contracts.types import MicrobatchLimitAction, SourceWriteStrategy def build_model_entry( diff --git a/tests/unit/src/sqlbuild/cli/output/main/plan/test_plan_text.py b/tests/unit/src/sqlbuild/cli/output/main/plan/test_plan_text.py index 21cf8aca85..35b771b1b4 100644 --- a/tests/unit/src/sqlbuild/cli/output/main/plan/test_plan_text.py +++ b/tests/unit/src/sqlbuild/cli/output/main/plan/test_plan_text.py @@ -14,6 +14,7 @@ PythonPlanEntry, PythonRunPhase, SchemaChangeKind, + SourceWriteStrategy, WarningSeverity, format_plan, ) @@ -30,7 +31,6 @@ PlanReason, ) from sqlbuild.compiler.python_nodes.types import PythonNodeKind -from sqlbuild.spec.contracts.types import SourceWriteStrategy from tests.unit.src.sqlbuild.cli.output.main.plan._test_types import ( FormatPlanColorTestCase, FormatPlanTestCase, diff --git a/tests/unit/src/sqlbuild/cli/output/main/plan_json/test_plan_json.py b/tests/unit/src/sqlbuild/cli/output/main/plan_json/test_plan_json.py index b714af570a..9a9f860197 100644 --- a/tests/unit/src/sqlbuild/cli/output/main/plan_json/test_plan_json.py +++ b/tests/unit/src/sqlbuild/cli/output/main/plan_json/test_plan_json.py @@ -8,6 +8,8 @@ import pytest from sqlbuild._native import ( + FutureCursorAction, + MicrobatchLimitAction, PythonIdentityStatus, PythonPlanEntry, PythonRunPhase, @@ -29,7 +31,6 @@ PlanReason, ) from sqlbuild.compiler.python_nodes.types import PythonNodeKind -from sqlbuild.spec.contracts.types import FutureCursorAction, MicrobatchLimitAction from tests.unit.src.sqlbuild.cli.output.main.plan.helpers import ( build_discovered_provider_usage, build_model_entry, diff --git a/tests/unit/src/sqlbuild/compiler/auditing/helpers.py b/tests/unit/src/sqlbuild/compiler/auditing/helpers.py index 3288d3173e..ac8a127551 100644 --- a/tests/unit/src/sqlbuild/compiler/auditing/helpers.py +++ b/tests/unit/src/sqlbuild/compiler/auditing/helpers.py @@ -1,6 +1,6 @@ """Test helpers for auditing domain models.""" -from sqlbuild.compiler.auditing.models import MeasurementThresholdBound +from sqlbuild._native import MeasurementThresholdBound from sqlbuild.compiler.auditing.types import ThresholdOperator diff --git a/tests/unit/src/sqlbuild/compiler/auditing/main/_test_types.py b/tests/unit/src/sqlbuild/compiler/auditing/main/_test_types.py index ae1f3dd4ca..9014dc445c 100644 --- a/tests/unit/src/sqlbuild/compiler/auditing/main/_test_types.py +++ b/tests/unit/src/sqlbuild/compiler/auditing/main/_test_types.py @@ -1,6 +1,6 @@ from dataclasses import dataclass, field -from sqlbuild.compiler.auditing.models import MeasurementThresholds +from sqlbuild._native import MeasurementThresholds from sqlbuild.compiler.auditing.types import ( AuditAttachmentKind, AuditOutcome, diff --git a/tests/unit/src/sqlbuild/compiler/auditing/main/helpers.py b/tests/unit/src/sqlbuild/compiler/auditing/main/helpers.py index 79346aabf4..35dddf2ec8 100644 --- a/tests/unit/src/sqlbuild/compiler/auditing/main/helpers.py +++ b/tests/unit/src/sqlbuild/compiler/auditing/main/helpers.py @@ -7,7 +7,7 @@ from types import MappingProxyType from typing import cast -from sqlbuild._native import parse_audit_instances +from sqlbuild._native import SchemaAuditInstance, SourceEntry, parse_audit_instances from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter from sqlbuild.adapter.discovery.main.builtins import builtin_adapter_classes from sqlbuild.compiler.auditing.types import ( @@ -18,7 +18,6 @@ from sqlbuild.compiler.compile.models import CompiledObjectKey, CompiledRelationLocation from sqlbuild.compiler.compile.types import CompiledResourceType from sqlbuild.compiler.planner.models import AuditPlanEntry -from sqlbuild.spec.contracts.models import SchemaAuditInstance, SourceEntry def build_render_adapter(adapter_name: str | None) -> BaseAdapter | None: diff --git a/tests/unit/src/sqlbuild/compiler/auditing/main/test_evaluate_measurement.py b/tests/unit/src/sqlbuild/compiler/auditing/main/test_evaluate_measurement.py index 8b2dfc7135..5ad5c96978 100644 --- a/tests/unit/src/sqlbuild/compiler/auditing/main/test_evaluate_measurement.py +++ b/tests/unit/src/sqlbuild/compiler/auditing/main/test_evaluate_measurement.py @@ -2,11 +2,7 @@ import pytest -from sqlbuild._native import evaluate_measurement -from sqlbuild.compiler.auditing.models import ( - MeasurementThresholdBound, - MeasurementThresholds, -) +from sqlbuild._native import MeasurementThresholdBound, MeasurementThresholds, evaluate_measurement from sqlbuild.compiler.auditing.types import AuditOutcome, ThresholdOperator from tests.unit.src.sqlbuild.compiler.auditing.main._test_types import ( EvaluateMeasurementTestCase, diff --git a/tests/unit/src/sqlbuild/compiler/auditing/main/test_identity.py b/tests/unit/src/sqlbuild/compiler/auditing/main/test_identity.py index c9867d00a4..26ddcceb12 100644 --- a/tests/unit/src/sqlbuild/compiler/auditing/main/test_identity.py +++ b/tests/unit/src/sqlbuild/compiler/auditing/main/test_identity.py @@ -6,12 +6,12 @@ import pytest -from sqlbuild._native import build_audit_gate_identity -from sqlbuild.compiler.auditing.models import ( +from sqlbuild._native import ( AuditGateIdentity, AuditIdentity, MeasurementThresholdBound, MeasurementThresholds, + build_audit_gate_identity, ) from sqlbuild.compiler.auditing.types import ( AuditAttachmentKind, diff --git a/tests/unit/src/sqlbuild/compiler/auditing/main/test_parse_audit_instance.py b/tests/unit/src/sqlbuild/compiler/auditing/main/test_parse_audit_instance.py index e8731420e6..2ff4dfb4f9 100644 --- a/tests/unit/src/sqlbuild/compiler/auditing/main/test_parse_audit_instance.py +++ b/tests/unit/src/sqlbuild/compiler/auditing/main/test_parse_audit_instance.py @@ -4,8 +4,8 @@ import pytest +from sqlbuild._native import SchemaAuditInstance from sqlbuild.compiler.auditing.types import AuditSeverity, ThresholdOperator -from sqlbuild.spec.contracts.models import SchemaAuditInstance from tests.unit.src.sqlbuild.compiler.auditing.main._test_types import ( MeasurementPolicyParsingTestCase, ParseAuditInstanceErrorTestCase, diff --git a/tests/unit/src/sqlbuild/compiler/auditing/main/test_parse_audit_instances.py b/tests/unit/src/sqlbuild/compiler/auditing/main/test_parse_audit_instances.py index f9476e57a5..3466972905 100644 --- a/tests/unit/src/sqlbuild/compiler/auditing/main/test_parse_audit_instances.py +++ b/tests/unit/src/sqlbuild/compiler/auditing/main/test_parse_audit_instances.py @@ -4,8 +4,7 @@ import pytest -from sqlbuild._native import parse_audit_instances -from sqlbuild.spec.contracts.models import SchemaAuditInstance +from sqlbuild._native import SchemaAuditInstance, parse_audit_instances from tests.unit.src.sqlbuild.compiler.auditing.main._test_types import ( ParseAuditInstancesErrorTestCase, ParseAuditInstancesTestCase, diff --git a/tests/unit/src/sqlbuild/compiler/auditing/main/test_rendering.py b/tests/unit/src/sqlbuild/compiler/auditing/main/test_rendering.py index 35e91797fe..d155968bc5 100644 --- a/tests/unit/src/sqlbuild/compiler/auditing/main/test_rendering.py +++ b/tests/unit/src/sqlbuild/compiler/auditing/main/test_rendering.py @@ -4,10 +4,9 @@ import pytest -from sqlbuild._native import render_audit_sql +from sqlbuild._native import SourceEntry, render_audit_sql from sqlbuild.adapter.contract.types import BuiltinAdapter from sqlbuild.compiler.compile.models import CompiledRelationLocation -from sqlbuild.spec.contracts.models import SourceEntry from tests.unit.src.sqlbuild.compiler.auditing.main._test_types import ( RenderAuditSqlTestCase, ) diff --git a/tests/unit/src/sqlbuild/compiler/auditing/test_models.py b/tests/unit/src/sqlbuild/compiler/auditing/test_models.py index 6b698c908b..a3fa5deb10 100644 --- a/tests/unit/src/sqlbuild/compiler/auditing/test_models.py +++ b/tests/unit/src/sqlbuild/compiler/auditing/test_models.py @@ -2,12 +2,8 @@ import pytest +from sqlbuild._native import MeasurementContract, MeasurementThresholdBound, MeasurementThresholds from sqlbuild.audits import below -from sqlbuild.compiler.auditing.models import ( - MeasurementContract, - MeasurementThresholdBound, - MeasurementThresholds, -) from sqlbuild.compiler.auditing.types import ThresholdOperator from tests.unit.src.sqlbuild.compiler.auditing._test_types import ( BooleanThresholdTestCase, diff --git a/tests/unit/src/sqlbuild/compiler/compile/_helpers/_test_types.py b/tests/unit/src/sqlbuild/compiler/compile/_helpers/_test_types.py index 2c3f460f5c..fa5ec544d3 100644 --- a/tests/unit/src/sqlbuild/compiler/compile/_helpers/_test_types.py +++ b/tests/unit/src/sqlbuild/compiler/compile/_helpers/_test_types.py @@ -5,7 +5,15 @@ from pathlib import Path from typing import TYPE_CHECKING -from sqlbuild._native import CompiledLineageColumnFact, ExpansionSpan, UsageRecord +from sqlbuild._native import ( + CompiledLineageColumnFact, + ExpansionSpan, + ResolvedTimeTravelRetention, + SourceLocation, + SqlLexicalSyntax, + SqlValue, + UsageRecord, +) from sqlbuild.adapter.contract.models import ExpressionInferenceProfile from sqlbuild.compiler.compile.models import CompiledObjectKey, CompileSqlReference, InferredColumn from sqlbuild.compiler.compile.types import AttachedAuditTargetKind @@ -13,9 +21,6 @@ InferredNullability, ) from sqlbuild.compiler.references.types import SqlReferenceKind -from sqlbuild.compiler.sql_analysis.models import SqlLexicalSyntax -from sqlbuild.spec.contracts.models import ResolvedTimeTravelRetention, SourceLocation -from sqlbuild.sql_values.models import SqlValue if TYPE_CHECKING: from sqlbuild._native import CompileInputsView diff --git a/tests/unit/src/sqlbuild/compiler/compile/_helpers/analysis/test_dynamic_pivot.py b/tests/unit/src/sqlbuild/compiler/compile/_helpers/analysis/test_dynamic_pivot.py index 547af16fd6..8b870dd850 100644 --- a/tests/unit/src/sqlbuild/compiler/compile/_helpers/analysis/test_dynamic_pivot.py +++ b/tests/unit/src/sqlbuild/compiler/compile/_helpers/analysis/test_dynamic_pivot.py @@ -5,10 +5,10 @@ from sqlbuild._native import ( DynamicColumnContractProof, NativePivotTables, + SchemaDynamicColumnFamily, prove_native_dynamic_contracts, ) from sqlbuild._native import dynamic_contract_proof as contract_proof -from sqlbuild.spec.contracts.models import SchemaDynamicColumnFamily from tests.unit.src.sqlbuild.compiler.compile._helpers.analysis._test_types import ( DynamicPivotAnalysisTestCase, ) diff --git a/tests/unit/src/sqlbuild/compiler/compile/_helpers/config/_test_types.py b/tests/unit/src/sqlbuild/compiler/compile/_helpers/config/_test_types.py index abcfe1084f..2d518305b9 100644 --- a/tests/unit/src/sqlbuild/compiler/compile/_helpers/config/_test_types.py +++ b/tests/unit/src/sqlbuild/compiler/compile/_helpers/config/_test_types.py @@ -1,7 +1,6 @@ from dataclasses import dataclass -from sqlbuild.spec.contracts.models import MaterializationDefaultsConfig, TargetConfig -from sqlbuild.spec.contracts.types import TableType, TableTypeSource +from sqlbuild._native import MaterializationDefaultsConfig, TableType, TableTypeSource, TargetConfig @dataclass(frozen=True) diff --git a/tests/unit/src/sqlbuild/compiler/compile/_helpers/config/helpers.py b/tests/unit/src/sqlbuild/compiler/compile/_helpers/config/helpers.py index 0412e70802..e22b83aaab 100644 --- a/tests/unit/src/sqlbuild/compiler/compile/_helpers/config/helpers.py +++ b/tests/unit/src/sqlbuild/compiler/compile/_helpers/config/helpers.py @@ -8,23 +8,23 @@ import sqlbuild._native as _native from sqlbuild._native import ( CompileModelConfig, - NativeModelConfigInputs, - NativeModelConfigSession, - build_compile_inputs, - discover_project_inputs, - native_model_config_session, - record_template_reads, -) -from sqlbuild.spec.contracts.models import ( DefaultsConfig, MaterializationDefaultsConfig, + NativeModelConfigInputs, + NativeModelConfigSession, ProjectConfig, ResolvedTableType, ResolvedTimeTravelRetention, SchemaColumn, + TableType, + TableTypeSource, TargetConfig, + TimeTravelRetentionSource, + build_compile_inputs, + discover_project_inputs, + native_model_config_session, + record_template_reads, ) -from sqlbuild.spec.contracts.types import TableType, TableTypeSource, TimeTravelRetentionSource from tests.unit.src.sqlbuild.compiler.compile._helpers.helpers import ( DUCKDB_COMPILE_ADAPTER_CONTEXT, ) diff --git a/tests/unit/src/sqlbuild/compiler/compile/_helpers/config/test_table_type.py b/tests/unit/src/sqlbuild/compiler/compile/_helpers/config/test_table_type.py index dec868dd0b..7b1e157a0c 100644 --- a/tests/unit/src/sqlbuild/compiler/compile/_helpers/config/test_table_type.py +++ b/tests/unit/src/sqlbuild/compiler/compile/_helpers/config/test_table_type.py @@ -2,16 +2,17 @@ import pytest -from sqlbuild._native import CompileModelConfig -from sqlbuild.compiler.compile.exceptions import CompileInputError -from sqlbuild.spec.contracts.models import ( +from sqlbuild._native import ( + CompileModelConfig, DefaultsConfig, MaterializationDefaultsConfig, MaterializationRetentionDefaults, ResolvedTableType, + TableType, + TableTypeSource, TargetConfig, ) -from sqlbuild.spec.contracts.types import TableType, TableTypeSource +from sqlbuild.compiler.compile.exceptions import CompileInputError from tests.unit.src.sqlbuild.compiler.compile._helpers.config._test_types import ( TableTypeResolutionErrorTestCase, TableTypeResolutionTestCase, diff --git a/tests/unit/src/sqlbuild/compiler/compile/_helpers/test_audit_factories.py b/tests/unit/src/sqlbuild/compiler/compile/_helpers/test_audit_factories.py index 3404d677c3..8f541715e3 100644 --- a/tests/unit/src/sqlbuild/compiler/compile/_helpers/test_audit_factories.py +++ b/tests/unit/src/sqlbuild/compiler/compile/_helpers/test_audit_factories.py @@ -8,9 +8,13 @@ import pytest -from sqlbuild._native import build_compile_inputs, discover_project_inputs +from sqlbuild._native import ( + SchemaModelEntry, + SourceLocation, + build_compile_inputs, + discover_project_inputs, +) from sqlbuild.compiler.compile.exceptions import CompileInputError -from sqlbuild.spec.contracts.models import SchemaModelEntry, SourceLocation from tests.unit.src.sqlbuild.compiler.compile._helpers._test_types import ( AuditFactoryAttachmentTestCase, ) diff --git a/tests/unit/src/sqlbuild/compiler/compile/_helpers/test_declaration_scopes.py b/tests/unit/src/sqlbuild/compiler/compile/_helpers/test_declaration_scopes.py index 86b583e1d2..091c2703d9 100644 --- a/tests/unit/src/sqlbuild/compiler/compile/_helpers/test_declaration_scopes.py +++ b/tests/unit/src/sqlbuild/compiler/compile/_helpers/test_declaration_scopes.py @@ -14,6 +14,7 @@ DeclarationKind, ResourceIdentity, ResourceKind, + SourceEntry, UsageKind, UsageRecord, assemble_project, @@ -28,7 +29,6 @@ DiscoveredSourceFile, SqlHookEntry, ) -from sqlbuild.spec.contracts.models import SourceEntry from tests.unit.src.sqlbuild.compiler.compile._helpers._test_types import ( ExpectedBooleanTestCase, ExpectedModelDeclarationGrantTestCase, diff --git a/tests/unit/src/sqlbuild/compiler/compile/_helpers/test_microbatch_redesign.py b/tests/unit/src/sqlbuild/compiler/compile/_helpers/test_microbatch_redesign.py index 7e93361a45..0be866e8d0 100644 --- a/tests/unit/src/sqlbuild/compiler/compile/_helpers/test_microbatch_redesign.py +++ b/tests/unit/src/sqlbuild/compiler/compile/_helpers/test_microbatch_redesign.py @@ -6,13 +6,14 @@ from sqlbuild._native import ( CompileModelConfig, + DateValue, + TimestampValue, compute_cursor_bounds, resolve_effective_timestamp_grain, watermark_type_is_compatible, ) from sqlbuild.compiler.compile.exceptions import CompileInputError from sqlbuild.compiler.planner.models import CursorBounds, CursorInputRelation, ModelCursorSnapshot -from sqlbuild.cursor_algebra.models import DateValue, TimestampValue from sqlbuild.executor.run._helpers.validation.cursor_bounds import ( resolve_effective_timestamp_grain as resolve_runtime_effective_timestamp_grain, ) diff --git a/tests/unit/src/sqlbuild/compiler/compile/_helpers/test_model_config_validation.py b/tests/unit/src/sqlbuild/compiler/compile/_helpers/test_model_config_validation.py index c0948d8dfe..17b779ab5a 100644 --- a/tests/unit/src/sqlbuild/compiler/compile/_helpers/test_model_config_validation.py +++ b/tests/unit/src/sqlbuild/compiler/compile/_helpers/test_model_config_validation.py @@ -6,10 +6,13 @@ import pytest -from sqlbuild._native import CompileModelConfig, ConfigValueTypeError +from sqlbuild._native import ( + CompileModelConfig, + ConfigValueTypeError, + ResolvedTimeTravelRetention, + TimeTravelRetentionSource, +) from sqlbuild.compiler.compile.exceptions import CompileInputError -from sqlbuild.spec.contracts.models import ResolvedTimeTravelRetention -from sqlbuild.spec.contracts.types import TimeTravelRetentionSource from tests.unit.src.sqlbuild.compiler.compile._helpers._test_types import ( ChangePolicyHelpTestCase, ContractConfigErrorTestCase, diff --git a/tests/unit/src/sqlbuild/compiler/compile/_helpers/test_model_schemas.py b/tests/unit/src/sqlbuild/compiler/compile/_helpers/test_model_schemas.py index 7440b70bba..a341e5b8e3 100644 --- a/tests/unit/src/sqlbuild/compiler/compile/_helpers/test_model_schemas.py +++ b/tests/unit/src/sqlbuild/compiler/compile/_helpers/test_model_schemas.py @@ -8,6 +8,9 @@ import pytest from sqlbuild._native import ( + SchemaAuditInstance, + SchemaModelEntry, + SourceLocation, assemble_project, build_compile_inputs, build_model_version_identity_metadata_json, @@ -21,7 +24,6 @@ ) from sqlbuild.compiler.compile.types import CompiledModelView, CompiledProjectView from sqlbuild.compiler.discovery.models import DiscoveredProjectInputs -from sqlbuild.spec.contracts.models import SchemaAuditInstance, SchemaModelEntry, SourceLocation from tests.unit.src.sqlbuild.compiler.compile._helpers._test_types import ( DynamicModelSchemaTestCase, ExpectedCountTestCase, diff --git a/tests/unit/src/sqlbuild/compiler/compile/_helpers/test_parameterized_sql_tests.py b/tests/unit/src/sqlbuild/compiler/compile/_helpers/test_parameterized_sql_tests.py index 1a3b2a3110..02e5a47452 100644 --- a/tests/unit/src/sqlbuild/compiler/compile/_helpers/test_parameterized_sql_tests.py +++ b/tests/unit/src/sqlbuild/compiler/compile/_helpers/test_parameterized_sql_tests.py @@ -7,7 +7,13 @@ import pytest -from sqlbuild._native import assemble_project, scope_overlaps +from sqlbuild._native import ( + SqlLogicalType, + SqlValue, + SqlValueKind, + assemble_project, + scope_overlaps, +) from sqlbuild.adapters.bigquery.classes.bigquery_adapter import BigQueryAdapter from sqlbuild.adapters.sqlserver.classes.sqlserver_adapter import SqlServerAdapter from sqlbuild.compiler.compile.exceptions import CompileInputError @@ -20,8 +26,6 @@ CompiledResourceType, TypedSqlValueRenderer, ) -from sqlbuild.sql_values.models import SqlLogicalType, SqlValue -from sqlbuild.sql_values.types import SqlValueKind from tests.unit.src.sqlbuild.compiler.compile._helpers._test_types import ( ParameterizedSqlTestAdapterRenderingTestCase, ParameterizedSqlTestCompilationErrorTestCase, diff --git a/tests/unit/src/sqlbuild/compiler/compile/_helpers/test_rebind_declaration_scope.py b/tests/unit/src/sqlbuild/compiler/compile/_helpers/test_rebind_declaration_scope.py index 69457b3fb4..7bbf5d55e4 100644 --- a/tests/unit/src/sqlbuild/compiler/compile/_helpers/test_rebind_declaration_scope.py +++ b/tests/unit/src/sqlbuild/compiler/compile/_helpers/test_rebind_declaration_scope.py @@ -11,6 +11,7 @@ DeclarationIdentity, DeclarationKind, DeclarationScopeBuild, + SqlLexicalSyntax, build_declaration_scope, discover_project_inputs, rebind_declaration_scope, @@ -18,7 +19,6 @@ from sqlbuild.compiler.compile._helpers.render.user_macros import load_project_macros from sqlbuild.compiler.compile.models import LoadedMacro from sqlbuild.compiler.discovery.models import DiscoveredProjectInputs -from sqlbuild.compiler.sql_analysis.models import SqlLexicalSyntax from tests.unit.src.sqlbuild.compiler.compile._helpers._test_types import ( RebindDeclarationScopeTestCase, ) diff --git a/tests/unit/src/sqlbuild/compiler/compile/_helpers/test_sql_references.py b/tests/unit/src/sqlbuild/compiler/compile/_helpers/test_sql_references.py index fcb2625574..e182b40a38 100644 --- a/tests/unit/src/sqlbuild/compiler/compile/_helpers/test_sql_references.py +++ b/tests/unit/src/sqlbuild/compiler/compile/_helpers/test_sql_references.py @@ -6,7 +6,7 @@ import pytest -from sqlbuild._native import collect_compile_diagnostics, extract_sql_references +from sqlbuild._native import SqlLexicalSyntax, collect_compile_diagnostics, extract_sql_references from sqlbuild.adapters.bigquery.classes.bigquery_adapter import BigQueryAdapter from sqlbuild.adapters.databricks.classes.databricks_adapter import DatabricksAdapter from sqlbuild.adapters.duckdb.classes.duckdb_adapter import DuckDbAdapter @@ -15,7 +15,6 @@ from sqlbuild.compiler.compile.exceptions import CompileInputError from sqlbuild.compiler.compile.models import CompileSqlReference from sqlbuild.compiler.references.types import SqlReferenceKind -from sqlbuild.compiler.sql_analysis.models import SqlLexicalSyntax from tests.unit.src.sqlbuild.compiler.compile._helpers._test_types import ( CollectedReferenceCallSyntaxTestCase, DialectSqlScanTestCase, diff --git a/tests/unit/src/sqlbuild/compiler/compile/_test_types.py b/tests/unit/src/sqlbuild/compiler/compile/_test_types.py index f1952b41bd..6039b5cc98 100644 --- a/tests/unit/src/sqlbuild/compiler/compile/_test_types.py +++ b/tests/unit/src/sqlbuild/compiler/compile/_test_types.py @@ -1,11 +1,11 @@ from dataclasses import dataclass, field -from sqlbuild.spec.contracts.models import ( +from sqlbuild._native import ( DefaultsConfig, MaterializationDefaultsConfig, TargetConfig, + TimeTravelRetentionSource, ) -from sqlbuild.spec.contracts.types import TimeTravelRetentionSource @dataclass(frozen=True) diff --git a/tests/unit/src/sqlbuild/compiler/compile/test_effective_runtime.py b/tests/unit/src/sqlbuild/compiler/compile/test_effective_runtime.py index 6617c86aa4..bb8bc01f11 100644 --- a/tests/unit/src/sqlbuild/compiler/compile/test_effective_runtime.py +++ b/tests/unit/src/sqlbuild/compiler/compile/test_effective_runtime.py @@ -4,14 +4,15 @@ import pytest -from sqlbuild._native import build_effective_runtime_config, resolve_run_id -from sqlbuild.compiler.discovery.models import DiscoveredProjectInputs -from sqlbuild.spec.contracts.models import ( +from sqlbuild._native import ( LocalConfig, LocalTargetConfig, ProjectConfig, TargetConfig, + build_effective_runtime_config, + resolve_run_id, ) +from sqlbuild.compiler.discovery.models import DiscoveredProjectInputs from tests.unit.src.sqlbuild.compiler.compile._test_types import ( BuildEffectiveRuntimeConfigTestCase, RunIdGenerationTestCase, diff --git a/tests/unit/src/sqlbuild/compiler/compile/test_main.py b/tests/unit/src/sqlbuild/compiler/compile/test_main.py index 8729c83215..48712b6902 100644 --- a/tests/unit/src/sqlbuild/compiler/compile/test_main.py +++ b/tests/unit/src/sqlbuild/compiler/compile/test_main.py @@ -9,21 +9,9 @@ import pytest from sqlbuild._native import ( - CompileModelConfig, - ResourceIdentityError, - SpecConfigError, - build_compile_inputs, - discover_project_inputs, - resolve_target_config, -) -from sqlbuild.compiler.discovery.models import ( - DiscoveredProjectInputs, - PythonHookEntry, - SqlHookEntry, -) -from sqlbuild.spec.contracts.models import ( AuthoredTimeTravelRetention, ClonePolicy, + CompileModelConfig, DefaultsConfig, LocalClonePolicy, LocalConfig, @@ -32,9 +20,19 @@ MaterializationRetentionDefaults, ProjectConfig, ResolvedTimeTravelRetention, + ResourceIdentityError, + SpecConfigError, TargetConfig, + TimeTravelRetentionSource, + build_compile_inputs, + discover_project_inputs, + resolve_target_config, +) +from sqlbuild.compiler.discovery.models import ( + DiscoveredProjectInputs, + PythonHookEntry, + SqlHookEntry, ) -from sqlbuild.spec.contracts.types import TimeTravelRetentionSource from tests.unit.src.sqlbuild.compiler.compile._helpers.config.helpers import build_config_natively from tests.unit.src.sqlbuild.compiler.compile._helpers.helpers import ( DUCKDB_COMPILE_ADAPTER_CONTEXT, diff --git a/tests/unit/src/sqlbuild/compiler/contract_adoption/_helpers/test_repository_edits.py b/tests/unit/src/sqlbuild/compiler/contract_adoption/_helpers/test_repository_edits.py index 6360d6504a..3aed31fcf7 100644 --- a/tests/unit/src/sqlbuild/compiler/contract_adoption/_helpers/test_repository_edits.py +++ b/tests/unit/src/sqlbuild/compiler/contract_adoption/_helpers/test_repository_edits.py @@ -1,10 +1,14 @@ import pytest -from sqlbuild._native import ContractEvidence, compare_contract_evidence, edit_contract_model +from sqlbuild._native import ( + ContractEvidence, + SchemaDynamicColumnFamily, + compare_contract_evidence, + edit_contract_model, +) from sqlbuild.adapter.contract.models import ColumnInfo from sqlbuild.adapters.duckdb.classes.duckdb_adapter import DuckDbAdapter from sqlbuild.compiler.compile.types import CompiledModelView, CompiledResourceType -from sqlbuild.spec.contracts.models import SchemaDynamicColumnFamily from tests.unit.src.sqlbuild.compiler.contract_adoption._helpers._test_types import ( DynamicContractAdoptionTestCase, ) diff --git a/tests/unit/src/sqlbuild/compiler/contracts/helpers.py b/tests/unit/src/sqlbuild/compiler/contracts/helpers.py index e8f754aaa2..b038b893ce 100644 --- a/tests/unit/src/sqlbuild/compiler/contracts/helpers.py +++ b/tests/unit/src/sqlbuild/compiler/contracts/helpers.py @@ -3,7 +3,17 @@ from pathlib import Path from typing import cast -from sqlbuild._native import CompileModelConfig, DynamicColumnContractProof +from sqlbuild._native import ( + ColumnContractMode, + CompileModelConfig, + DynamicColumnContractProof, + SchemaAuditInstance, + SchemaColumn, + SchemaDynamicColumnFamily, + SchemaModelEntry, + SettingsConfig, + SourceLocation, +) from sqlbuild.compiler.compile.models import ( CompiledModel, CompiledObjectKey, @@ -13,15 +23,6 @@ ) from sqlbuild.compiler.compile.types import CompiledProjectView, CompiledResourceType from sqlbuild.compiler.lineage.types import InferredNullability -from sqlbuild.spec.contracts.models import ( - SchemaAuditInstance, - SchemaColumn, - SchemaDynamicColumnFamily, - SchemaModelEntry, - SettingsConfig, - SourceLocation, -) -from sqlbuild.spec.contracts.types import ColumnContractMode def make_contract_project( diff --git a/tests/unit/src/sqlbuild/compiler/contracts/test_validate.py b/tests/unit/src/sqlbuild/compiler/contracts/test_validate.py index 47fb4428f7..6bb37dd1d1 100644 --- a/tests/unit/src/sqlbuild/compiler/contracts/test_validate.py +++ b/tests/unit/src/sqlbuild/compiler/contracts/test_validate.py @@ -8,11 +8,12 @@ ContractValidationResult, DynamicColumnContractProof, DynamicColumnFamilyProof, + SchemaDynamicColumnFamily, + SourceLocation, evaluate_model_contracts, ) from sqlbuild.adapter.contract.types import TypeDialect from sqlbuild.compiler.lineage.types import InferredNullability -from sqlbuild.spec.contracts.models import SchemaDynamicColumnFamily, SourceLocation from tests.unit.src.sqlbuild.compiler.contracts._test_types import ( ContractLocationTestCase, ContractMissingDeclarationsTestCase, diff --git a/tests/unit/src/sqlbuild/compiler/dag/main/helpers.py b/tests/unit/src/sqlbuild/compiler/dag/main/helpers.py index 49834cbe00..e3d2acdf3f 100644 --- a/tests/unit/src/sqlbuild/compiler/dag/main/helpers.py +++ b/tests/unit/src/sqlbuild/compiler/dag/main/helpers.py @@ -13,6 +13,11 @@ DiscoveredSeedFile, DiscoveredSqlTestBlock, FunctionArgument, + SchemaColumn, + SchemaModelEntry, + SchemaSeedEntry, + SourceColumnEntry, + SourceEntry, ) from sqlbuild.compiler.auditing.types import AuditSeverity from sqlbuild.compiler.compile.models import ( @@ -37,13 +42,6 @@ DiscoveredSqlTestFile, ) from sqlbuild.compiler.pipeline.models import ProjectGraph -from sqlbuild.spec.contracts.models import ( - SchemaColumn, - SchemaModelEntry, - SchemaSeedEntry, - SourceColumnEntry, - SourceEntry, -) from tests.unit.src.sqlbuild.compiler.graph.helpers import project_graph_from_indexes diff --git a/tests/unit/src/sqlbuild/compiler/discovery/_helpers/_test_types.py b/tests/unit/src/sqlbuild/compiler/discovery/_helpers/_test_types.py index 12997eba18..1b4112bbc6 100644 --- a/tests/unit/src/sqlbuild/compiler/discovery/_helpers/_test_types.py +++ b/tests/unit/src/sqlbuild/compiler/discovery/_helpers/_test_types.py @@ -2,9 +2,13 @@ from decimal import Decimal from pathlib import Path -from sqlbuild._native import CollectionRendering -from sqlbuild.spec.contracts.models import AuthoredTimeTravelRetention, TargetWarehousesConfig -from sqlbuild.spec.contracts.types import MissingMigrationOriginPolicy, RetentionDecreasePolicy +from sqlbuild._native import ( + AuthoredTimeTravelRetention, + CollectionRendering, + MissingMigrationOriginPolicy, + RetentionDecreasePolicy, + TargetWarehousesConfig, +) @dataclass(frozen=True) diff --git a/tests/unit/src/sqlbuild/compiler/discovery/_helpers/helpers.py b/tests/unit/src/sqlbuild/compiler/discovery/_helpers/helpers.py index 0d0d63f921..9dc0bcbff9 100644 --- a/tests/unit/src/sqlbuild/compiler/discovery/_helpers/helpers.py +++ b/tests/unit/src/sqlbuild/compiler/discovery/_helpers/helpers.py @@ -9,6 +9,8 @@ DiscoveredPythonFunctionFile, DiscoveredSqlTestBlock, DiscoveredSqlTestCase, + LocalConfig, + ProjectConfig, discover_audit_files, discover_constant_files, discover_enum_files, @@ -19,7 +21,6 @@ discover_sql_hook_files, ) from sqlbuild.compiler.discovery.models import DiscoveredMacroFile, SqlTestParameterDeclaration -from sqlbuild.spec.contracts.models import LocalConfig, ProjectConfig from tests.unit.src.sqlbuild.compiler.discovery._helpers._test_types import ( LoadProjectConfigTestCase, ) diff --git a/tests/unit/src/sqlbuild/compiler/discovery/_helpers/test_filesystem_python_nodes.py b/tests/unit/src/sqlbuild/compiler/discovery/_helpers/test_filesystem_python_nodes.py index 34dfb34ca0..e571870469 100644 --- a/tests/unit/src/sqlbuild/compiler/discovery/_helpers/test_filesystem_python_nodes.py +++ b/tests/unit/src/sqlbuild/compiler/discovery/_helpers/test_filesystem_python_nodes.py @@ -21,7 +21,6 @@ DiscoveredTaskFunction, ) from sqlbuild.compiler.python_nodes.main.identity import build_python_node_identity -from sqlbuild.compiler.python_nodes.models import PythonNodeIdentity from tests.unit.src.sqlbuild.compiler.discovery._helpers._test_types import ( DiscoverCheckFunctionsTestCase, DiscoverPythonRootImportTestCase, diff --git a/tests/unit/src/sqlbuild/compiler/discovery/_helpers/test_sql_declarations.py b/tests/unit/src/sqlbuild/compiler/discovery/_helpers/test_sql_declarations.py index 2c8db4c589..c466332472 100644 --- a/tests/unit/src/sqlbuild/compiler/discovery/_helpers/test_sql_declarations.py +++ b/tests/unit/src/sqlbuild/compiler/discovery/_helpers/test_sql_declarations.py @@ -15,9 +15,9 @@ DiscoveredModelSchemaFile, ModelSchemaDeclaration, ResourceIdentityError, + SqlValue, ) from sqlbuild.compiler.discovery.models import ConstantDeclaration, EnumDeclaration -from sqlbuild.sql_values.models import SqlValue from tests.unit.src.sqlbuild.compiler.discovery._helpers._test_types import ( DeclarationResourceIdentityErrorTestCase, ParseDeclarationFileErrorTestCase, diff --git a/tests/unit/src/sqlbuild/compiler/discovery/_helpers/test_sql_models.py b/tests/unit/src/sqlbuild/compiler/discovery/_helpers/test_sql_models.py index 35921e30df..9905d83633 100644 --- a/tests/unit/src/sqlbuild/compiler/discovery/_helpers/test_sql_models.py +++ b/tests/unit/src/sqlbuild/compiler/discovery/_helpers/test_sql_models.py @@ -10,11 +10,11 @@ AuthoredSqlValueCall, DiscoveryFileFault, NamedSqlHookEntry, + SourceLocation, discover_model_files, model_output_column_locations, ) from sqlbuild.compiler.discovery.models import DiscoveredSqlModelFile, PythonHookEntry, SqlHookEntry -from sqlbuild.spec.contracts.models import SourceLocation from tests.unit.src.sqlbuild.compiler.discovery._helpers._test_types import ( DeferredModelOutputLocationTestCase, ExpectedBooleanTestCase, diff --git a/tests/unit/src/sqlbuild/compiler/discovery/_helpers/test_sql_tests.py b/tests/unit/src/sqlbuild/compiler/discovery/_helpers/test_sql_tests.py index 35f64e200f..051cce0841 100644 --- a/tests/unit/src/sqlbuild/compiler/discovery/_helpers/test_sql_tests.py +++ b/tests/unit/src/sqlbuild/compiler/discovery/_helpers/test_sql_tests.py @@ -10,12 +10,12 @@ DiscoveredSqlTestBlock, DiscoveredSqlTestCase, DiscoveryFileFault, + SqlLexicalSyntax, SqlTestParseError, discover_test_files, omitted_ceremonial_select, ) from sqlbuild.compiler.discovery.models import DiscoveredSqlTestFile, SqlTestParameterDeclaration -from sqlbuild.compiler.sql_analysis.models import SqlLexicalSyntax from tests.unit.src.sqlbuild.compiler.discovery._helpers._test_types import ( ExpectedCountTestCase, OmittedSelectPerformanceTestCase, diff --git a/tests/unit/src/sqlbuild/compiler/discovery/_helpers/test_target_warehouses.py b/tests/unit/src/sqlbuild/compiler/discovery/_helpers/test_target_warehouses.py index 1bfc803853..6a8407e8f8 100644 --- a/tests/unit/src/sqlbuild/compiler/discovery/_helpers/test_target_warehouses.py +++ b/tests/unit/src/sqlbuild/compiler/discovery/_helpers/test_target_warehouses.py @@ -6,9 +6,8 @@ import pytest -from sqlbuild._native import ProjectConfigError, discover_project_inputs +from sqlbuild._native import ProjectConfigError, TargetWarehousesConfig, discover_project_inputs from sqlbuild.compiler.discovery.models import DiscoveredProjectInputs -from sqlbuild.spec.contracts.models import TargetWarehousesConfig from tests.unit.src.sqlbuild.compiler.discovery._helpers._test_types import ( TargetWarehousesConfigErrorTestCase, TargetWarehousesConfigTestCase, diff --git a/tests/unit/src/sqlbuild/compiler/discovery/_helpers/test_yml_project.py b/tests/unit/src/sqlbuild/compiler/discovery/_helpers/test_yml_project.py index a70f2a3a52..81809b20ff 100644 --- a/tests/unit/src/sqlbuild/compiler/discovery/_helpers/test_yml_project.py +++ b/tests/unit/src/sqlbuild/compiler/discovery/_helpers/test_yml_project.py @@ -6,16 +6,18 @@ import pytest -from sqlbuild._native import CollectionRendering, ProjectConfigError -from sqlbuild.runtime.event_exporting.types import LifecycleEventKind -from sqlbuild.spec.contracts.models import ( +from sqlbuild._native import ( AuthoredTimeTravelRetention, + CollectionRendering, ExecutionLimitsConfig, LocalConfig, + MissingMigrationOriginPolicy, ProjectConfig, + ProjectConfigError, + RetentionDecreasePolicy, TargetConfig, ) -from sqlbuild.spec.contracts.types import MissingMigrationOriginPolicy, RetentionDecreasePolicy +from sqlbuild.runtime.event_exporting.types import LifecycleEventKind from tests.unit.src.sqlbuild.compiler.discovery._helpers._test_types import ( ColumnContractModeConfigErrorTestCase, ColumnContractModeConfigTestCase, diff --git a/tests/unit/src/sqlbuild/compiler/discovery/_helpers/test_yml_schema.py b/tests/unit/src/sqlbuild/compiler/discovery/_helpers/test_yml_schema.py index c6440d4e47..1da48302b4 100644 --- a/tests/unit/src/sqlbuild/compiler/discovery/_helpers/test_yml_schema.py +++ b/tests/unit/src/sqlbuild/compiler/discovery/_helpers/test_yml_schema.py @@ -4,12 +4,7 @@ import pytest -from sqlbuild.spec.contracts.models import ( - SchemaColumn, - SchemaModelEntry, - SchemaSeedEntry, - SeedCsvSettings, -) +from sqlbuild._native import SchemaColumn, SchemaModelEntry, SchemaSeedEntry, SeedCsvSettings from tests.unit.src.sqlbuild.compiler.discovery._helpers._test_types import ( ParseSchemaYamlErrorTestCase, ParseSchemaYamlTestCase, diff --git a/tests/unit/src/sqlbuild/compiler/discovery/_helpers/test_yml_sources.py b/tests/unit/src/sqlbuild/compiler/discovery/_helpers/test_yml_sources.py index 721e2e09c3..fd57f1a902 100644 --- a/tests/unit/src/sqlbuild/compiler/discovery/_helpers/test_yml_sources.py +++ b/tests/unit/src/sqlbuild/compiler/discovery/_helpers/test_yml_sources.py @@ -5,11 +5,9 @@ import pytest -from sqlbuild._native import SourceParseError +from sqlbuild._native import SourceEntry, SourceParseError, SourceWriteStrategy from sqlbuild.integrations.dlt.models import DltSourceConfig from sqlbuild.integrations.ingestr.models import IngestrSourceConfig -from sqlbuild.spec.contracts.models import SourceEntry -from sqlbuild.spec.contracts.types import SourceWriteStrategy from tests.unit.src.sqlbuild.compiler.discovery._helpers._test_types import ( ParseSourcesYamlDltTestCase, ParseSourcesYamlErrorTestCase, diff --git a/tests/unit/src/sqlbuild/compiler/discovery/helpers.py b/tests/unit/src/sqlbuild/compiler/discovery/helpers.py index 40699cae19..edf8456e4b 100644 --- a/tests/unit/src/sqlbuild/compiler/discovery/helpers.py +++ b/tests/unit/src/sqlbuild/compiler/discovery/helpers.py @@ -1,14 +1,14 @@ from pathlib import Path import sqlbuild._native as _native -from sqlbuild._native import DiscoveredSqlTestBlock -from sqlbuild.compiler.discovery.models import DiscoveredSqlModelFile, DiscoveredSqlScenarioFile -from sqlbuild.spec.contracts.models import ( +from sqlbuild._native import ( + DiscoveredSqlTestBlock, SchemaModelEntry, SchemaSeedEntry, SourceEntry, SourceLocation, ) +from sqlbuild.compiler.discovery.models import DiscoveredSqlModelFile, DiscoveredSqlScenarioFile REMOVED_SQL_MODEL_HEADER_KEYS: frozenset[str] = frozenset({"run_despite_unchanged"}) SQL_MODEL_HEADER_KEYS: frozenset[str] = frozenset( diff --git a/tests/unit/src/sqlbuild/compiler/fingerprints/main/helpers.py b/tests/unit/src/sqlbuild/compiler/fingerprints/main/helpers.py index e12d8992c8..18bcca05ab 100644 --- a/tests/unit/src/sqlbuild/compiler/fingerprints/main/helpers.py +++ b/tests/unit/src/sqlbuild/compiler/fingerprints/main/helpers.py @@ -5,8 +5,7 @@ from types import MappingProxyType from typing import Any -from sqlbuild._native import build_read_latest_sql -from sqlbuild.compiler.fingerprints.models import Fingerprint +from sqlbuild._native import Fingerprint, build_read_latest_sql class FakeFingerprintResult: diff --git a/tests/unit/src/sqlbuild/compiler/fingerprints/main/shared/_helpers/test_sql.py b/tests/unit/src/sqlbuild/compiler/fingerprints/main/shared/_helpers/test_sql.py index 04bd93ed9d..7576a962a5 100644 --- a/tests/unit/src/sqlbuild/compiler/fingerprints/main/shared/_helpers/test_sql.py +++ b/tests/unit/src/sqlbuild/compiler/fingerprints/main/shared/_helpers/test_sql.py @@ -5,12 +5,16 @@ import pytest -from sqlbuild._native import build_create_table_sql, build_read_latest_sql, write_fingerprint +from sqlbuild._native import ( + Fingerprint, + build_create_table_sql, + build_read_latest_sql, + write_fingerprint, +) from sqlbuild.adapter.contract.types import FrameworkType from sqlbuild.adapters.bigquery.classes.bigquery_adapter import BigQueryAdapter from sqlbuild.adapters.duckdb.classes.duckdb_adapter import DuckDbAdapter from sqlbuild.compiler.fingerprints.constants import FINGERPRINT_TABLE_NAME -from sqlbuild.compiler.fingerprints.models import Fingerprint from tests.unit.src.sqlbuild.compiler.fingerprints.main.shared._helpers._test_types import ( BuildCreateTableSqlTestCase, BuildInsertSqlTestCase, diff --git a/tests/unit/src/sqlbuild/compiler/fingerprints/main/test_read.py b/tests/unit/src/sqlbuild/compiler/fingerprints/main/test_read.py index 2461ffca0f..aa2738c008 100644 --- a/tests/unit/src/sqlbuild/compiler/fingerprints/main/test_read.py +++ b/tests/unit/src/sqlbuild/compiler/fingerprints/main/test_read.py @@ -5,9 +5,14 @@ import pytest -from sqlbuild._native import FingerprintInputError, read_latest_fingerprints, write_fingerprint +from sqlbuild._native import ( + Fingerprint, + FingerprintInputError, + FingerprintSet, + read_latest_fingerprints, + write_fingerprint, +) from sqlbuild.adapters.duckdb.classes.duckdb_adapter import DuckDbAdapter -from sqlbuild.compiler.fingerprints.models import Fingerprint, FingerprintSet from tests.unit.src.sqlbuild.compiler.fingerprints.main._test_types import ( ReadLatestFingerprintsErrorTestCase, ReadLatestFingerprintsFilterTestCase, diff --git a/tests/unit/src/sqlbuild/compiler/fingerprints/main/test_write.py b/tests/unit/src/sqlbuild/compiler/fingerprints/main/test_write.py index dc537b040f..be01fb9f6e 100644 --- a/tests/unit/src/sqlbuild/compiler/fingerprints/main/test_write.py +++ b/tests/unit/src/sqlbuild/compiler/fingerprints/main/test_write.py @@ -5,9 +5,8 @@ import pytest -from sqlbuild._native import write_fingerprint, write_fingerprints +from sqlbuild._native import Fingerprint, write_fingerprint, write_fingerprints from sqlbuild.adapters.duckdb.classes.duckdb_adapter import DuckDbAdapter -from sqlbuild.compiler.fingerprints.models import Fingerprint from tests.unit.src.sqlbuild.compiler.fingerprints.main._test_types import ( WriteFingerprintBatchTestCase, WriteFingerprintPartialFailureTestCase, diff --git a/tests/unit/src/sqlbuild/compiler/lineage/helpers.py b/tests/unit/src/sqlbuild/compiler/lineage/helpers.py index f0e2445b1d..be57e440aa 100644 --- a/tests/unit/src/sqlbuild/compiler/lineage/helpers.py +++ b/tests/unit/src/sqlbuild/compiler/lineage/helpers.py @@ -10,6 +10,9 @@ DiscoveredSchemaFile, DiscoveredSeedFile, ProjectColumnLineage, + SchemaColumn, + SchemaSeedEntry, + SettingsConfig, ) from sqlbuild.compiler.compile.models import ( CompiledModel, @@ -24,7 +27,6 @@ CompiledResourceType, CompiledSeedView, ) -from sqlbuild.spec.contracts.models import SchemaColumn, SchemaSeedEntry, SettingsConfig def make_compiled_project( diff --git a/tests/unit/src/sqlbuild/compiler/macro_bridge/_helpers/helpers.py b/tests/unit/src/sqlbuild/compiler/macro_bridge/_helpers/helpers.py index acb50cbbe0..fbbcb1722b 100644 --- a/tests/unit/src/sqlbuild/compiler/macro_bridge/_helpers/helpers.py +++ b/tests/unit/src/sqlbuild/compiler/macro_bridge/_helpers/helpers.py @@ -14,14 +14,13 @@ from pathlib import Path import sqlbuild._native as _native +from sqlbuild._native import SqlLogicalType, SqlValue, SqlValueKind from sqlbuild.compiler.compile.models import DeclarationResolutionContext, LoadedMacro, MacroContext from sqlbuild.compiler.discovery.models import ConstantDeclaration from sqlbuild.compiler.macro_bridge._helpers.store_environment import ( project_fingerprint, store_environment, ) -from sqlbuild.sql_values.models import SqlLogicalType, SqlValue -from sqlbuild.sql_values.types import SqlValueKind BASE_CONTEXT: MacroContext = MacroContext( adapter_name="duckdb", diff --git a/tests/unit/src/sqlbuild/compiler/manifest/helpers.py b/tests/unit/src/sqlbuild/compiler/manifest/helpers.py index 2d66988996..b408195099 100644 --- a/tests/unit/src/sqlbuild/compiler/manifest/helpers.py +++ b/tests/unit/src/sqlbuild/compiler/manifest/helpers.py @@ -13,6 +13,11 @@ DiscoveredSchemaFile, DiscoveredSeedFile, DiscoveredSqlHookFile, + SchemaColumn, + SchemaModelEntry, + SchemaSeedEntry, + SourceColumnEntry, + SourceEntry, build_manifest, ) from sqlbuild.compiler.auditing.types import ( @@ -48,13 +53,6 @@ PlanAction, PlanReason, ) -from sqlbuild.spec.contracts.models import ( - SchemaColumn, - SchemaModelEntry, - SchemaSeedEntry, - SourceColumnEntry, - SourceEntry, -) def model_key(name: str) -> CompiledObjectKey: diff --git a/tests/unit/src/sqlbuild/compiler/manifest/test_main.py b/tests/unit/src/sqlbuild/compiler/manifest/test_main.py index 4e76c3eb44..de7ac3479e 100644 --- a/tests/unit/src/sqlbuild/compiler/manifest/test_main.py +++ b/tests/unit/src/sqlbuild/compiler/manifest/test_main.py @@ -16,8 +16,10 @@ MacroMetadata, OwnershipRoot, ResourceKind, + SchemaColumn, ScopeIndex, ScopeKind, + SourceColumnEntry, build_manifest, project_with, ) @@ -33,7 +35,6 @@ ) from sqlbuild.compiler.discovery.models import SqlHookEntry from sqlbuild.compiler.planner.models import AuditPlanEntry, ChainStep, PlanOutput, SqlTestPlanEntry -from sqlbuild.spec.contracts.models import SchemaColumn, SourceColumnEntry from tests.unit.src.sqlbuild.compiler.manifest._test_types import ( FqnTestCase, ManifestAuditNodeTestCase, diff --git a/tests/unit/src/sqlbuild/compiler/pipeline/_helpers/helpers.py b/tests/unit/src/sqlbuild/compiler/pipeline/_helpers/helpers.py index b40947306c..f135f86c8d 100644 --- a/tests/unit/src/sqlbuild/compiler/pipeline/_helpers/helpers.py +++ b/tests/unit/src/sqlbuild/compiler/pipeline/_helpers/helpers.py @@ -5,7 +5,7 @@ from datetime import datetime from pathlib import Path -from sqlbuild._native import CompileModelConfig +from sqlbuild._native import CompileModelConfig, Fingerprint from sqlbuild.compiler.compile.models import ( CompiledModel, CompiledObjectKey, @@ -17,7 +17,6 @@ CompiledProjectView, CompiledResourceType, ) -from sqlbuild.compiler.fingerprints.models import Fingerprint def build_single_model_project( diff --git a/tests/unit/src/sqlbuild/compiler/pipeline/_helpers/target_validation/_test_types.py b/tests/unit/src/sqlbuild/compiler/pipeline/_helpers/target_validation/_test_types.py index 85af573fe4..0c27390378 100644 --- a/tests/unit/src/sqlbuild/compiler/pipeline/_helpers/target_validation/_test_types.py +++ b/tests/unit/src/sqlbuild/compiler/pipeline/_helpers/target_validation/_test_types.py @@ -1,8 +1,8 @@ from dataclasses import dataclass +from sqlbuild._native import LocalConfig, ProjectConfig from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter from sqlbuild.compiler.compile.models import CompiledRelationLocation -from sqlbuild.spec.contracts.models import LocalConfig, ProjectConfig @dataclass(frozen=True) diff --git a/tests/unit/src/sqlbuild/compiler/pipeline/_helpers/target_validation/test_main.py b/tests/unit/src/sqlbuild/compiler/pipeline/_helpers/target_validation/test_main.py index 4620e2c515..d416cf0605 100644 --- a/tests/unit/src/sqlbuild/compiler/pipeline/_helpers/target_validation/test_main.py +++ b/tests/unit/src/sqlbuild/compiler/pipeline/_helpers/target_validation/test_main.py @@ -5,6 +5,11 @@ import pytest from sqlbuild._native import ( + LocalConfig, + LocalTargetConfig, + ProjectConfig, + SourceEntry, + TargetConfig, validate_managed_loader_target_isolation, validate_managed_write_schemas, validate_named_target_schema_strategy, @@ -23,13 +28,6 @@ ) from sqlbuild.compiler.compile.types import CompiledProjectView from sqlbuild.compiler.discovery.models import DiscoveredProjectInputs, DiscoveredSourceFile -from sqlbuild.spec.contracts.models import ( - LocalConfig, - LocalTargetConfig, - ProjectConfig, - SourceEntry, - TargetConfig, -) from tests.unit.src.sqlbuild.compiler.pipeline._helpers.target_validation._test_types import ( ValidateManagedLoaderConnectionTestCase, ValidateManagedWriteSchemaTestCase, diff --git a/tests/unit/src/sqlbuild/compiler/pipeline/_helpers/test_clone.py b/tests/unit/src/sqlbuild/compiler/pipeline/_helpers/test_clone.py index 34b1eacb9d..903b611bc9 100644 --- a/tests/unit/src/sqlbuild/compiler/pipeline/_helpers/test_clone.py +++ b/tests/unit/src/sqlbuild/compiler/pipeline/_helpers/test_clone.py @@ -6,11 +6,13 @@ ClonePipelineConnection, ClonePipelineOptions, ClonePipelineResult, + LocalConfig, + ProjectConfig, + TargetConfig, run_clone_pipeline_with_options, ) from sqlbuild.adapters.postgres.classes.postgres_adapter import PostgresAdapter from sqlbuild.compiler.discovery.models import DiscoveredProjectInputs -from sqlbuild.spec.contracts.models import LocalConfig, ProjectConfig, TargetConfig from tests.unit.src.sqlbuild.compiler.pipeline._helpers._test_types import ( CloneConnectionInjectionTestCase, CloneTargetSchemaTestCase, diff --git a/tests/unit/src/sqlbuild/compiler/pipeline/_helpers/test_deferred_locations.py b/tests/unit/src/sqlbuild/compiler/pipeline/_helpers/test_deferred_locations.py index 2bead4698d..5a06eb78a2 100644 --- a/tests/unit/src/sqlbuild/compiler/pipeline/_helpers/test_deferred_locations.py +++ b/tests/unit/src/sqlbuild/compiler/pipeline/_helpers/test_deferred_locations.py @@ -4,14 +4,13 @@ import pytest -from sqlbuild._native import build_deferred_locations +from sqlbuild._native import TargetConfig, build_deferred_locations from sqlbuild.adapters.bigquery.classes.bigquery_adapter import BigQueryAdapter from sqlbuild.adapters.duckdb.classes.duckdb_adapter import DuckDbAdapter from sqlbuild.compiler.compile.models import ( CompiledRelationLocation, ) from sqlbuild.compiler.compile.types import CompiledProjectView -from sqlbuild.spec.contracts.models import TargetConfig from tests.unit.src.sqlbuild.compiler.pipeline._helpers._test_types import ( DeferredTargetTestCase, ) diff --git a/tests/unit/src/sqlbuild/compiler/pipeline/_helpers/test_python_plan_entries.py b/tests/unit/src/sqlbuild/compiler/pipeline/_helpers/test_python_plan_entries.py index 368bc2777b..ec6bb7e0b5 100644 --- a/tests/unit/src/sqlbuild/compiler/pipeline/_helpers/test_python_plan_entries.py +++ b/tests/unit/src/sqlbuild/compiler/pipeline/_helpers/test_python_plan_entries.py @@ -8,15 +8,17 @@ import pytest from sqlbuild._native import ( + Fingerprint, + FingerprintSet, PythonIdentityStatus, + PythonNodeGraph, + PythonNodeIdentity, PythonPlanEntry, PythonSqlRunSelection, build_python_plan_entries, build_python_sql_run_lifecycle, read_latest_fingerprints, ) -from sqlbuild.compiler.fingerprints.models import Fingerprint, FingerprintSet -from sqlbuild.compiler.python_nodes.models import PythonNodeGraph, PythonNodeIdentity from tests.unit.src.sqlbuild.compiler.fingerprints.main.helpers import ( FakeFingerprintExecute, render_qualified_name, diff --git a/tests/unit/src/sqlbuild/compiler/pipeline/main/helpers.py b/tests/unit/src/sqlbuild/compiler/pipeline/main/helpers.py index 73fef6bf95..a465467b65 100644 --- a/tests/unit/src/sqlbuild/compiler/pipeline/main/helpers.py +++ b/tests/unit/src/sqlbuild/compiler/pipeline/main/helpers.py @@ -3,7 +3,7 @@ from pathlib import Path from typing import Any, ClassVar -from sqlbuild._native import CompiledSource +from sqlbuild._native import CompiledSource, SourceEntry from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter from sqlbuild.compiler.compile.models import ( CompiledObjectKey, @@ -14,7 +14,6 @@ from sqlbuild.compiler.discovery.models import DiscoveredSourceFile from sqlbuild.compiler.planner.models import ModelPlanEntry, PlanOutput from sqlbuild.compiler.planner.types import MaterializationType, PlanAction, PlanReason -from sqlbuild.spec.contracts.models import SourceEntry class RelationTargetTestAdapter(BaseAdapter): diff --git a/tests/unit/src/sqlbuild/compiler/pipeline/main/test_relation_targets.py b/tests/unit/src/sqlbuild/compiler/pipeline/main/test_relation_targets.py index 5779d71d6c..0bcce0d858 100644 --- a/tests/unit/src/sqlbuild/compiler/pipeline/main/test_relation_targets.py +++ b/tests/unit/src/sqlbuild/compiler/pipeline/main/test_relation_targets.py @@ -4,14 +4,17 @@ import pytest -from sqlbuild._native import build_python_relation_targets +from sqlbuild._native import ( + DiscoveredPythonNode, + PythonNodeGraph, + SourceEntry, + build_python_relation_targets, +) from sqlbuild.compiler.compile.types import CompiledProjectView from sqlbuild.compiler.planner.models import PlanOutput -from sqlbuild.compiler.python_nodes.models import DiscoveredPythonNode, PythonNodeGraph from sqlbuild.compiler.python_nodes.types import PythonNodeKind from sqlbuild.python_nodes.models import SqlResourceRef from sqlbuild.refs import model, source -from sqlbuild.spec.contracts.models import SourceEntry from tests.unit.src.sqlbuild.compiler.pipeline.main._test_types import ( PythonRelationTargetScopeTestCase, PythonRelationTargetsTestCase, diff --git a/tests/unit/src/sqlbuild/compiler/planner/_helpers/_test_types.py b/tests/unit/src/sqlbuild/compiler/planner/_helpers/_test_types.py index 65b24a5dce..dd66df0f50 100644 --- a/tests/unit/src/sqlbuild/compiler/planner/_helpers/_test_types.py +++ b/tests/unit/src/sqlbuild/compiler/planner/_helpers/_test_types.py @@ -4,6 +4,7 @@ from sqlbuild._native import ( ChangeKind, + Fingerprint, MissingUpstream, ParsedSelector, PathSelector, @@ -11,6 +12,8 @@ ScenarioArtifactIdentity, ScenarioGraphPlan, SchemaAction, + SourceEntry, + SqlLexicalSyntax, WarningSeverity, ) from sqlbuild.compiler.auditing.types import ( @@ -23,7 +26,6 @@ InferredColumn, ) from sqlbuild.compiler.compile.types import AttachedAuditTargetKind, CompiledProjectView -from sqlbuild.compiler.fingerprints.models import Fingerprint from sqlbuild.compiler.planner.models import ( CursorBounds, ScenarioRelationMap, @@ -38,8 +40,6 @@ PlanReason, ) from sqlbuild.compiler.references.types import SqlReferenceKind -from sqlbuild.compiler.sql_analysis.models import SqlLexicalSyntax -from sqlbuild.spec.contracts.models import SourceEntry from tests.unit.src.sqlbuild.compiler.planner._helpers.sql_test_assembly._test_types import ( PlanTestChainTestCase as PlanTestChainTestCase, ) diff --git a/tests/unit/src/sqlbuild/compiler/planner/_helpers/changes/_test_helpers.py b/tests/unit/src/sqlbuild/compiler/planner/_helpers/changes/_test_helpers.py index 4c8bf2aeff..f4bf072891 100644 --- a/tests/unit/src/sqlbuild/compiler/planner/_helpers/changes/_test_helpers.py +++ b/tests/unit/src/sqlbuild/compiler/planner/_helpers/changes/_test_helpers.py @@ -11,8 +11,11 @@ ChangeKind, CompiledFunction, CompileModelConfig, + Fingerprint, FunctionArgument, PlannerScope, + SchemaColumn, + SchemaModelEntry, WarehouseFingerprints, WarehouseSnapshot, build_version_identity_metadata_json, @@ -32,8 +35,6 @@ CompiledResourceType, FunctionLanguage, ) -from sqlbuild.compiler.fingerprints.models import Fingerprint -from sqlbuild.spec.contracts.models import SchemaColumn, SchemaModelEntry from tests.unit.src.sqlbuild.compiler.planner._helpers.changes._test_types import ( DetectModelChangesTestCase, DetectModelMetadataTestCase, diff --git a/tests/unit/src/sqlbuild/compiler/planner/_helpers/changes/_test_types.py b/tests/unit/src/sqlbuild/compiler/planner/_helpers/changes/_test_types.py index f84a55b035..e5ebba7ad7 100644 --- a/tests/unit/src/sqlbuild/compiler/planner/_helpers/changes/_test_types.py +++ b/tests/unit/src/sqlbuild/compiler/planner/_helpers/changes/_test_types.py @@ -4,6 +4,7 @@ BackfillResult, ChangeKind, ReferenceRename, + SchemaDynamicColumnFamily, ) from sqlbuild.adapter.contract.models import ColumnInfo from sqlbuild.compiler.compile.models import InferredColumn @@ -15,7 +16,6 @@ PlanAction, PlanReason, ) -from sqlbuild.spec.contracts.models import SchemaDynamicColumnFamily @dataclass(frozen=True) diff --git a/tests/unit/src/sqlbuild/compiler/planner/_helpers/changes/test_detect.py b/tests/unit/src/sqlbuild/compiler/planner/_helpers/changes/test_detect.py index 42e61cb32f..7fd6a56d13 100644 --- a/tests/unit/src/sqlbuild/compiler/planner/_helpers/changes/test_detect.py +++ b/tests/unit/src/sqlbuild/compiler/planner/_helpers/changes/test_detect.py @@ -9,6 +9,7 @@ BackfillResult, ChangeDetectionResult, ChangeKind, + Fingerprint, PlannerChangeResults, PlannerScope, PlanWarning, @@ -26,7 +27,6 @@ from sqlbuild.compiler.compile.models import InferredColumn from sqlbuild.compiler.compile.types import CompiledModelView from sqlbuild.compiler.discovery.models import SqlHookEntry -from sqlbuild.compiler.fingerprints.models import Fingerprint from sqlbuild.compiler.planner.types import ( BackfillAction, PlanAction, diff --git a/tests/unit/src/sqlbuild/compiler/planner/_helpers/changes/test_reference_renames.py b/tests/unit/src/sqlbuild/compiler/planner/_helpers/changes/test_reference_renames.py index 955b73c549..89d0a6e12a 100644 --- a/tests/unit/src/sqlbuild/compiler/planner/_helpers/changes/test_reference_renames.py +++ b/tests/unit/src/sqlbuild/compiler/planner/_helpers/changes/test_reference_renames.py @@ -7,11 +7,11 @@ import pytest from sqlbuild._native import ( + Fingerprint, ReferenceRename, compute_query_hash, origin_reference_names, ) -from sqlbuild.compiler.fingerprints.models import Fingerprint from tests.unit.src.sqlbuild.compiler.planner._helpers.changes._test_types import ( OriginReferenceNamesTestCase, ) diff --git a/tests/unit/src/sqlbuild/compiler/planner/_helpers/changes/test_schema.py b/tests/unit/src/sqlbuild/compiler/planner/_helpers/changes/test_schema.py index 4b9ad9fa4c..0cbf307b5f 100644 --- a/tests/unit/src/sqlbuild/compiler/planner/_helpers/changes/test_schema.py +++ b/tests/unit/src/sqlbuild/compiler/planner/_helpers/changes/test_schema.py @@ -5,12 +5,12 @@ from sqlbuild._native import ( SchemaChangeKind, SchemaColumnSource, + SchemaDynamicColumnFamily, detect_schema_changes, ) from sqlbuild.adapter.contract.models import ColumnInfo from sqlbuild.compiler.compile.models import InferredColumn from sqlbuild.compiler.planner.models import SchemaFinding -from sqlbuild.spec.contracts.models import SchemaDynamicColumnFamily from tests.unit.src.sqlbuild.compiler.planner._helpers.changes._test_types import ( DetectSchemaChangesTestCase, ) diff --git a/tests/unit/src/sqlbuild/compiler/planner/_helpers/helpers.py b/tests/unit/src/sqlbuild/compiler/planner/_helpers/helpers.py index 9a281f15c5..5d9a86562b 100644 --- a/tests/unit/src/sqlbuild/compiler/planner/_helpers/helpers.py +++ b/tests/unit/src/sqlbuild/compiler/planner/_helpers/helpers.py @@ -23,14 +23,24 @@ CompileModelConfig, CompileSqlScenarioCte, CompileSqlTestCte, + DefaultsConfig, DiscoveredAuditBlock, DiscoveredAuditFile, DiscoveredSchemaFile, DiscoveredSeedFile, DiscoveredSqlTestBlock, + Fingerprint, FunctionArgument, PlannerChangeResults, ScenarioArtifactIdentity, + SchemaColumn, + SchemaModelEntry, + SchemaSeedEntry, + SeedCsvSettings, + SourceColumnEntry, + SourceEntry, + SourceWriteStrategy, + SqlLexicalSyntax, WarehouseSnapshot, build_downstream_deps, build_scenario_relation_map, @@ -70,7 +80,6 @@ DiscoveredSqlScenarioFile, DiscoveredSqlTestFile, ) -from sqlbuild.compiler.fingerprints.models import Fingerprint from sqlbuild.compiler.pipeline.models import CompilePipelineResult from sqlbuild.compiler.planner.models import ( PlanOutput, @@ -81,19 +90,6 @@ PlanReason, ) from sqlbuild.compiler.references.types import SqlReferenceKind -from sqlbuild.compiler.sql_analysis.models import SqlLexicalSyntax -from sqlbuild.spec.contracts.models import ( - DefaultsConfig, - SchemaColumn, - SchemaModelEntry, - SchemaSeedEntry, - SeedCsvSettings, - SourceColumnEntry, - SourceEntry, -) -from sqlbuild.spec.contracts.types import ( - SourceWriteStrategy, -) from tests.unit.src.sqlbuild.compiler.planner._helpers._test_types import ( BuildModelWarningsTestCase, IncrementalStrategyErrorTestCase, diff --git a/tests/unit/src/sqlbuild/compiler/planner/_helpers/migrations/test_column_migration_candidates.py b/tests/unit/src/sqlbuild/compiler/planner/_helpers/migrations/test_column_migration_candidates.py index 0dea21e99b..b2bdc06b7c 100644 --- a/tests/unit/src/sqlbuild/compiler/planner/_helpers/migrations/test_column_migration_candidates.py +++ b/tests/unit/src/sqlbuild/compiler/planner/_helpers/migrations/test_column_migration_candidates.py @@ -11,6 +11,7 @@ from sqlbuild._native import ( CompileModelConfig, DeferralInputs, + Fingerprint, PlannerOverrides, PlannerRuntime, PlannerScope, @@ -26,7 +27,6 @@ CompiledRelationLocation, ) from sqlbuild.compiler.compile.types import CompiledModelView, CompiledResourceType -from sqlbuild.compiler.fingerprints.models import Fingerprint from sqlbuild.compiler.planner.exceptions import PlannerInputError from tests.unit.src.sqlbuild.compiler.planner._helpers.migrations._test_types import ( UnfingerprintableCandidateTestCase, diff --git a/tests/unit/src/sqlbuild/compiler/planner/_helpers/planning/_test_types.py b/tests/unit/src/sqlbuild/compiler/planner/_helpers/planning/_test_types.py index 5b420844ba..3193713d2a 100644 --- a/tests/unit/src/sqlbuild/compiler/planner/_helpers/planning/_test_types.py +++ b/tests/unit/src/sqlbuild/compiler/planner/_helpers/planning/_test_types.py @@ -4,10 +4,11 @@ from sqlbuild._native import ( ChangeDetectionResult, + TableType, + TableTypeDowngradePolicy, ) from sqlbuild.adapter.contract.models import RetentionState from sqlbuild.compiler.planner.types import RetentionDirection, RetentionPlanPhase -from sqlbuild.spec.contracts.types import TableType, TableTypeDowngradePolicy @dataclass(frozen=True) diff --git a/tests/unit/src/sqlbuild/compiler/planner/_helpers/planning/helpers.py b/tests/unit/src/sqlbuild/compiler/planner/_helpers/planning/helpers.py index 16d6999afb..79f6e51659 100644 --- a/tests/unit/src/sqlbuild/compiler/planner/_helpers/planning/helpers.py +++ b/tests/unit/src/sqlbuild/compiler/planner/_helpers/planning/helpers.py @@ -6,10 +6,17 @@ from sqlbuild._native import ( CompileModelConfig, + LocalConfig, PlannerRelationsContext, PlannerRuntime, PlannerScope, PlannerWarehouseState, + ProjectConfig, + ResolvedTableType, + ResolvedTimeTravelRetention, + TableTypeDowngradePolicy, + TargetConfig, + TimeTravelRetentionSource, WarehouseSnapshot, ) from sqlbuild.adapter.contract.classes.retention_adapter import RetentionAdapterMixin @@ -25,17 +32,6 @@ CompiledProjectView, CompiledResourceType, ) -from sqlbuild.spec.contracts.models import ( - LocalConfig, - ProjectConfig, - ResolvedTableType, - ResolvedTimeTravelRetention, - TargetConfig, -) -from sqlbuild.spec.contracts.types import ( - TableTypeDowngradePolicy, - TimeTravelRetentionSource, -) def build_retention_planner_inputs( diff --git a/tests/unit/src/sqlbuild/compiler/planner/_helpers/planning/test_retention.py b/tests/unit/src/sqlbuild/compiler/planner/_helpers/planning/test_retention.py index 7871c50b6d..d15cc6fbc7 100644 --- a/tests/unit/src/sqlbuild/compiler/planner/_helpers/planning/test_retention.py +++ b/tests/unit/src/sqlbuild/compiler/planner/_helpers/planning/test_retention.py @@ -7,9 +7,17 @@ import pytest from sqlbuild._native import ( + Fingerprint, + LocalConfig, PlannerRuntime, PlannerScope, PlannerWarehouseState, + ProjectConfig, + ResolvedTableType, + TableType, + TableTypeDowngradePolicy, + TableTypeSource, + TargetConfig, WarehouseFingerprints, plan_retention, plan_table_types, @@ -25,7 +33,6 @@ RetentionScope, ) from sqlbuild.adapters.snowflake.classes.snowflake_adapter import SnowflakeAdapter -from sqlbuild.compiler.fingerprints.models import Fingerprint from sqlbuild.compiler.planner.exceptions import PlannerInputError from sqlbuild.compiler.planner.models import RetentionPlanEntry, TableTypePlanEntry from sqlbuild.compiler.planner.types import ( @@ -33,13 +40,6 @@ RetentionDirection, RetentionPlanPhase, ) -from sqlbuild.spec.contracts.models import ( - LocalConfig, - ProjectConfig, - ResolvedTableType, - TargetConfig, -) -from sqlbuild.spec.contracts.types import TableType, TableTypeDowngradePolicy, TableTypeSource from tests.unit.src.sqlbuild.compiler.planner._helpers.planning._test_types import ( ForeignSnapshotRelationTestCase, RetentionPlanningErrorTestCase, diff --git a/tests/unit/src/sqlbuild/compiler/planner/_helpers/resolve/_test_types.py b/tests/unit/src/sqlbuild/compiler/planner/_helpers/resolve/_test_types.py index e0cd13ede3..419550374e 100644 --- a/tests/unit/src/sqlbuild/compiler/planner/_helpers/resolve/_test_types.py +++ b/tests/unit/src/sqlbuild/compiler/planner/_helpers/resolve/_test_types.py @@ -5,9 +5,9 @@ from dataclasses import dataclass, field from datetime import date, datetime +from sqlbuild._native import SourceEntry from sqlbuild.adapter.contract.models import ColumnInfo from sqlbuild.compiler.planner.models import CursorBounds, CursorOverrides, ModelCursorSnapshot -from sqlbuild.spec.contracts.models import SourceEntry @dataclass(frozen=True) diff --git a/tests/unit/src/sqlbuild/compiler/planner/_helpers/resolve/test_cursor.py b/tests/unit/src/sqlbuild/compiler/planner/_helpers/resolve/test_cursor.py index 1ad561525f..790c2aba2f 100644 --- a/tests/unit/src/sqlbuild/compiler/planner/_helpers/resolve/test_cursor.py +++ b/tests/unit/src/sqlbuild/compiler/planner/_helpers/resolve/test_cursor.py @@ -8,7 +8,10 @@ import pytest from sqlbuild._native import ( + AlignedInterval, CursorWatermarkMode, + DateValue, + TimestampValue, apply_typed_cursor_replay_policy, compute_cursor_bounds, ) @@ -24,7 +27,6 @@ CursorGrain, CursorType, ) -from sqlbuild.cursor_algebra.models import AlignedInterval, DateValue, TimestampValue from sqlbuild.cursor_algebra.types import CursorScalar from tests.unit.src.sqlbuild.compiler.planner._helpers.resolve._test_types import ( CursorBoundsTestCase, diff --git a/tests/unit/src/sqlbuild/compiler/planner/_helpers/resolve/test_hooks.py b/tests/unit/src/sqlbuild/compiler/planner/_helpers/resolve/test_hooks.py index 142a004783..4e4e38a8d7 100644 --- a/tests/unit/src/sqlbuild/compiler/planner/_helpers/resolve/test_hooks.py +++ b/tests/unit/src/sqlbuild/compiler/planner/_helpers/resolve/test_hooks.py @@ -8,12 +8,12 @@ from sqlbuild._native import ( ModelPlanContext, + SourceEntry, resolve_model_hook_entries, ) from sqlbuild.adapters.duckdb.classes.duckdb_adapter import DuckDbAdapter from sqlbuild.compiler.compile.types import CompiledModelView from sqlbuild.compiler.discovery.models import PythonHookEntry, SqlHookEntry -from sqlbuild.spec.contracts.models import SourceEntry from tests.unit.src.sqlbuild.compiler.planner._helpers.resolve._test_types import ( HookSqlResolutionTestCase, ) diff --git a/tests/unit/src/sqlbuild/compiler/planner/_helpers/resolve/test_sources.py b/tests/unit/src/sqlbuild/compiler/planner/_helpers/resolve/test_sources.py index 62a2c40c5c..af90946cb8 100644 --- a/tests/unit/src/sqlbuild/compiler/planner/_helpers/resolve/test_sources.py +++ b/tests/unit/src/sqlbuild/compiler/planner/_helpers/resolve/test_sources.py @@ -4,7 +4,7 @@ import pytest -from sqlbuild._native import resolve_source_references +from sqlbuild._native import SourceColumnEntry, SourceEntry, resolve_source_references from sqlbuild.adapter.contract.models import ColumnInfo from sqlbuild.adapter.contract.types import CursorKind from sqlbuild.adapters.bigquery.classes.bigquery_adapter import BigQueryAdapter @@ -13,7 +13,6 @@ from sqlbuild.adapters.snowflake.classes.snowflake_adapter import SnowflakeAdapter from sqlbuild.adapters.sqlserver.classes.sqlserver_adapter import SqlServerAdapter from sqlbuild.compiler.planner.models import CursorBounds -from sqlbuild.spec.contracts.models import SourceColumnEntry, SourceEntry from tests.unit.src.sqlbuild.compiler.planner._helpers.resolve._test_types import ( AdapterSourceResolutionTestCase, ReservedSourceColumnTestCase, diff --git a/tests/unit/src/sqlbuild/compiler/planner/_helpers/sql_test_assembly/helpers.py b/tests/unit/src/sqlbuild/compiler/planner/_helpers/sql_test_assembly/helpers.py index 7d13ab9bc9..b4c972e37f 100644 --- a/tests/unit/src/sqlbuild/compiler/planner/_helpers/sql_test_assembly/helpers.py +++ b/tests/unit/src/sqlbuild/compiler/planner/_helpers/sql_test_assembly/helpers.py @@ -15,6 +15,7 @@ DiscoveredSqlTestBlock, NativeSqlTestPlan, PlanWarning, + SqlLexicalSyntax, SqlTestPlanResult, expand_sql_macros, extract_sql_references, @@ -48,7 +49,6 @@ ) from sqlbuild.compiler.references.types import SqlReferenceKind from sqlbuild.compiler.sql_analysis.main.import_polyglot_sql import import_polyglot_sql -from sqlbuild.compiler.sql_analysis.models import SqlLexicalSyntax from tests.unit.src.sqlbuild.compiler.planner._helpers.sql_test_assembly._test_types import ( PlanTestChainTestCase, ) diff --git a/tests/unit/src/sqlbuild/compiler/planner/_helpers/test_audit_entry.py b/tests/unit/src/sqlbuild/compiler/planner/_helpers/test_audit_entry.py index a371b8a5d6..d45e637c1f 100644 --- a/tests/unit/src/sqlbuild/compiler/planner/_helpers/test_audit_entry.py +++ b/tests/unit/src/sqlbuild/compiler/planner/_helpers/test_audit_entry.py @@ -2,7 +2,7 @@ import pytest -from sqlbuild._native import plan_audit +from sqlbuild._native import SourceEntry, plan_audit from sqlbuild.compiler.auditing.types import AuditAttachmentKind, AuditRunScope from sqlbuild.compiler.compile.models import ( CompiledObjectKey, @@ -12,7 +12,6 @@ from sqlbuild.compiler.compile.types import AttachedAuditTargetKind, CompiledAuditView from sqlbuild.compiler.planner.models import AuditPlanEntry from sqlbuild.compiler.references.types import SqlReferenceKind -from sqlbuild.spec.contracts.models import SourceEntry from tests.unit.src.sqlbuild.compiler.planner._helpers._test_types import ( PlanAttachedAuditTestCase, PlanAuditTestCase, diff --git a/tests/unit/src/sqlbuild/compiler/planner/_helpers/test_fixture_planning_work.py b/tests/unit/src/sqlbuild/compiler/planner/_helpers/test_fixture_planning_work.py index a025b8a56d..d0e79d1349 100644 --- a/tests/unit/src/sqlbuild/compiler/planner/_helpers/test_fixture_planning_work.py +++ b/tests/unit/src/sqlbuild/compiler/planner/_helpers/test_fixture_planning_work.py @@ -13,6 +13,7 @@ PlanWarning, RelationFixtureCompletion, RelationFixturePlanningContext, + SettingsConfig, SqlTestSelection, build_selected_test_entries, compiled_model_record, @@ -30,7 +31,6 @@ ) from sqlbuild.compiler.planner.models import SqlTestPlanEntry from sqlbuild.compiler.references.types import SqlReferenceKind -from sqlbuild.spec.contracts.models import SettingsConfig from tests.unit.src.sqlbuild.compiler.planner._helpers._test_types import ( CompleteFixturePlanningTestCase, FixturePlanningWorkTestCase, diff --git a/tests/unit/src/sqlbuild/compiler/planner/_helpers/test_microbatch_limit.py b/tests/unit/src/sqlbuild/compiler/planner/_helpers/test_microbatch_limit.py index 0fd1466f3b..8692e584a6 100644 --- a/tests/unit/src/sqlbuild/compiler/planner/_helpers/test_microbatch_limit.py +++ b/tests/unit/src/sqlbuild/compiler/planner/_helpers/test_microbatch_limit.py @@ -5,6 +5,7 @@ import pytest from sqlbuild._native import ( + MicrobatchLimitAction, PlanEntryBuildInputs, PlanWarning, apply_configured_microbatch_limits, @@ -28,7 +29,6 @@ PlanAction, PlanReason, ) -from sqlbuild.spec.contracts.types import MicrobatchLimitAction from tests.unit.src.sqlbuild.compiler.planner._helpers._test_types import ( ConfiguredMicrobatchPolicyTestCase, EffectiveMicrobatchLimitPlanningTestCase, diff --git a/tests/unit/src/sqlbuild/compiler/planner/_helpers/test_scenario_cli.py b/tests/unit/src/sqlbuild/compiler/planner/_helpers/test_scenario_cli.py index 8b9ef461f2..09c98105d2 100644 --- a/tests/unit/src/sqlbuild/compiler/planner/_helpers/test_scenario_cli.py +++ b/tests/unit/src/sqlbuild/compiler/planner/_helpers/test_scenario_cli.py @@ -7,6 +7,7 @@ from sqlbuild._native import ( ScenarioArtifactName, + SqlLexicalSyntax, build_cli_scenario_plan, ) from sqlbuild.compiler.compile.models import CompiledSqlScenario @@ -14,7 +15,6 @@ from sqlbuild.compiler.planner.models import ( ScenarioExecutionPlan, ) -from sqlbuild.compiler.sql_analysis.models import SqlLexicalSyntax from tests.unit.src.sqlbuild.compiler.planner._helpers._test_types import ( ScenarioCliPlanIdentifierLimitTestCase, ) diff --git a/tests/unit/src/sqlbuild/compiler/planner/_helpers/test_scenario_relations.py b/tests/unit/src/sqlbuild/compiler/planner/_helpers/test_scenario_relations.py index e15464501e..2d645c67f2 100644 --- a/tests/unit/src/sqlbuild/compiler/planner/_helpers/test_scenario_relations.py +++ b/tests/unit/src/sqlbuild/compiler/planner/_helpers/test_scenario_relations.py @@ -9,6 +9,7 @@ from sqlbuild._native import ( CompileSqlScenarioCte, ScenarioGraphPlan, + SqlLexicalSyntax, build_scenario_execution_plan, build_scenario_fixture_plans, build_scenario_relation_plan, @@ -30,7 +31,6 @@ ScenarioFixturePlan, ScenarioRelationPlan, ) -from sqlbuild.compiler.sql_analysis.models import SqlLexicalSyntax from tests.unit.src.sqlbuild.compiler.planner._helpers._test_types import ( ScenarioCheckSqlResolutionErrorTestCase, ScenarioCheckSqlResolutionTestCase, diff --git a/tests/unit/src/sqlbuild/compiler/planner/_helpers/test_seed_identity.py b/tests/unit/src/sqlbuild/compiler/planner/_helpers/test_seed_identity.py index 3f34e9ed29..a8163a1205 100644 --- a/tests/unit/src/sqlbuild/compiler/planner/_helpers/test_seed_identity.py +++ b/tests/unit/src/sqlbuild/compiler/planner/_helpers/test_seed_identity.py @@ -6,8 +6,7 @@ import pytest -from sqlbuild._native import build_seed_identity -from sqlbuild.spec.contracts.models import SeedCsvSettings +from sqlbuild._native import SeedCsvSettings, build_seed_identity from tests.unit.src.sqlbuild.compiler.planner._helpers._test_types import ( SeedIdentityCsvConfigTestCase, SeedIdentityTestCase, diff --git a/tests/unit/src/sqlbuild/compiler/planner/_helpers/test_selection_staleness.py b/tests/unit/src/sqlbuild/compiler/planner/_helpers/test_selection_staleness.py index ed26cc5d80..627793ba5e 100644 --- a/tests/unit/src/sqlbuild/compiler/planner/_helpers/test_selection_staleness.py +++ b/tests/unit/src/sqlbuild/compiler/planner/_helpers/test_selection_staleness.py @@ -10,6 +10,7 @@ ChangeKind, DirectModelVersionIdentities, DirectSourceFreshnessPlanningResult, + Fingerprint, PlannerChangeResults, PlannerPolicies, PlannerScope, @@ -30,7 +31,6 @@ CompiledResourceType, ) from sqlbuild.compiler.fingerprints.constants import NODE_TYPE_SEED -from sqlbuild.compiler.fingerprints.models import Fingerprint from tests.unit.src.sqlbuild.compiler.planner._helpers._test_types import ( CompleteModelChangesWorkTestCase, PlannerStaleWarningScopeTestCase, diff --git a/tests/unit/src/sqlbuild/compiler/planner/_helpers/test_source_columns.py b/tests/unit/src/sqlbuild/compiler/planner/_helpers/test_source_columns.py index d3f480cabb..15f88861cb 100644 --- a/tests/unit/src/sqlbuild/compiler/planner/_helpers/test_source_columns.py +++ b/tests/unit/src/sqlbuild/compiler/planner/_helpers/test_source_columns.py @@ -9,6 +9,7 @@ from sqlbuild._native import ( PlannerRelationsContext, PlannerScope, + SourceEntry, build_cursor_input_relations, build_planner_relations_context, gather_source_columns, @@ -31,7 +32,6 @@ from sqlbuild.compiler.planner.exceptions import PlannerInputError from sqlbuild.compiler.planner.models import CursorInputRelation from sqlbuild.compiler.references.types import SqlReferenceKind -from sqlbuild.spec.contracts.models import SourceEntry from tests.unit.src.sqlbuild.compiler.planner._helpers._test_types import ( CursorRuntimeOwnershipTestCase, KnownSourceColumnsReuseTestCase, diff --git a/tests/unit/src/sqlbuild/compiler/planner/_helpers/test_source_freshness.py b/tests/unit/src/sqlbuild/compiler/planner/_helpers/test_source_freshness.py index 20cd18c124..cf12bac7fb 100644 --- a/tests/unit/src/sqlbuild/compiler/planner/_helpers/test_source_freshness.py +++ b/tests/unit/src/sqlbuild/compiler/planner/_helpers/test_source_freshness.py @@ -8,12 +8,14 @@ DirectSourceFreshnessPlanningResult, PlannerRelationsContext, PlannerScope, + SourceEntry, + SourceFreshnessConfig, + SourceFreshnessStrategy, + SourceFreshnessValueKind, build_planner_source_freshness_result, ) from sqlbuild.adapters.duckdb.classes.duckdb_adapter import DuckDbAdapter from sqlbuild.compiler.compile.models import CompiledProject -from sqlbuild.spec.contracts.models import SourceEntry, SourceFreshnessConfig -from sqlbuild.spec.contracts.types import SourceFreshnessStrategy, SourceFreshnessValueKind from tests.unit.src.sqlbuild.compiler.planner._helpers._test_types import ( PlannerSourceFreshnessReadMapTestCase, ) diff --git a/tests/unit/src/sqlbuild/compiler/planner/_helpers/test_source_load_nodes.py b/tests/unit/src/sqlbuild/compiler/planner/_helpers/test_source_load_nodes.py index 54bbd30a4f..b367074ae4 100644 --- a/tests/unit/src/sqlbuild/compiler/planner/_helpers/test_source_load_nodes.py +++ b/tests/unit/src/sqlbuild/compiler/planner/_helpers/test_source_load_nodes.py @@ -9,6 +9,8 @@ from sqlbuild._native import ( CompileModelConfig, + SourceEntry, + SourceWriteStrategy, build_intermediate_source_map, build_source_load_entries, build_source_load_map, @@ -31,8 +33,6 @@ from sqlbuild.compiler.discovery.models import DiscoveredLoaderFunction from sqlbuild.compiler.planner.models import SourceLoadPlanEntry from sqlbuild.runtime.contracts.types import ExecutionResourceKind -from sqlbuild.spec.contracts.models import SourceEntry -from sqlbuild.spec.contracts.types import SourceWriteStrategy from tests.unit.src.sqlbuild.compiler.planner._helpers._test_types import ( LoaderDagExpansionTestCase, LoaderDependencyNamesTestCase, diff --git a/tests/unit/src/sqlbuild/compiler/planner/_helpers/warehouse/_test_types.py b/tests/unit/src/sqlbuild/compiler/planner/_helpers/warehouse/_test_types.py index a9a2f1a7f9..e21ef80d4f 100644 --- a/tests/unit/src/sqlbuild/compiler/planner/_helpers/warehouse/_test_types.py +++ b/tests/unit/src/sqlbuild/compiler/planner/_helpers/warehouse/_test_types.py @@ -2,7 +2,7 @@ from dataclasses import dataclass -from sqlbuild.spec.contracts.models import SourceEntry +from sqlbuild._native import SourceEntry @dataclass(frozen=True) diff --git a/tests/unit/src/sqlbuild/compiler/planner/_helpers/warehouse/helpers.py b/tests/unit/src/sqlbuild/compiler/planner/_helpers/warehouse/helpers.py index 7cd8253437..5bbfaec0a1 100644 --- a/tests/unit/src/sqlbuild/compiler/planner/_helpers/warehouse/helpers.py +++ b/tests/unit/src/sqlbuild/compiler/planner/_helpers/warehouse/helpers.py @@ -10,6 +10,7 @@ CompiledSource, PlannerRelationsContext, PlannerScope, + SourceEntry, check_selected_source_tables_exist, ) from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter @@ -17,7 +18,6 @@ from sqlbuild.compiler.compile.models import CompiledObjectKey, CompiledProject from sqlbuild.compiler.compile.types import CompiledResourceType, CompiledSourceView from sqlbuild.compiler.discovery.models import DiscoveredSourceFile -from sqlbuild.spec.contracts.models import SourceEntry def column_names(columns: tuple[ColumnInfo, ...]) -> tuple[str, ...]: diff --git a/tests/unit/src/sqlbuild/compiler/planner/_helpers/warehouse/test_semantic_sources.py b/tests/unit/src/sqlbuild/compiler/planner/_helpers/warehouse/test_semantic_sources.py index 973e792d11..f82650e67c 100644 --- a/tests/unit/src/sqlbuild/compiler/planner/_helpers/warehouse/test_semantic_sources.py +++ b/tests/unit/src/sqlbuild/compiler/planner/_helpers/warehouse/test_semantic_sources.py @@ -11,6 +11,7 @@ import pytest from sqlbuild._native import ( + SourceEntry, build_compiled_project, discover_project_inputs, get_semantic_source_columns, @@ -20,7 +21,6 @@ from sqlbuild.adapter.relations.main.open_inspection_catalog import open_inspection_catalog from sqlbuild.adapters.duckdb.classes.duckdb_adapter import DuckDbAdapter from sqlbuild.compiler.compile.types import CompiledProjectView -from sqlbuild.spec.contracts.models import SourceEntry from tests.unit.src.sqlbuild.adapters.snowflake.inspection.helpers import ( FakeRelation, build_offline_snowflake, diff --git a/tests/unit/src/sqlbuild/compiler/planner/_helpers/warehouse/test_source_tables.py b/tests/unit/src/sqlbuild/compiler/planner/_helpers/warehouse/test_source_tables.py index 574fbf0e21..31af47d331 100644 --- a/tests/unit/src/sqlbuild/compiler/planner/_helpers/warehouse/test_source_tables.py +++ b/tests/unit/src/sqlbuild/compiler/planner/_helpers/warehouse/test_source_tables.py @@ -2,9 +2,9 @@ import pytest +from sqlbuild._native import SourceEntry from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter from sqlbuild.compiler.planner.exceptions import PlannerInputError -from sqlbuild.spec.contracts.models import SourceEntry from tests.unit.src.sqlbuild.adapter.contract.main.helpers import ( ADAPTERS_BY_NAME, BinderException, diff --git a/tests/unit/src/sqlbuild/compiler/planner/_test_types.py b/tests/unit/src/sqlbuild/compiler/planner/_test_types.py index 5563adc22d..1e9c357578 100644 --- a/tests/unit/src/sqlbuild/compiler/planner/_test_types.py +++ b/tests/unit/src/sqlbuild/compiler/planner/_test_types.py @@ -2,7 +2,7 @@ from dataclasses import dataclass -from sqlbuild.spec.contracts.types import MissingMigrationOriginPolicy +from sqlbuild._native import MissingMigrationOriginPolicy @dataclass(frozen=True) diff --git a/tests/unit/src/sqlbuild/compiler/planner/main/_test_types.py b/tests/unit/src/sqlbuild/compiler/planner/main/_test_types.py index 9aebae157b..7741b24229 100644 --- a/tests/unit/src/sqlbuild/compiler/planner/main/_test_types.py +++ b/tests/unit/src/sqlbuild/compiler/planner/main/_test_types.py @@ -2,13 +2,13 @@ from datetime import datetime from sqlbuild._native import ( + FutureCursorsConfig, SelectionStalenessGraph, SelectionStalenessWarning, ) from sqlbuild.compiler.planner.models import ( CursorBounds, ) -from sqlbuild.spec.contracts.models import FutureCursorsConfig @dataclass(frozen=True) diff --git a/tests/unit/src/sqlbuild/compiler/planner/main/helpers.py b/tests/unit/src/sqlbuild/compiler/planner/main/helpers.py index 7bd5efe351..cb819512bb 100644 --- a/tests/unit/src/sqlbuild/compiler/planner/main/helpers.py +++ b/tests/unit/src/sqlbuild/compiler/planner/main/helpers.py @@ -8,9 +8,12 @@ from sqlbuild._native import ( CompileModelConfig, DeferralInputs, + Fingerprint, PlannerOverrides, PlannerPolicies, PlannerSelection, + SchemaColumn, + SchemaModelEntry, SelectionStalenessGraph, SelectionStalenessNodeKey, build_execution_plan, @@ -30,9 +33,7 @@ CompiledProjectView, CompiledResourceType, ) -from sqlbuild.compiler.fingerprints.models import Fingerprint from sqlbuild.compiler.planner.models import PlanOutput -from sqlbuild.spec.contracts.models import SchemaColumn, SchemaModelEntry def model_definition_hash(project: CompiledProjectView, name: str) -> str: diff --git a/tests/unit/src/sqlbuild/compiler/planner/main/identity/helpers.py b/tests/unit/src/sqlbuild/compiler/planner/main/identity/helpers.py index 0e14fac65d..179ef8ee7a 100644 --- a/tests/unit/src/sqlbuild/compiler/planner/main/identity/helpers.py +++ b/tests/unit/src/sqlbuild/compiler/planner/main/identity/helpers.py @@ -10,6 +10,8 @@ FunctionArgument, FunctionReturnColumn, PlannerScope, + SchemaColumn, + SeedCsvSettings, build_direct_model_version_identities, build_downstream_deps, build_model_version_identity_metadata_json, @@ -26,7 +28,6 @@ CompiledResourceType, FunctionLanguage, ) -from sqlbuild.spec.contracts.models import SchemaColumn, SeedCsvSettings from tests.unit.src.sqlbuild.compiler.planner._helpers.helpers import build_compiled_function diff --git a/tests/unit/src/sqlbuild/compiler/planner/main/test_future_cursor_safety.py b/tests/unit/src/sqlbuild/compiler/planner/main/test_future_cursor_safety.py index 62d897ec76..c795785666 100644 --- a/tests/unit/src/sqlbuild/compiler/planner/main/test_future_cursor_safety.py +++ b/tests/unit/src/sqlbuild/compiler/planner/main/test_future_cursor_safety.py @@ -4,14 +4,12 @@ import pytest +from sqlbuild._native import FutureCursorAction, FutureCursorsConfig, future_cursor_cap_warning from sqlbuild._native import cursor_sentinel_to_token as sentinel_to_token -from sqlbuild._native import future_cursor_cap_warning from sqlbuild._native import resolve_future_cursor_safety as apply_future_cursor_safety from sqlbuild.compiler.planner.exceptions import FutureCursorSafetyError from sqlbuild.compiler.planner.models import CursorBounds from sqlbuild.compiler.planner.types import CursorGrain, CursorType -from sqlbuild.spec.contracts.models import FutureCursorsConfig -from sqlbuild.spec.contracts.types import FutureCursorAction from tests.unit.src.sqlbuild.compiler.planner.main._test_types import FutureCursorSafetyTestCase diff --git a/tests/unit/src/sqlbuild/compiler/planner/main/test_maximum_start_safety.py b/tests/unit/src/sqlbuild/compiler/planner/main/test_maximum_start_safety.py index 6c10df7d30..592cbea885 100644 --- a/tests/unit/src/sqlbuild/compiler/planner/main/test_maximum_start_safety.py +++ b/tests/unit/src/sqlbuild/compiler/planner/main/test_maximum_start_safety.py @@ -5,7 +5,12 @@ import pytest -from sqlbuild._native import compute_cursor_bounds +from sqlbuild._native import ( + FutureCursorAction, + FutureCursorsConfig, + StartCursorsConfig, + compute_cursor_bounds, +) from sqlbuild._native import cursor_sentinel_to_token as sentinel_to_token from sqlbuild._native import resolve_future_cursor_safety as apply_future_cursor_safety from sqlbuild.compiler.planner.exceptions import MaximumAutomaticStartError @@ -14,8 +19,6 @@ MaximumStartPolicyInputs, ModelCursorSnapshot, ) -from sqlbuild.spec.contracts.models import FutureCursorsConfig, StartCursorsConfig -from sqlbuild.spec.contracts.types import FutureCursorAction from tests.unit.src.sqlbuild.compiler.planner.main._test_types import ( MaximumStartCapTestCase, MaximumStartErrorTestCase, diff --git a/tests/unit/src/sqlbuild/compiler/planner/main/test_static_command_planning.py b/tests/unit/src/sqlbuild/compiler/planner/main/test_static_command_planning.py index 50a5f7cfac..74aae170d8 100644 --- a/tests/unit/src/sqlbuild/compiler/planner/main/test_static_command_planning.py +++ b/tests/unit/src/sqlbuild/compiler/planner/main/test_static_command_planning.py @@ -12,6 +12,7 @@ from sqlbuild._native import ( CompileModelConfig, DiscoveredSeedFile, + Fingerprint, PlannerRelationsContext, PlannerScope, PlannerSelection, @@ -25,7 +26,6 @@ resolve_static_relation_context, ) from sqlbuild.compiler.compile.types import CompiledProjectView, CompiledSeedView -from sqlbuild.compiler.fingerprints.models import Fingerprint from sqlbuild.compiler.planner.models import PlanOutput, SeedPlanEntry from sqlbuild.compiler.planner.types import PlanReason from tests.unit.src.sqlbuild.compiler.planner._helpers.helpers import ( diff --git a/tests/unit/src/sqlbuild/compiler/planner/test_models.py b/tests/unit/src/sqlbuild/compiler/planner/test_models.py index 847e0b6039..11959af4a7 100644 --- a/tests/unit/src/sqlbuild/compiler/planner/test_models.py +++ b/tests/unit/src/sqlbuild/compiler/planner/test_models.py @@ -6,9 +6,9 @@ import pytest +from sqlbuild._native import MissingMigrationOriginPolicy from sqlbuild.compiler.migrations.types import MigrationDecision from sqlbuild.compiler.planner.models import ModelMigrationPlanEntry -from sqlbuild.spec.contracts.types import MissingMigrationOriginPolicy from tests.unit.src.sqlbuild.compiler.planner._test_types import MissingOriginBlockTestCase from tests.unit.src.sqlbuild.compiler.planner.main.pre_build.helpers import migration_entry diff --git a/tests/unit/src/sqlbuild/compiler/python_nodes/_helpers/helpers.py b/tests/unit/src/sqlbuild/compiler/python_nodes/_helpers/helpers.py index 883331fe33..87a9179fe7 100644 --- a/tests/unit/src/sqlbuild/compiler/python_nodes/_helpers/helpers.py +++ b/tests/unit/src/sqlbuild/compiler/python_nodes/_helpers/helpers.py @@ -13,7 +13,12 @@ from sqlbuild._native import ( CompiledSource, CompileModelConfig, + LocalConfig, + ProjectConfig, + PythonNodeGraph, PythonSqlRunSelection, + SourceColumnEntry, + SourceEntry, build_python_node_graph, compiled_graph_keys, resolve_python_sql_run_selection, @@ -40,17 +45,10 @@ ) from sqlbuild.compiler.discovery.types import LoaderConnectionMode from sqlbuild.compiler.pipeline.models import ProjectGraph -from sqlbuild.compiler.python_nodes.models import PythonNodeGraph from sqlbuild.providers import Provider from sqlbuild.python_nodes.models import SqlResourceRef from sqlbuild.python_nodes.types import PythonCheckSeverity from sqlbuild.refs import model, source -from sqlbuild.spec.contracts.models import ( - LocalConfig, - ProjectConfig, - SourceColumnEntry, - SourceEntry, -) from sqlbuild.tasks import task from tests.unit.src.sqlbuild.compiler.graph.helpers import project_graph_from_indexes diff --git a/tests/unit/src/sqlbuild/compiler/python_nodes/_helpers/test_identity.py b/tests/unit/src/sqlbuild/compiler/python_nodes/_helpers/test_identity.py index 62a7809ca1..0f68b1caee 100644 --- a/tests/unit/src/sqlbuild/compiler/python_nodes/_helpers/test_identity.py +++ b/tests/unit/src/sqlbuild/compiler/python_nodes/_helpers/test_identity.py @@ -8,8 +8,8 @@ import pytest +from sqlbuild._native import PythonNodeIdentity from sqlbuild.compiler.python_nodes.main.identity import build_python_node_identity -from sqlbuild.compiler.python_nodes.models import PythonNodeIdentity from tests.unit.src.sqlbuild.compiler.python_nodes._helpers._test_types import ( PythonNodeIdentityChangeTestCase, PythonNodeIdentityTestCase, diff --git a/tests/unit/src/sqlbuild/compiler/python_nodes/_helpers/test_inventory.py b/tests/unit/src/sqlbuild/compiler/python_nodes/_helpers/test_inventory.py index e25150d0a8..55e0862df4 100644 --- a/tests/unit/src/sqlbuild/compiler/python_nodes/_helpers/test_inventory.py +++ b/tests/unit/src/sqlbuild/compiler/python_nodes/_helpers/test_inventory.py @@ -9,7 +9,15 @@ import pytest -from sqlbuild._native import build_python_node_graph +from sqlbuild._native import ( + DiscoveredPythonNode, + LocalConfig, + ProjectConfig, + PythonNodeGraph, + PythonNodeIdentity, + SourceColumnEntry, + build_python_node_graph, +) from sqlbuild.compiler.discovery.models import ( DiscoveredAssetFunction, DiscoveredCheckFunction, @@ -19,15 +27,9 @@ DiscoveredTaskFunction, ) from sqlbuild.compiler.discovery.types import LoaderConnectionMode -from sqlbuild.compiler.python_nodes.models import ( - DiscoveredPythonNode, - PythonNodeGraph, - PythonNodeIdentity, -) from sqlbuild.compiler.python_nodes.types import PythonNodeKind from sqlbuild.python_nodes.models import RetryPolicy from sqlbuild.python_nodes.types import PythonCheckSeverity -from sqlbuild.spec.contracts.models import LocalConfig, ProjectConfig, SourceColumnEntry from tests.unit.src.sqlbuild.compiler.python_nodes._helpers._test_types import ( PythonNodeGraphInventoryTestCase, PythonNodeIdentityConfigTestCase, diff --git a/tests/unit/src/sqlbuild/compiler/python_nodes/_helpers/test_run_lifecycle.py b/tests/unit/src/sqlbuild/compiler/python_nodes/_helpers/test_run_lifecycle.py index 9af5d91594..35854c6f9f 100644 --- a/tests/unit/src/sqlbuild/compiler/python_nodes/_helpers/test_run_lifecycle.py +++ b/tests/unit/src/sqlbuild/compiler/python_nodes/_helpers/test_run_lifecycle.py @@ -4,10 +4,14 @@ import pytest -from sqlbuild._native import PythonSqlRunSelection, build_python_sql_run_lifecycle +from sqlbuild._native import ( + PythonNodeGraph, + PythonSqlRunLifecyclePlan, + PythonSqlRunSelection, + build_python_sql_run_lifecycle, +) from sqlbuild.compiler.compile.models import CompiledObjectKey from sqlbuild.compiler.compile.types import CompiledResourceType -from sqlbuild.compiler.python_nodes.models import PythonNodeGraph, PythonSqlRunLifecyclePlan from tests.unit.src.sqlbuild.compiler.python_nodes._helpers._test_types import ( PythonSqlRunLifecycleTestCase, ) diff --git a/tests/unit/src/sqlbuild/compiler/python_nodes/_helpers/test_run_selectors.py b/tests/unit/src/sqlbuild/compiler/python_nodes/_helpers/test_run_selectors.py index 452fbbdd0e..b77e06aa77 100644 --- a/tests/unit/src/sqlbuild/compiler/python_nodes/_helpers/test_run_selectors.py +++ b/tests/unit/src/sqlbuild/compiler/python_nodes/_helpers/test_run_selectors.py @@ -4,9 +4,8 @@ import pytest -from sqlbuild._native import PythonSqlRunSelection +from sqlbuild._native import PythonNodeGraph, PythonSqlRunSelection from sqlbuild.compiler.pipeline.models import ProjectGraph -from sqlbuild.compiler.python_nodes.models import PythonNodeGraph from tests.unit.src.sqlbuild.compiler.python_nodes._helpers._test_types import ( PythonSqlSelectorErrorTestCase, PythonSqlSelectorTestCase, diff --git a/tests/unit/src/sqlbuild/compiler/python_nodes/_helpers/test_selectors.py b/tests/unit/src/sqlbuild/compiler/python_nodes/_helpers/test_selectors.py index 5bdff1d5fb..6d3c8e1827 100644 --- a/tests/unit/src/sqlbuild/compiler/python_nodes/_helpers/test_selectors.py +++ b/tests/unit/src/sqlbuild/compiler/python_nodes/_helpers/test_selectors.py @@ -6,11 +6,11 @@ from sqlbuild._native import ( ParsedSelector, + PythonNodeGraph, SelectorKind, parse_project_selector, resolve_python_node_selection, ) -from sqlbuild.compiler.python_nodes.models import PythonNodeGraph from tests.unit.src.sqlbuild.compiler.python_nodes._helpers._test_types import ( PythonNodeParseSelectorTestCase, PythonNodeSelectorErrorTestCase, diff --git a/tests/unit/src/sqlbuild/compiler/python_nodes/_helpers/test_unified_selectors.py b/tests/unit/src/sqlbuild/compiler/python_nodes/_helpers/test_unified_selectors.py index aef7c08cb9..ca2d6ae1a0 100644 --- a/tests/unit/src/sqlbuild/compiler/python_nodes/_helpers/test_unified_selectors.py +++ b/tests/unit/src/sqlbuild/compiler/python_nodes/_helpers/test_unified_selectors.py @@ -4,10 +4,9 @@ import pytest -from sqlbuild._native import PythonSqlRunSelection, resolve_project_selectors +from sqlbuild._native import PythonNodeGraph, PythonSqlRunSelection, resolve_project_selectors from sqlbuild.compiler.compile.models import CompiledObjectKey from sqlbuild.compiler.pipeline.models import ProjectGraph -from sqlbuild.compiler.python_nodes.models import PythonNodeGraph from sqlbuild.refs import seed from tests.unit.src.sqlbuild.compiler.python_nodes._helpers._test_types import ( PythonSqlSelectorErrorTestCase, diff --git a/tests/unit/src/sqlbuild/compiler/scopes/helpers.py b/tests/unit/src/sqlbuild/compiler/scopes/helpers.py index 26426537ce..c25736e064 100644 --- a/tests/unit/src/sqlbuild/compiler/scopes/helpers.py +++ b/tests/unit/src/sqlbuild/compiler/scopes/helpers.py @@ -15,9 +15,11 @@ EnumMemberMetadata, EnumMetadata, GrantRecord, + LocalConfig, MacroMetadata, OwnershipRoot, OwnershipRootKind, + ProjectConfig, ResourceIdentity, ResourceKind, ResourceRecord, @@ -34,7 +36,6 @@ DiscoveredSqlModelFile, EnumDeclaration, ) -from sqlbuild.spec.contracts.models import LocalConfig, ProjectConfig SCOPE_CACHE_PROJECT: str = 'name = "demo"\nadapter = "duckdb"\n' SCOPE_CACHE_MODEL: str = "MODEL(description 'Test model.');\nSELECT 1 AS id\n" diff --git a/tests/unit/src/sqlbuild/compiler/scopes/test_builder_and_visibility.py b/tests/unit/src/sqlbuild/compiler/scopes/test_builder_and_visibility.py index 715dba24c5..fd4fe7233a 100644 --- a/tests/unit/src/sqlbuild/compiler/scopes/test_builder_and_visibility.py +++ b/tests/unit/src/sqlbuild/compiler/scopes/test_builder_and_visibility.py @@ -29,6 +29,7 @@ ScopeLookup, ScopeTargetQuery, ScopeValidationError, + SourceEntry, VisibilityReason, VisibilityResolution, build_scope_index, @@ -47,7 +48,6 @@ DiscoveredSqlTestFile, EnumDeclaration, ) -from sqlbuild.spec.contracts.models import SourceEntry from tests.unit.src.sqlbuild.compiler.scopes._test_types import ( ExpectedBooleanCase, ResourceExpectation, diff --git a/tests/unit/src/sqlbuild/compiler/source_freshness/_helpers/test_age_policy.py b/tests/unit/src/sqlbuild/compiler/source_freshness/_helpers/test_age_policy.py index cadbc02b3b..bc760126dd 100644 --- a/tests/unit/src/sqlbuild/compiler/source_freshness/_helpers/test_age_policy.py +++ b/tests/unit/src/sqlbuild/compiler/source_freshness/_helpers/test_age_policy.py @@ -4,8 +4,7 @@ import pytest -from sqlbuild._native import evaluate_source_freshness_age_policy -from sqlbuild.spec.contracts.models import SourceFreshnessAgePolicy +from sqlbuild._native import SourceFreshnessAgePolicy, evaluate_source_freshness_age_policy from tests.unit.src.sqlbuild.compiler.source_freshness._helpers._test_types import ( SourceFreshnessAgePolicyDurationTestCase, ) diff --git a/tests/unit/src/sqlbuild/compiler/source_freshness/main/_test_types.py b/tests/unit/src/sqlbuild/compiler/source_freshness/main/_test_types.py index 10db904107..d7b594c0b5 100644 --- a/tests/unit/src/sqlbuild/compiler/source_freshness/main/_test_types.py +++ b/tests/unit/src/sqlbuild/compiler/source_freshness/main/_test_types.py @@ -2,9 +2,8 @@ from datetime import UTC, datetime from typing import Any -from sqlbuild._native import SourceFreshnessUnknownReason +from sqlbuild._native import SourceFreshnessUnknownReason, SourceFreshnessValueKind from sqlbuild.adapter.contract.models import TableFreshnessMetadata -from sqlbuild.spec.contracts.types import SourceFreshnessValueKind @dataclass(frozen=True) diff --git a/tests/unit/src/sqlbuild/compiler/source_freshness/main/configured_observation/_test_types.py b/tests/unit/src/sqlbuild/compiler/source_freshness/main/configured_observation/_test_types.py index bfde40ce0b..98a069d2cb 100644 --- a/tests/unit/src/sqlbuild/compiler/source_freshness/main/configured_observation/_test_types.py +++ b/tests/unit/src/sqlbuild/compiler/source_freshness/main/configured_observation/_test_types.py @@ -1,6 +1,6 @@ from dataclasses import dataclass -from sqlbuild.spec.contracts.types import SourceFreshnessValueKind +from sqlbuild._native import SourceFreshnessValueKind @dataclass(frozen=True) diff --git a/tests/unit/src/sqlbuild/compiler/source_freshness/main/configured_observation/test_observation.py b/tests/unit/src/sqlbuild/compiler/source_freshness/main/configured_observation/test_observation.py index 60a11450b1..558e7417bb 100644 --- a/tests/unit/src/sqlbuild/compiler/source_freshness/main/configured_observation/test_observation.py +++ b/tests/unit/src/sqlbuild/compiler/source_freshness/main/configured_observation/test_observation.py @@ -6,15 +6,17 @@ import pytest from sqlbuild._native import ( + SourceEntry, + SourceFreshnessConfig, SourceFreshnessObservation, SourceFreshnessObservationError, + SourceFreshnessStrategy, + SourceFreshnessValueKind, normalize_source_freshness_data_version, observe_configured_source_freshness, ) from sqlbuild.adapter.contract.models import TableFreshnessMetadata from sqlbuild.adapters.duckdb.classes.duckdb_adapter import DuckDbAdapter -from sqlbuild.spec.contracts.models import SourceEntry, SourceFreshnessConfig -from sqlbuild.spec.contracts.types import SourceFreshnessStrategy, SourceFreshnessValueKind from tests.unit.src.sqlbuild.compiler.source_freshness.main.configured_observation._test_types import ( SourceFreshnessObservationErrorTestCase, SourceFreshnessObservationTestCase, diff --git a/tests/unit/src/sqlbuild/compiler/source_freshness/main/helpers.py b/tests/unit/src/sqlbuild/compiler/source_freshness/main/helpers.py index 37cd5b2a76..e581a5d9d0 100644 --- a/tests/unit/src/sqlbuild/compiler/source_freshness/main/helpers.py +++ b/tests/unit/src/sqlbuild/compiler/source_freshness/main/helpers.py @@ -6,8 +6,13 @@ from typing import Any from sqlbuild._native import ( + SourceEntry, + SourceFreshnessConfig, SourceFreshnessIdentity, + SourceFreshnessRecord, SourceFreshnessRenderers, + SourceFreshnessStrategy, + SourceFreshnessValueKind, build_source_freshness_read_latest_sql, source_freshness_data_version_hash, write_source_freshness_records, @@ -23,9 +28,6 @@ from sqlbuild.compiler.compile.models import CompiledObjectKey from sqlbuild.compiler.compile.types import CompiledResourceType from sqlbuild.compiler.source_freshness.constants import SOURCE_FRESHNESS_TABLE_NAME -from sqlbuild.compiler.source_freshness.models import SourceFreshnessRecord -from sqlbuild.spec.contracts.models import SourceEntry, SourceFreshnessConfig -from sqlbuild.spec.contracts.types import SourceFreshnessStrategy, SourceFreshnessValueKind def state_table_exists_map( diff --git a/tests/unit/src/sqlbuild/compiler/source_freshness/main/test_adapter_observation.py b/tests/unit/src/sqlbuild/compiler/source_freshness/main/test_adapter_observation.py index f9f2dd9bfc..5097f6fddc 100644 --- a/tests/unit/src/sqlbuild/compiler/source_freshness/main/test_adapter_observation.py +++ b/tests/unit/src/sqlbuild/compiler/source_freshness/main/test_adapter_observation.py @@ -8,6 +8,8 @@ from sqlbuild._native import ( AdapterSourceFreshnessBatch, DirectSourceFreshnessPlanningResult, + SourceFreshnessConfig, + SourceFreshnessStrategy, SourceFreshnessUnknownReason, build_direct_source_freshness_planning_result, observe_adapter_sources_freshness, @@ -19,8 +21,6 @@ dispatcher_scope, invocation_scope, ) -from sqlbuild.spec.contracts.models import SourceFreshnessConfig -from sqlbuild.spec.contracts.types import SourceFreshnessStrategy from tests.unit.src.sqlbuild.compiler.source_freshness.main._test_types import ( AdapterBatchIsolationTestCase, ) diff --git a/tests/unit/src/sqlbuild/compiler/source_freshness/main/test_age_policy.py b/tests/unit/src/sqlbuild/compiler/source_freshness/main/test_age_policy.py index 9d80b24ac6..4adc4cfcc3 100644 --- a/tests/unit/src/sqlbuild/compiler/source_freshness/main/test_age_policy.py +++ b/tests/unit/src/sqlbuild/compiler/source_freshness/main/test_age_policy.py @@ -4,8 +4,11 @@ import pytest -from sqlbuild._native import SourceFreshnessAgeStatus, evaluate_source_freshness_age_policy -from sqlbuild.spec.contracts.models import SourceFreshnessAgePolicy +from sqlbuild._native import ( + SourceFreshnessAgePolicy, + SourceFreshnessAgeStatus, + evaluate_source_freshness_age_policy, +) from tests.unit.src.sqlbuild.compiler.source_freshness.main._test_types import ( SourceFreshnessAgeEvaluationTestCase, SourceFreshnessNaiveDatetimeTestCase, diff --git a/tests/unit/src/sqlbuild/compiler/source_freshness/main/test_models.py b/tests/unit/src/sqlbuild/compiler/source_freshness/main/test_models.py index 2373169f04..e9d8628b6f 100644 --- a/tests/unit/src/sqlbuild/compiler/source_freshness/main/test_models.py +++ b/tests/unit/src/sqlbuild/compiler/source_freshness/main/test_models.py @@ -4,9 +4,12 @@ import pytest -from sqlbuild._native import SourceFreshnessObservation -from sqlbuild.compiler.source_freshness.models import SourceFreshnessRecord -from sqlbuild.spec.contracts.types import SourceFreshnessStrategy, SourceFreshnessValueKind +from sqlbuild._native import ( + SourceFreshnessObservation, + SourceFreshnessRecord, + SourceFreshnessStrategy, + SourceFreshnessValueKind, +) from tests.unit.src.sqlbuild.compiler.source_freshness.main._test_types import ( SourceFreshnessDatetimeNormalizationTestCase, SourceFreshnessNaiveDatetimeTestCase, diff --git a/tests/unit/src/sqlbuild/compiler/source_freshness/main/test_observation.py b/tests/unit/src/sqlbuild/compiler/source_freshness/main/test_observation.py index 728bc319c4..49003234d5 100644 --- a/tests/unit/src/sqlbuild/compiler/source_freshness/main/test_observation.py +++ b/tests/unit/src/sqlbuild/compiler/source_freshness/main/test_observation.py @@ -6,7 +6,11 @@ import pytest from sqlbuild._native import ( + SourceEntry, + SourceFreshnessConfig, SourceFreshnessObservation, + SourceFreshnessStrategy, + SourceFreshnessValueKind, observe_configured_source_freshness, ) from sqlbuild.adapter.contract.models import QueryResult, TableFreshnessMetadata @@ -17,8 +21,6 @@ dispatcher_scope, invocation_scope, ) -from sqlbuild.spec.contracts.models import SourceEntry, SourceFreshnessConfig -from sqlbuild.spec.contracts.types import SourceFreshnessStrategy, SourceFreshnessValueKind from tests.unit.src.sqlbuild.compiler.source_freshness.main._test_types import ( SharedSourceFreshnessColumnSqlTestCase, SharedSourceFreshnessExpressionSubqueryTestCase, diff --git a/tests/unit/src/sqlbuild/compiler/source_freshness/main/test_planning.py b/tests/unit/src/sqlbuild/compiler/source_freshness/main/test_planning.py index d5b889758c..38faa71844 100644 --- a/tests/unit/src/sqlbuild/compiler/source_freshness/main/test_planning.py +++ b/tests/unit/src/sqlbuild/compiler/source_freshness/main/test_planning.py @@ -8,7 +8,13 @@ from sqlbuild._native import ( DirectSourceFreshnessPlanningResult, + SourceEntry, + SourceFreshnessAgePolicy, + SourceFreshnessConfig, + SourceFreshnessRecord, SourceFreshnessRenderers, + SourceFreshnessStrategy, + SourceFreshnessValueKind, build_direct_source_freshness_planning_result, source_freshness_data_version_hash, write_source_freshness_records, @@ -16,13 +22,6 @@ from sqlbuild.adapter.contract.models import TableFreshnessMetadata, TableFreshnessRequest from sqlbuild.adapter.contract.types import FrameworkType from sqlbuild.adapters.duckdb.classes.duckdb_adapter import DuckDbAdapter -from sqlbuild.compiler.source_freshness.models import SourceFreshnessRecord -from sqlbuild.spec.contracts.models import ( - SourceEntry, - SourceFreshnessAgePolicy, - SourceFreshnessConfig, -) -from sqlbuild.spec.contracts.types import SourceFreshnessStrategy, SourceFreshnessValueKind from tests.unit.src.sqlbuild.compiler.source_freshness.main._test_types import ( DirectSourceFreshnessAdapterDefaultTestCase, DirectSourceFreshnessAgePolicyTestCase, diff --git a/tests/unit/src/sqlbuild/compiler/source_freshness/main/test_read.py b/tests/unit/src/sqlbuild/compiler/source_freshness/main/test_read.py index b619eb4fed..fbc2e35032 100644 --- a/tests/unit/src/sqlbuild/compiler/source_freshness/main/test_read.py +++ b/tests/unit/src/sqlbuild/compiler/source_freshness/main/test_read.py @@ -7,13 +7,13 @@ from sqlbuild._native import ( SourceFreshnessIdentity, SourceFreshnessInputError, + SourceFreshnessRecord, SourceFreshnessRenderers, SourceFreshnessSet, read_latest_source_freshness, write_source_freshness_records, ) from sqlbuild.adapters.duckdb.classes.duckdb_adapter import DuckDbAdapter -from sqlbuild.compiler.source_freshness.models import SourceFreshnessRecord from tests.unit.src.sqlbuild.compiler.source_freshness.main._test_types import ( ReadLatestSourceFreshnessErrorTestCase, ReadLatestSourceFreshnessRendererTestCase, diff --git a/tests/unit/src/sqlbuild/compiler/source_freshness/main/test_state.py b/tests/unit/src/sqlbuild/compiler/source_freshness/main/test_state.py index 793d2035ba..50a1a49c3a 100644 --- a/tests/unit/src/sqlbuild/compiler/source_freshness/main/test_state.py +++ b/tests/unit/src/sqlbuild/compiler/source_freshness/main/test_state.py @@ -5,12 +5,13 @@ import pytest from sqlbuild._native import ( + SourceFreshnessRecord, + SourceFreshnessStrategy, + SourceFreshnessValueKind, normalize_source_freshness_data_version, source_freshness_data_version_hash, source_freshness_records_equivalent, ) -from sqlbuild.compiler.source_freshness.models import SourceFreshnessRecord -from sqlbuild.spec.contracts.types import SourceFreshnessStrategy, SourceFreshnessValueKind from tests.unit.src.sqlbuild.compiler.source_freshness.main._test_types import ( SharedSourceFreshnessHashTestCase, SourceFreshnessStateCompatibilityTestCase, diff --git a/tests/unit/src/sqlbuild/compiler/sql_analysis/_helpers/_test_types.py b/tests/unit/src/sqlbuild/compiler/sql_analysis/_helpers/_test_types.py index 64026852d6..5948b906a5 100644 --- a/tests/unit/src/sqlbuild/compiler/sql_analysis/_helpers/_test_types.py +++ b/tests/unit/src/sqlbuild/compiler/sql_analysis/_helpers/_test_types.py @@ -2,7 +2,7 @@ from dataclasses import dataclass -from sqlbuild.compiler.sql_analysis.models import SqlLexicalSyntax +from sqlbuild._native import SqlLexicalSyntax @dataclass(frozen=True) diff --git a/tests/unit/src/sqlbuild/compiler/sql_analysis/_helpers/helpers.py b/tests/unit/src/sqlbuild/compiler/sql_analysis/_helpers/helpers.py index 79b8981a2d..f00fa74306 100644 --- a/tests/unit/src/sqlbuild/compiler/sql_analysis/_helpers/helpers.py +++ b/tests/unit/src/sqlbuild/compiler/sql_analysis/_helpers/helpers.py @@ -3,7 +3,7 @@ from __future__ import annotations import sqlbuild._native as _native -from sqlbuild.compiler.sql_analysis.models import SqlLexicalSyntax +from sqlbuild._native import SqlLexicalSyntax def generic_scanner(*, sql: str, context: str = "SQL") -> _native.SqlCodeScanner: diff --git a/tests/unit/src/sqlbuild/compiler/sql_analysis/_helpers/test_scanning.py b/tests/unit/src/sqlbuild/compiler/sql_analysis/_helpers/test_scanning.py index 850e3434e6..dbd5725a51 100644 --- a/tests/unit/src/sqlbuild/compiler/sql_analysis/_helpers/test_scanning.py +++ b/tests/unit/src/sqlbuild/compiler/sql_analysis/_helpers/test_scanning.py @@ -3,8 +3,8 @@ import pytest import sqlbuild._native as _native +from sqlbuild._native import SqlLexicalSyntax from sqlbuild.compiler.compile.exceptions import CompileInputError -from sqlbuild.compiler.sql_analysis.models import SqlLexicalSyntax from tests.unit.src.sqlbuild.compiler.sql_analysis._helpers._test_types import ( DialectMatchingParenTestCase, DialectNonCodeErrorTestCase, diff --git a/tests/unit/src/sqlbuild/cursor_algebra/main/test_cursor_algebra.py b/tests/unit/src/sqlbuild/cursor_algebra/main/test_cursor_algebra.py index 9968b12aef..da1df0f0cf 100644 --- a/tests/unit/src/sqlbuild/cursor_algebra/main/test_cursor_algebra.py +++ b/tests/unit/src/sqlbuild/cursor_algebra/main/test_cursor_algebra.py @@ -4,6 +4,7 @@ import pytest +from sqlbuild._native import AlignedInterval, IntegerValue, TimestampValue from sqlbuild._native import cursor_cap_from_end as cap_from_end from sqlbuild._native import cursor_cap_from_start as cap_from_start from sqlbuild._native import cursor_clamp as clamp @@ -25,7 +26,6 @@ from sqlbuild._native import cursor_sentinel_to_token as sentinel_to_token from sqlbuild._native import cursor_split_into_batches as split_into_batches from sqlbuild.compiler.planner.types import CursorGrain, CursorType -from sqlbuild.cursor_algebra.models import AlignedInterval, IntegerValue, TimestampValue from sqlbuild.cursor_algebra.types import BoundSentinel, CursorScalar from tests.unit.src.sqlbuild.cursor_algebra.main._test_types import ( CursorAlgebraMatrixCase, diff --git a/tests/unit/src/sqlbuild/cursor_algebra/test_duration.py b/tests/unit/src/sqlbuild/cursor_algebra/test_duration.py index 8056cfdafe..8b4e8473d9 100644 --- a/tests/unit/src/sqlbuild/cursor_algebra/test_duration.py +++ b/tests/unit/src/sqlbuild/cursor_algebra/test_duration.py @@ -4,7 +4,7 @@ import pytest -from sqlbuild.cursor_algebra.models import Duration +from sqlbuild._native import Duration from tests.unit.src.sqlbuild.cursor_algebra._test_types import ( DurationParseNoneTestCase, DurationParseTestCase, diff --git a/tests/unit/src/sqlbuild/executor/auditing/main/helpers.py b/tests/unit/src/sqlbuild/executor/auditing/main/helpers.py index 438cd53137..a453b512c3 100644 --- a/tests/unit/src/sqlbuild/executor/auditing/main/helpers.py +++ b/tests/unit/src/sqlbuild/executor/auditing/main/helpers.py @@ -6,7 +6,7 @@ from types import SimpleNamespace from typing import Any, cast -from sqlbuild.compiler.auditing.models import MeasurementThresholdBound, MeasurementThresholds +from sqlbuild._native import MeasurementThresholdBound, MeasurementThresholds from sqlbuild.compiler.auditing.types import ( AuditAttachmentKind, AuditEvaluationMode, diff --git a/tests/unit/src/sqlbuild/executor/build/_helpers/helpers.py b/tests/unit/src/sqlbuild/executor/build/_helpers/helpers.py index d453bfd9a6..e97b231303 100644 --- a/tests/unit/src/sqlbuild/executor/build/_helpers/helpers.py +++ b/tests/unit/src/sqlbuild/executor/build/_helpers/helpers.py @@ -6,6 +6,7 @@ from typing import Any from unittest.mock import Mock +from sqlbuild._native import SeedCsvSettings, SourceColumnEntry, SourceEntry, SourceWriteStrategy from sqlbuild.adapter.contract.classes.retention_adapter import RetentionAdapterMixin from sqlbuild.compiler.auditing.types import ( AuditAttachmentKind, @@ -31,8 +32,6 @@ PlanReason, ) from sqlbuild.executor.auditing.models import AuditExecutionResult -from sqlbuild.spec.contracts.models import SeedCsvSettings, SourceColumnEntry, SourceEntry -from sqlbuild.spec.contracts.types import SourceWriteStrategy def build_audit_result( diff --git a/tests/unit/src/sqlbuild/executor/build/_helpers/test_external_source_loads.py b/tests/unit/src/sqlbuild/executor/build/_helpers/test_external_source_loads.py index b194c12040..1a0cc81d66 100644 --- a/tests/unit/src/sqlbuild/executor/build/_helpers/test_external_source_loads.py +++ b/tests/unit/src/sqlbuild/executor/build/_helpers/test_external_source_loads.py @@ -6,6 +6,7 @@ import pytest +from sqlbuild._native import SourceEntry from sqlbuild.compiler.compile.models import CompiledObjectKey from sqlbuild.compiler.compile.types import CompiledResourceType from sqlbuild.compiler.discovery.models import DiscoveredLoaderFunction @@ -28,7 +29,6 @@ invocation_scope, ) from sqlbuild.runtime.contracts.types import ExecutionResourceKind -from sqlbuild.spec.contracts.models import SourceEntry from tests.unit.src.sqlbuild.executor.build._helpers._test_types import ( ExternalBuildSourceLoadTestCase, ) diff --git a/tests/unit/src/sqlbuild/executor/build/_helpers/test_scheduler.py b/tests/unit/src/sqlbuild/executor/build/_helpers/test_scheduler.py index bc41d529fa..5e0371dc78 100644 --- a/tests/unit/src/sqlbuild/executor/build/_helpers/test_scheduler.py +++ b/tests/unit/src/sqlbuild/executor/build/_helpers/test_scheduler.py @@ -9,7 +9,7 @@ import pytest -from sqlbuild._native import NativeProgressProjector +from sqlbuild._native import NativeProgressProjector, SourceEntry, SourceWriteStrategy from sqlbuild.adapter.contract.types import TablePromotionMode from sqlbuild.adapters.duckdb.classes.duckdb_adapter import DuckDbAdapter from sqlbuild.compiler.compile.models import CompiledObjectKey @@ -38,8 +38,6 @@ invocation_scope, ) from sqlbuild.runtime.contracts.types import ExecutionResourceKind -from sqlbuild.spec.contracts.models import SourceEntry -from sqlbuild.spec.contracts.types import SourceWriteStrategy from tests.unit.src.sqlbuild.executor.build._helpers._test_types import ( BuildSchedulerModelHookTestCase, BuildSchedulerPlannedSkipTestCase, diff --git a/tests/unit/src/sqlbuild/executor/clone/_helpers/fingerprinting/helpers.py b/tests/unit/src/sqlbuild/executor/clone/_helpers/fingerprinting/helpers.py index 46b1b95b02..40b63bbbea 100644 --- a/tests/unit/src/sqlbuild/executor/clone/_helpers/fingerprinting/helpers.py +++ b/tests/unit/src/sqlbuild/executor/clone/_helpers/fingerprinting/helpers.py @@ -5,14 +5,13 @@ import pytest +from sqlbuild._native import Fingerprint, FingerprintSet, SeedCsvSettings from sqlbuild.adapter.contract.models import RelationInfo from sqlbuild.compiler.compile.models import CompiledObjectKey, CompiledRelationLocation from sqlbuild.compiler.compile.types import CompiledResourceType from sqlbuild.compiler.fingerprints.constants import FINGERPRINT_TABLE_NAME -from sqlbuild.compiler.fingerprints.models import Fingerprint, FingerprintSet from sqlbuild.compiler.planner.models import ModelPlanEntry, SeedPlanEntry from sqlbuild.compiler.planner.types import MaterializationType, PlanAction, PlanReason -from sqlbuild.spec.contracts.models import SeedCsvSettings class CloneFingerprintAdapter: diff --git a/tests/unit/src/sqlbuild/executor/clone/_helpers/fingerprinting/test_fingerprinting.py b/tests/unit/src/sqlbuild/executor/clone/_helpers/fingerprinting/test_fingerprinting.py index c4220e6bdd..f6eaf6b7cc 100644 --- a/tests/unit/src/sqlbuild/executor/clone/_helpers/fingerprinting/test_fingerprinting.py +++ b/tests/unit/src/sqlbuild/executor/clone/_helpers/fingerprinting/test_fingerprinting.py @@ -5,9 +5,9 @@ import pytest +from sqlbuild._native import Fingerprint, FingerprintSet from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter from sqlbuild.compiler.fingerprints.constants import NODE_TYPE_MODEL, NODE_TYPE_SEED -from sqlbuild.compiler.fingerprints.models import Fingerprint, FingerprintSet from sqlbuild.compiler.planner.models import ModelPlanEntry, SeedPlanEntry from sqlbuild.compiler.planner.types import MaterializationType from sqlbuild.executor.clone._helpers.fingerprinting import copy_clone_fingerprints diff --git a/tests/unit/src/sqlbuild/executor/clone/_helpers/helpers.py b/tests/unit/src/sqlbuild/executor/clone/_helpers/helpers.py index 985dd7a269..cc48d4ce0a 100644 --- a/tests/unit/src/sqlbuild/executor/clone/_helpers/helpers.py +++ b/tests/unit/src/sqlbuild/executor/clone/_helpers/helpers.py @@ -3,7 +3,11 @@ from pathlib import Path from typing import Any, ClassVar -from sqlbuild._native import CompileModelConfig +from sqlbuild._native import ( + CompileModelConfig, + ResolvedTimeTravelRetention, + TimeTravelRetentionSource, +) from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter from sqlbuild.adapter.contract.models import RelationInfo from sqlbuild.compiler.compile.models import ( @@ -19,8 +23,6 @@ ) from sqlbuild.compiler.planner.models import FunctionPlanEntry, ModelPlanEntry from sqlbuild.compiler.planner.types import MaterializationType, PlanAction, PlanReason -from sqlbuild.spec.contracts.models import ResolvedTimeTravelRetention -from sqlbuild.spec.contracts.types import TimeTravelRetentionSource class FakeCloneAdapter(BaseAdapter): diff --git a/tests/unit/src/sqlbuild/executor/janitor/main/helpers.py b/tests/unit/src/sqlbuild/executor/janitor/main/helpers.py index 43417dd2fc..d0a5035533 100644 --- a/tests/unit/src/sqlbuild/executor/janitor/main/helpers.py +++ b/tests/unit/src/sqlbuild/executor/janitor/main/helpers.py @@ -9,7 +9,14 @@ from pathlib import Path from typing import Any, ClassVar, cast -from sqlbuild._native import CompiledSeed, CompiledSource, CompileModelConfig +from sqlbuild._native import ( + CompiledSeed, + CompiledSource, + CompileModelConfig, + SchemaSeedEntry, + SeedCsvSettings, + SourceEntry, +) from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter from sqlbuild.adapter.contract.classes.statement_recorder import StatementRecorder from sqlbuild.adapter.contract.models import ( @@ -36,7 +43,6 @@ from sqlbuild.compiler.source_freshness.constants import SOURCE_FRESHNESS_TABLE_NAME from sqlbuild.executor.janitor_events.constants import JANITOR_EVENTS_TABLE_NAME from sqlbuild.spec.contracts.constants import DEFAULT_SEED_CSV_SETTINGS -from sqlbuild.spec.contracts.models import SchemaSeedEntry, SeedCsvSettings, SourceEntry UNQUOTED_ADDRESSING_REASON: str = ( "relation name is not a plain lowercase identifier and may require quoting; " diff --git a/tests/unit/src/sqlbuild/executor/load/_test_types.py b/tests/unit/src/sqlbuild/executor/load/_test_types.py index d7883ac3d9..bc458caba8 100644 --- a/tests/unit/src/sqlbuild/executor/load/_test_types.py +++ b/tests/unit/src/sqlbuild/executor/load/_test_types.py @@ -6,12 +6,12 @@ from datetime import date, datetime from typing import Any +from sqlbuild._native import SourceColumnEntry from sqlbuild.adapter.contract.types import CursorKind from sqlbuild.compiler.discovery.types import LoaderConnectionMode from sqlbuild.compiler.python_nodes.types import SkipMode from sqlbuild.cursor_algebra.types import CursorScalar from sqlbuild.executor.scheduling.types import ExecutionStatus -from sqlbuild.spec.contracts.models import SourceColumnEntry @dataclass(frozen=True) diff --git a/tests/unit/src/sqlbuild/executor/load/test_cursors.py b/tests/unit/src/sqlbuild/executor/load/test_cursors.py index 12e9540275..249c33ddd3 100644 --- a/tests/unit/src/sqlbuild/executor/load/test_cursors.py +++ b/tests/unit/src/sqlbuild/executor/load/test_cursors.py @@ -7,10 +7,10 @@ import pytest +from sqlbuild._native import DateValue, IntegerValue, TimestampValue from sqlbuild._native import cursor_render as render from sqlbuild.adapter.contract.types import CursorKind from sqlbuild.cursor_algebra.exceptions import CursorAlgebraError -from sqlbuild.cursor_algebra.models import DateValue, IntegerValue, TimestampValue from sqlbuild.cursor_algebra.types import CursorScalar from sqlbuild.errors.contracts.exceptions import ExecutorInputError from sqlbuild.executor.load._helpers.cursors import _parse_loader_cursor, infer_cursor_kind diff --git a/tests/unit/src/sqlbuild/executor/load/test_dag_runtime.py b/tests/unit/src/sqlbuild/executor/load/test_dag_runtime.py index cabeea0117..d06f82daf1 100644 --- a/tests/unit/src/sqlbuild/executor/load/test_dag_runtime.py +++ b/tests/unit/src/sqlbuild/executor/load/test_dag_runtime.py @@ -6,6 +6,7 @@ import pytest +from sqlbuild._native import SourceEntry from sqlbuild.executor.load._helpers.dag_runtime import ( build_load_dag_state, complete_dag_source, @@ -19,7 +20,6 @@ LoadRuntimeParams, ) from sqlbuild.executor.scheduling.types import ExecutionStatus -from sqlbuild.spec.contracts.models import SourceEntry from tests.unit.src.sqlbuild.executor.load._test_types import ( LoadDagStateSchedulingTestCase, LoadDagWorkerFailureTestCase, diff --git a/tests/unit/src/sqlbuild/executor/load/test_execute.py b/tests/unit/src/sqlbuild/executor/load/test_execute.py index 9e5203a0d5..c7c9ebc22f 100644 --- a/tests/unit/src/sqlbuild/executor/load/test_execute.py +++ b/tests/unit/src/sqlbuild/executor/load/test_execute.py @@ -9,7 +9,7 @@ import pytest -from sqlbuild._native import CostContext +from sqlbuild._native import CostContext, SourceEntry, SourceWriteStrategy from sqlbuild.adapter.contract.classes.statement_recorder import StatementRecorder from sqlbuild.compiler.discovery.models import DiscoveredLoaderFunction from sqlbuild.compiler.discovery.types import LoaderConnectionMode @@ -31,8 +31,6 @@ from sqlbuild.provider.classes.container import ProviderContainer from sqlbuild.provider.classes.session import ProviderSession from sqlbuild.runtime.observability.classes.statement_lifecycle import StatementLifecycle -from sqlbuild.spec.contracts.models import SourceEntry -from sqlbuild.spec.contracts.types import SourceWriteStrategy from tests.unit.src.sqlbuild.executor.load._test_types import ( ExternalLoaderContractTestCase, LoaderOperationLifecycleTestCase, diff --git a/tests/unit/src/sqlbuild/executor/load/test_rows.py b/tests/unit/src/sqlbuild/executor/load/test_rows.py index afcc0b9c14..b4b006dfaf 100644 --- a/tests/unit/src/sqlbuild/executor/load/test_rows.py +++ b/tests/unit/src/sqlbuild/executor/load/test_rows.py @@ -8,6 +8,7 @@ import duckdb import pytest +from sqlbuild._native import SourceColumnEntry from sqlbuild.adapters.duckdb.classes.duckdb_adapter import DuckDbAdapter from sqlbuild.errors.contracts.exceptions import ExecutorInputError from sqlbuild.executor.load._helpers.rows import ( @@ -17,7 +18,6 @@ update_loader_rows_schema, ) from sqlbuild.executor.load.models import LoaderRowsSchema -from sqlbuild.spec.contracts.models import SourceColumnEntry from tests.unit.src.sqlbuild.executor.load._test_types import ( LoaderRowsBatchTestCase, LoaderRowsErrorTestCase, diff --git a/tests/unit/src/sqlbuild/executor/load/test_run.py b/tests/unit/src/sqlbuild/executor/load/test_run.py index ecab9d43a5..3158a8ae0a 100644 --- a/tests/unit/src/sqlbuild/executor/load/test_run.py +++ b/tests/unit/src/sqlbuild/executor/load/test_run.py @@ -8,7 +8,7 @@ import pytest -from sqlbuild._native import LoadProgressReporter, NativeProgressProjector +from sqlbuild._native import LoadProgressReporter, NativeProgressProjector, SourceEntry from sqlbuild.compiler.discovery.models import DiscoveredLoaderFunction from sqlbuild.compiler.discovery.types import LoaderConnectionMode from sqlbuild.compiler.python_nodes.types import SkipMode @@ -26,7 +26,6 @@ dispatcher_scope, invocation_scope, ) -from sqlbuild.spec.contracts.models import SourceEntry from tests.unit.src.sqlbuild.executor.load._test_types import ( ConcurrentLoadProgressTestCase, ExternalLoadPipelineTestCase, diff --git a/tests/unit/src/sqlbuild/executor/pipeline/_helpers/helpers.py b/tests/unit/src/sqlbuild/executor/pipeline/_helpers/helpers.py index a680ad3f23..3ce1e7ef7a 100644 --- a/tests/unit/src/sqlbuild/executor/pipeline/_helpers/helpers.py +++ b/tests/unit/src/sqlbuild/executor/pipeline/_helpers/helpers.py @@ -10,6 +10,8 @@ from sqlbuild._native import ( ScenarioGraphPlan, + SeedCsvSettings, + SqlLexicalSyntax, ) from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter from sqlbuild.adapter.contract.classes.statement_recorder import StatementRecorder @@ -37,7 +39,6 @@ ScenarioRelationPlan, SeedPlanEntry, ) -from sqlbuild.compiler.sql_analysis.models import SqlLexicalSyntax from sqlbuild.errors.contracts.exceptions import ExecutorInputError from sqlbuild.executor.auditing.models import AuditExecutionResult from sqlbuild.executor.scenario.constants import SCENARIO_LOCAL_JSONL_INVALID @@ -47,7 +48,6 @@ ) from sqlbuild.observability import LifecycleEvent from sqlbuild.spec.contracts.constants import DEFAULT_SEED_CSV_SETTINGS -from sqlbuild.spec.contracts.models import SeedCsvSettings from tests.unit.src.sqlbuild.executor.pipeline._helpers._test_types import ( ScenarioLocalPipelineTestCase, ) diff --git a/tests/unit/src/sqlbuild/executor/pipeline/_helpers/test_scenario.py b/tests/unit/src/sqlbuild/executor/pipeline/_helpers/test_scenario.py index 02d472bc5b..abd9c47adc 100644 --- a/tests/unit/src/sqlbuild/executor/pipeline/_helpers/test_scenario.py +++ b/tests/unit/src/sqlbuild/executor/pipeline/_helpers/test_scenario.py @@ -7,10 +7,10 @@ import pytest +from sqlbuild._native import SqlLexicalSyntax from sqlbuild.compiler.compile.models import CompiledSqlScenario from sqlbuild.compiler.pipeline.models import CompilePipelineResult from sqlbuild.compiler.planner.models import ScenarioExecutionPlan -from sqlbuild.compiler.sql_analysis.models import SqlLexicalSyntax from sqlbuild.executor.pipeline._helpers import scenario as scenario_pipeline from sqlbuild.executor.scenario._helpers.local import execution as local_scenario from sqlbuild.executor.scenario.constants import ( diff --git a/tests/unit/src/sqlbuild/executor/pipeline/main/helpers.py b/tests/unit/src/sqlbuild/executor/pipeline/main/helpers.py index b3d0b38db2..5e619402d2 100644 --- a/tests/unit/src/sqlbuild/executor/pipeline/main/helpers.py +++ b/tests/unit/src/sqlbuild/executor/pipeline/main/helpers.py @@ -3,6 +3,7 @@ from pathlib import Path from typing import ClassVar +from sqlbuild._native import SourceEntry from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter from sqlbuild.adapter.contract.classes.statement_recorder import StatementRecorder from sqlbuild.adapter.contract.models import ColumnInfo @@ -18,7 +19,6 @@ from sqlbuild.compiler.planner.types import MaterializationType, PlanAction, PlanReason from sqlbuild.runtime.contracts.types import ExecutionResourceKind from sqlbuild.spec.contracts.constants import DEFAULT_SEED_CSV_SETTINGS -from sqlbuild.spec.contracts.models import SourceEntry class BuildSchemaPreflightAdapter(BaseAdapter): diff --git a/tests/unit/src/sqlbuild/executor/pipeline/main/test_identity.py b/tests/unit/src/sqlbuild/executor/pipeline/main/test_identity.py index bcf529cd61..3c1789e893 100644 --- a/tests/unit/src/sqlbuild/executor/pipeline/main/test_identity.py +++ b/tests/unit/src/sqlbuild/executor/pipeline/main/test_identity.py @@ -7,7 +7,7 @@ import pytest import sqlbuild.executor.pipeline.main.run as run_module -from sqlbuild._native import CostContext +from sqlbuild._native import CostContext, SettingsConfig from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter from sqlbuild.compiler.planner.models import PlanOutput from sqlbuild.cost.models import CostResourceContext @@ -21,7 +21,6 @@ dispatcher_scope, invocation_scope, ) -from sqlbuild.spec.contracts.models import SettingsConfig from tests.unit.src.sqlbuild.executor.pipeline.main._test_types import ( PipelineIdentityTestCase, RunTerminalTestCase, diff --git a/tests/unit/src/sqlbuild/executor/python_nodes/_helpers/helpers.py b/tests/unit/src/sqlbuild/executor/python_nodes/_helpers/helpers.py index cbdb2539f1..03901f89a4 100644 --- a/tests/unit/src/sqlbuild/executor/python_nodes/_helpers/helpers.py +++ b/tests/unit/src/sqlbuild/executor/python_nodes/_helpers/helpers.py @@ -13,7 +13,13 @@ from sqlbuild._native import ( CostContext, + DiscoveredPythonNode, + LocalConfig, + ProjectConfig, + PythonNodeGraph, + PythonSqlRunLifecyclePlan, PythonSqlRunSelection, + SourceEntry, build_python_node_graph, build_python_sql_run_lifecycle, ) @@ -28,11 +34,6 @@ DiscoveredProjectInputs, DiscoveredTaskFunction, ) -from sqlbuild.compiler.python_nodes.models import ( - DiscoveredPythonNode, - PythonNodeGraph, - PythonSqlRunLifecyclePlan, -) from sqlbuild.compiler.python_nodes.types import SkipMode from sqlbuild.cost.models import CostResourceContext from sqlbuild.errors.contracts.exceptions import ExecutorInputError @@ -53,7 +54,6 @@ from sqlbuild.providers import Provider from sqlbuild.python_nodes.types import PythonCheckSeverity from sqlbuild.refs import model -from sqlbuild.spec.contracts.models import LocalConfig, ProjectConfig, SourceEntry from sqlbuild.tasks import TaskContext, task from tests.unit.src.sqlbuild.compiler.python_nodes._helpers.helpers import ( build_intermediate_loader_asset_dependency_python_node_graph, diff --git a/tests/unit/src/sqlbuild/executor/python_nodes/_helpers/test_fingerprinting.py b/tests/unit/src/sqlbuild/executor/python_nodes/_helpers/test_fingerprinting.py index 24230a7762..f81724f4f0 100644 --- a/tests/unit/src/sqlbuild/executor/python_nodes/_helpers/test_fingerprinting.py +++ b/tests/unit/src/sqlbuild/executor/python_nodes/_helpers/test_fingerprinting.py @@ -4,7 +4,7 @@ import pytest -from sqlbuild.compiler.python_nodes.models import PythonNodeIdentity +from sqlbuild._native import PythonNodeIdentity from sqlbuild.executor.python_nodes._helpers.fingerprinting import ( try_write_python_node_identity_fingerprint, ) diff --git a/tests/unit/src/sqlbuild/executor/python_nodes/_helpers/test_ingress_execution.py b/tests/unit/src/sqlbuild/executor/python_nodes/_helpers/test_ingress_execution.py index 14e3f59a08..573a3696a3 100644 --- a/tests/unit/src/sqlbuild/executor/python_nodes/_helpers/test_ingress_execution.py +++ b/tests/unit/src/sqlbuild/executor/python_nodes/_helpers/test_ingress_execution.py @@ -7,7 +7,7 @@ import pytest -from sqlbuild.compiler.python_nodes.models import DiscoveredPythonNode, PythonNodeGraph +from sqlbuild._native import DiscoveredPythonNode, PythonNodeGraph, SourceEntry from sqlbuild.compiler.python_nodes.types import PythonNodeKind, PythonNodeStatus, SkipMode from sqlbuild.executor.load.main._resource_kind import load_resource_kind from sqlbuild.executor.python_nodes._helpers.ingress_execution import ( @@ -29,7 +29,6 @@ dispatcher_scope, invocation_scope, ) -from sqlbuild.spec.contracts.models import SourceEntry from tests.unit.src.sqlbuild.executor.python_nodes._helpers._test_types import ( IngressSchedulerSkipLifecycleTestCase, PythonIngressLoaderExecutorTestCase, diff --git a/tests/unit/src/sqlbuild/executor/python_nodes/_helpers/test_lifecycle_nodes.py b/tests/unit/src/sqlbuild/executor/python_nodes/_helpers/test_lifecycle_nodes.py index 98819bdafa..29ed327271 100644 --- a/tests/unit/src/sqlbuild/executor/python_nodes/_helpers/test_lifecycle_nodes.py +++ b/tests/unit/src/sqlbuild/executor/python_nodes/_helpers/test_lifecycle_nodes.py @@ -4,7 +4,7 @@ import pytest -from sqlbuild.compiler.python_nodes.models import PythonNodeGraph, PythonSqlRunLifecyclePlan +from sqlbuild._native import PythonNodeGraph, PythonSqlRunLifecyclePlan from sqlbuild.executor.python_nodes._helpers.lifecycle_nodes import build_ingress_lifecycle_nodes from sqlbuild.executor.scheduling.models import LifecycleExecutionNode from tests.unit.src.sqlbuild.executor.python_nodes._helpers._test_types import ( diff --git a/tests/unit/src/sqlbuild/executor/python_nodes/_helpers/test_python_checks.py b/tests/unit/src/sqlbuild/executor/python_nodes/_helpers/test_python_checks.py index dc5da97064..c94552e596 100644 --- a/tests/unit/src/sqlbuild/executor/python_nodes/_helpers/test_python_checks.py +++ b/tests/unit/src/sqlbuild/executor/python_nodes/_helpers/test_python_checks.py @@ -11,12 +11,12 @@ from sqlbuild._native import ( CostContext, ExecutionEventWriter, + PythonNodeGraph, TerminalEventIndex, format_check_json, terminal_event_index_scope, ) from sqlbuild.compiler.discovery.models import DiscoveredCheckFunction -from sqlbuild.compiler.python_nodes.models import PythonNodeGraph from sqlbuild.compiler.python_nodes.types import PythonNodeKind, PythonNodeStatus, SkipMode from sqlbuild.cost.models import CostResourceContext from sqlbuild.executor.node_results.models import NodeResultEnvelope diff --git a/tests/unit/src/sqlbuild/executor/python_nodes/_helpers/test_read_side_execution.py b/tests/unit/src/sqlbuild/executor/python_nodes/_helpers/test_read_side_execution.py index be3f30a40b..a40b19a781 100644 --- a/tests/unit/src/sqlbuild/executor/python_nodes/_helpers/test_read_side_execution.py +++ b/tests/unit/src/sqlbuild/executor/python_nodes/_helpers/test_read_side_execution.py @@ -4,7 +4,7 @@ import pytest -from sqlbuild.compiler.python_nodes.models import PythonNodeGraph +from sqlbuild._native import PythonNodeGraph from sqlbuild.compiler.python_nodes.types import PythonNodeStatus from sqlbuild.executor.python_nodes.classes.read_side_python_execution_tracker import ( ReadSidePythonExecutionTracker, diff --git a/tests/unit/src/sqlbuild/executor/run/_helpers/_test_types.py b/tests/unit/src/sqlbuild/executor/run/_helpers/_test_types.py index da728214e4..6e604826fb 100644 --- a/tests/unit/src/sqlbuild/executor/run/_helpers/_test_types.py +++ b/tests/unit/src/sqlbuild/executor/run/_helpers/_test_types.py @@ -1,5 +1,6 @@ from dataclasses import dataclass, field +from sqlbuild._native import SchemaDynamicColumnFamily from sqlbuild.adapter.contract.models import ColumnInfo, LifeCycleEvent from sqlbuild.compiler.planner.types import PlanReason from sqlbuild.executor.run.types import ( @@ -9,7 +10,6 @@ RuntimeCursorWatermarkMode, ) from sqlbuild.python_nodes.models import SqlResourceRef -from sqlbuild.spec.contracts.models import SchemaDynamicColumnFamily @dataclass(frozen=True) diff --git a/tests/unit/src/sqlbuild/executor/run/_helpers/helpers.py b/tests/unit/src/sqlbuild/executor/run/_helpers/helpers.py index 4c6844f2fe..5e5fa60084 100644 --- a/tests/unit/src/sqlbuild/executor/run/_helpers/helpers.py +++ b/tests/unit/src/sqlbuild/executor/run/_helpers/helpers.py @@ -2,6 +2,7 @@ from pathlib import Path from typing import Any +from sqlbuild._native import SchemaDynamicColumnFamily, SourceEntry from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter from sqlbuild.adapter.contract.classes.statement_recorder import StatementRecorder from sqlbuild.adapter.contract.models import ColumnInfo @@ -28,7 +29,6 @@ from sqlbuild.executor.run.models import HookContext, HookRelationLookup, HookRunContext from sqlbuild.executor.run.types import HookPhase from sqlbuild.python_nodes.models import SqlResourceRef -from sqlbuild.spec.contracts.models import SchemaDynamicColumnFamily, SourceEntry def insert_snapshot_hook_log(ctx: HookContext, phase: str) -> None: diff --git a/tests/unit/src/sqlbuild/executor/run/_helpers/test_contracts.py b/tests/unit/src/sqlbuild/executor/run/_helpers/test_contracts.py index 63cefeb15a..01b98ffe99 100644 --- a/tests/unit/src/sqlbuild/executor/run/_helpers/test_contracts.py +++ b/tests/unit/src/sqlbuild/executor/run/_helpers/test_contracts.py @@ -2,11 +2,11 @@ import pytest +from sqlbuild._native import SchemaDynamicColumnFamily from sqlbuild.adapter.contract.models import ColumnInfo from sqlbuild.compiler.planner.models import ModelPlanEntry from sqlbuild.errors.contracts.exceptions import ExecutorInputError from sqlbuild.executor.run._helpers.validation.contracts import validate_runtime_contract -from sqlbuild.spec.contracts.models import SchemaDynamicColumnFamily from tests.unit.src.sqlbuild.executor.run._helpers._test_types import ( RuntimeContractValidationTestCase, ) diff --git a/tests/unit/src/sqlbuild/executor/run/_helpers/test_cursor_bounds.py b/tests/unit/src/sqlbuild/executor/run/_helpers/test_cursor_bounds.py index 2fd1c8da80..571316f70f 100644 --- a/tests/unit/src/sqlbuild/executor/run/_helpers/test_cursor_bounds.py +++ b/tests/unit/src/sqlbuild/executor/run/_helpers/test_cursor_bounds.py @@ -10,6 +10,9 @@ from sqlbuild._native import ( CursorWatermarkMode, + FutureCursorAction, + FutureCursorsConfig, + StartCursorsConfig, normalize_cursor_snapshot_grain, ) from sqlbuild._native import cursor_render as render @@ -41,8 +44,6 @@ from sqlbuild.executor.run.exceptions import EmptyCursorInputsError from sqlbuild.executor.run.models import RuntimeCursorInputRelation, RuntimeCursorSpec from sqlbuild.executor.run.types import RuntimeCursorWatermarkMode -from sqlbuild.spec.contracts.models import FutureCursorsConfig, StartCursorsConfig -from sqlbuild.spec.contracts.types import FutureCursorAction from tests.unit.src.sqlbuild.executor.run._helpers._test_types import ( AuthoritativeRuntimeCursorOverrideTestCase, CursorSentinelSubstitutionErrorTestCase, diff --git a/tests/unit/src/sqlbuild/executor/run/_helpers/test_fingerprint_metadata.py b/tests/unit/src/sqlbuild/executor/run/_helpers/test_fingerprint_metadata.py index 1132488185..d6a13fb914 100644 --- a/tests/unit/src/sqlbuild/executor/run/_helpers/test_fingerprint_metadata.py +++ b/tests/unit/src/sqlbuild/executor/run/_helpers/test_fingerprint_metadata.py @@ -9,9 +9,9 @@ import pytest +from sqlbuild._native import Fingerprint from sqlbuild.compiler.auditing.types import AuditOutcome, AuditSeverity from sqlbuild.compiler.fingerprints.constants import AUDIT_GATE_METADATA_KEY -from sqlbuild.compiler.fingerprints.models import Fingerprint from sqlbuild.compiler.planner.models import AuditPlanEntry, ModelPlanEntry from sqlbuild.executor.auditing.models import AuditExecutionResult from sqlbuild.executor.run._helpers.reuse import fingerprinting diff --git a/tests/unit/src/sqlbuild/executor/run/_helpers/test_snapshot_lifecycle.py b/tests/unit/src/sqlbuild/executor/run/_helpers/test_snapshot_lifecycle.py index aee7401e98..3675bd3b9b 100644 --- a/tests/unit/src/sqlbuild/executor/run/_helpers/test_snapshot_lifecycle.py +++ b/tests/unit/src/sqlbuild/executor/run/_helpers/test_snapshot_lifecycle.py @@ -7,6 +7,7 @@ import pytest +from sqlbuild._native import SnapshotsConfig from sqlbuild.adapter.contract.classes.statement_recorder import StatementRecorder from sqlbuild.adapter.contract.models import ColumnInfo from sqlbuild.adapters.duckdb.classes.duckdb_adapter import DuckDbAdapter @@ -22,7 +23,6 @@ from sqlbuild.executor.run.models import ModelExecutionResult, ModelMaterializationContext from sqlbuild.executor.run.types import ExecutionPhase from sqlbuild.executor.scheduling.types import ExecutionStatus -from sqlbuild.spec.contracts.models import SnapshotsConfig from tests.unit.src.sqlbuild.executor.run._helpers._test_types import ( SnapshotHookFailureTestCase, SnapshotLifecycleTestCase, diff --git a/tests/unit/src/sqlbuild/executor/scenario/main/helpers.py b/tests/unit/src/sqlbuild/executor/scenario/main/helpers.py index 8001e4cf88..312696eee7 100644 --- a/tests/unit/src/sqlbuild/executor/scenario/main/helpers.py +++ b/tests/unit/src/sqlbuild/executor/scenario/main/helpers.py @@ -6,6 +6,7 @@ from sqlbuild._native import ( ScenarioGraphPlan, + SeedCsvSettings, ) from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter from sqlbuild.adapter.contract.classes.statement_recorder import StatementRecorder @@ -30,7 +31,6 @@ ScenarioArtifactKind, ) from sqlbuild.spec.contracts.constants import DEFAULT_SEED_CSV_SETTINGS -from sqlbuild.spec.contracts.models import SeedCsvSettings class ScenarioFixtureTestAdapter(BaseAdapter): diff --git a/tests/unit/src/sqlbuild/executor/seed/main/helpers.py b/tests/unit/src/sqlbuild/executor/seed/main/helpers.py index d1e9753df4..840da74ff4 100644 --- a/tests/unit/src/sqlbuild/executor/seed/main/helpers.py +++ b/tests/unit/src/sqlbuild/executor/seed/main/helpers.py @@ -4,11 +4,11 @@ from pathlib import Path +from sqlbuild._native import SeedCsvSettings from sqlbuild.adapter.contract.models import ColumnInfo from sqlbuild.compiler.compile.models import CompiledObjectKey, CompiledRelationLocation from sqlbuild.compiler.compile.types import CompiledResourceType from sqlbuild.compiler.planner.models import SeedPlanEntry -from sqlbuild.spec.contracts.models import SeedCsvSettings def build_seed_plan_entry(*, seed_name: str, file_path: Path) -> SeedPlanEntry: diff --git a/tests/unit/src/sqlbuild/integrations/dagster/test_resource.py b/tests/unit/src/sqlbuild/integrations/dagster/test_resource.py index a80d262c6a..4b2195d857 100644 --- a/tests/unit/src/sqlbuild/integrations/dagster/test_resource.py +++ b/tests/unit/src/sqlbuild/integrations/dagster/test_resource.py @@ -12,7 +12,12 @@ import pytest from _pytest.capture import CaptureResult -from sqlbuild._native import IntegrationResultEnvelope, serialize_maximum_start_safety +from sqlbuild._native import ( + FutureCursorAction, + IntegrationResultEnvelope, + MicrobatchLimitAction, + serialize_maximum_start_safety, +) from sqlbuild.compiler.planner.models import MaximumStartSafetyEvidence from sqlbuild.integrations.dagster import ( SqlBuildCliResource, @@ -23,7 +28,6 @@ _build_results_from_integration_result, ) from sqlbuild.integrations.dagster.classes.sqlbuild_cli_invocation import SqlBuildCliInvocation -from sqlbuild.spec.contracts.types import FutureCursorAction, MicrobatchLimitAction from tests.unit.src.sqlbuild.integrations.dagster._test_types import ( DagsterAuditIdentityTestCase, DagsterCliCloneFailureTestCase, diff --git a/tests/unit/src/sqlbuild/integrations/dbt/_test_types.py b/tests/unit/src/sqlbuild/integrations/dbt/_test_types.py index ff5d42a301..0da4a83d31 100644 --- a/tests/unit/src/sqlbuild/integrations/dbt/_test_types.py +++ b/tests/unit/src/sqlbuild/integrations/dbt/_test_types.py @@ -3,8 +3,13 @@ from dataclasses import dataclass, field from pathlib import Path -from sqlbuild._native import DbtCliOptions, DbtCommandResult, DbtInteropParsedArgs -from sqlbuild.spec.contracts.models import DbtConfig, LocalDbtConfig +from sqlbuild._native import ( + DbtCliOptions, + DbtCommandResult, + DbtConfig, + DbtInteropParsedArgs, + LocalDbtConfig, +) @dataclass(frozen=True) diff --git a/tests/unit/src/sqlbuild/integrations/dbt/helpers.py b/tests/unit/src/sqlbuild/integrations/dbt/helpers.py index 5fbed29ce9..25ed4886b8 100644 --- a/tests/unit/src/sqlbuild/integrations/dbt/helpers.py +++ b/tests/unit/src/sqlbuild/integrations/dbt/helpers.py @@ -19,6 +19,9 @@ DbtEventStream, DbtEventStreamUpdate, DbtRunner, + LocalConfig, + ProjectConfig, + SqlLexicalSyntax, assemble_project, build_dbt_combined_graph, build_dbt_ls_argv, @@ -40,8 +43,6 @@ PlanAction, PlanReason, ) -from sqlbuild.compiler.sql_analysis.models import SqlLexicalSyntax -from sqlbuild.spec.contracts.models import LocalConfig, ProjectConfig _GENERIC_SQL_SYNTAX: SqlLexicalSyntax = SqlLexicalSyntax() diff --git a/tests/unit/src/sqlbuild/integrations/dbt/profile/helpers.py b/tests/unit/src/sqlbuild/integrations/dbt/profile/helpers.py index 644ba08b2d..3498a89fa3 100644 --- a/tests/unit/src/sqlbuild/integrations/dbt/profile/helpers.py +++ b/tests/unit/src/sqlbuild/integrations/dbt/profile/helpers.py @@ -2,7 +2,7 @@ from pathlib import Path -from sqlbuild.spec.contracts.models import DbtConfig, ProjectConfig +from sqlbuild._native import DbtConfig, ProjectConfig def write_duckdb_dbt_workspace(root: Path) -> None: diff --git a/tests/unit/src/sqlbuild/integrations/dbt/profile/test_normalize.py b/tests/unit/src/sqlbuild/integrations/dbt/profile/test_normalize.py index 0cf38d3fce..ba5ac6adaa 100644 --- a/tests/unit/src/sqlbuild/integrations/dbt/profile/test_normalize.py +++ b/tests/unit/src/sqlbuild/integrations/dbt/profile/test_normalize.py @@ -4,9 +4,12 @@ import pytest -from sqlbuild._native import NormalizedDbtProfileConnection, normalize_dbt_profile_connection +from sqlbuild._native import ( + NormalizedDbtProfileConnection, + ResolvedDbtProfileOutput, + normalize_dbt_profile_connection, +) from sqlbuild.integrations.dbt.exceptions import DbtProfileError -from sqlbuild.integrations.dbt.models import ResolvedDbtProfileOutput from tests.unit.src.sqlbuild.integrations.dbt.profile._test_types import ( DbtProfileNormalizeErrorTestCase, DbtProfileNormalizeTestCase, diff --git a/tests/unit/src/sqlbuild/integrations/dbt/profile/test_render.py b/tests/unit/src/sqlbuild/integrations/dbt/profile/test_render.py index 0e43ada08a..184fad2b22 100644 --- a/tests/unit/src/sqlbuild/integrations/dbt/profile/test_render.py +++ b/tests/unit/src/sqlbuild/integrations/dbt/profile/test_render.py @@ -4,8 +4,8 @@ import pytest +from sqlbuild._native import ResolvedDbtProfileOutput, SelectedDbtProfileOutput from sqlbuild.integrations.dbt._helpers.profile.render import render_selected_dbt_profile_output -from sqlbuild.integrations.dbt.models import ResolvedDbtProfileOutput, SelectedDbtProfileOutput from tests.unit.src.sqlbuild.integrations.dbt.profile._test_types import ( DbtProfileRenderErrorTestCase, DbtProfileRenderTestCase, diff --git a/tests/unit/src/sqlbuild/integrations/dbt/test_config.py b/tests/unit/src/sqlbuild/integrations/dbt/test_config.py index d08b928f99..438225374a 100644 --- a/tests/unit/src/sqlbuild/integrations/dbt/test_config.py +++ b/tests/unit/src/sqlbuild/integrations/dbt/test_config.py @@ -5,12 +5,13 @@ import pytest from sqlbuild._native import ( + DbtConfig, DbtInteropConfigError, + LocalDbtConfig, ResolvedDbtConfig, resolve_dbt_config, resolve_dbt_vars_mapping, ) -from sqlbuild.spec.contracts.models import DbtConfig, LocalDbtConfig from tests.unit.src.sqlbuild.integrations.dbt._test_types import ( DbtConfigErrorTestCase, DbtConfigResolutionTestCase, diff --git a/tests/unit/src/sqlbuild/integrations/ingestr/test_command.py b/tests/unit/src/sqlbuild/integrations/ingestr/test_command.py index 6aa6f7607c..edc33a8eb8 100644 --- a/tests/unit/src/sqlbuild/integrations/ingestr/test_command.py +++ b/tests/unit/src/sqlbuild/integrations/ingestr/test_command.py @@ -4,9 +4,9 @@ import pytest +from sqlbuild._native import IntegrationLoaderConfig, SourceEntry from sqlbuild.integrations.ingestr._helpers.command import build_ingestr_command from sqlbuild.integrations.ingestr.models import IngestrSourceConfig -from sqlbuild.spec.contracts.models import IntegrationLoaderConfig, SourceEntry from tests.unit.src.sqlbuild.integrations.ingestr._test_types import IngestrCommandTestCase diff --git a/tests/unit/src/sqlbuild/lint/_test_types.py b/tests/unit/src/sqlbuild/lint/_test_types.py index 0ce8cb5187..67f78c8c4b 100644 --- a/tests/unit/src/sqlbuild/lint/_test_types.py +++ b/tests/unit/src/sqlbuild/lint/_test_types.py @@ -4,7 +4,7 @@ from dataclasses import dataclass, field -from sqlbuild.compiler.sql_analysis.models import SqlLexicalSyntax +from sqlbuild._native import SqlLexicalSyntax @dataclass(frozen=True) diff --git a/tests/unit/src/sqlbuild/lint/test_adapter_lexical_syntax.py b/tests/unit/src/sqlbuild/lint/test_adapter_lexical_syntax.py index 1199b16010..d174e5775e 100644 --- a/tests/unit/src/sqlbuild/lint/test_adapter_lexical_syntax.py +++ b/tests/unit/src/sqlbuild/lint/test_adapter_lexical_syntax.py @@ -10,10 +10,9 @@ import sqlbuild._native as fixture_formatting_module import sqlbuild._native as native_module -from sqlbuild._native import discover_project_inputs +from sqlbuild._native import SqlLexicalSyntax, discover_project_inputs from sqlbuild.adapters.duckdb.classes.duckdb_adapter import DuckDbAdapter from sqlbuild.compiler.discovery.models import DiscoveredProjectInputs -from sqlbuild.compiler.sql_analysis.models import SqlLexicalSyntax from tests.unit.src.sqlbuild.lint._test_types import PassedAdapterSyntaxTestCase from tests.unit.src.sqlbuild.lint.helpers import write_fixture_format_project diff --git a/tests/unit/src/sqlbuild/rule_engine/classes/shared_fact_encoder/helpers.py b/tests/unit/src/sqlbuild/rule_engine/classes/shared_fact_encoder/helpers.py index ed377026b0..cb01e5fb78 100644 --- a/tests/unit/src/sqlbuild/rule_engine/classes/shared_fact_encoder/helpers.py +++ b/tests/unit/src/sqlbuild/rule_engine/classes/shared_fact_encoder/helpers.py @@ -2,6 +2,7 @@ from sqlbuild._native import ( CompiledSource, + SourceEntry, project_with, ) from sqlbuild.compiler.compile.models import CompiledObjectKey @@ -12,7 +13,6 @@ ) from sqlbuild.compiler.discovery.models import DiscoveredSourceFile from sqlbuild.rule_engine.classes.rule_context import RuleFactViews, build_rule_fact_views -from sqlbuild.spec.contracts.models import SourceEntry from tests.unit.src.sqlbuild.rule_engine.main.evaluate.helpers import build_project SOURCES_CONTENTS: str = "sources:\n - name: raw_orders\n - name: raw_customers\n" diff --git a/tests/unit/src/sqlbuild/runtime/event_exporting/classes/test_dispatcher.py b/tests/unit/src/sqlbuild/runtime/event_exporting/classes/test_dispatcher.py index d46ed5d96d..47964dcdd6 100644 --- a/tests/unit/src/sqlbuild/runtime/event_exporting/classes/test_dispatcher.py +++ b/tests/unit/src/sqlbuild/runtime/event_exporting/classes/test_dispatcher.py @@ -19,7 +19,6 @@ from sqlbuild.runtime.output_capture.constants import COMMAND_OUTPUT_LOSS_RECORD_TYPE from sqlbuild.runtime.output_capture.models import BoundCommandOutputSink from sqlbuild.sinks import CommandOutputRecord, CommandOutputStream -from sqlbuild.spec.contracts.types import EventExportSeverity from tests.unit.src.sqlbuild.runtime.event_exporting.classes._test_types import ( EventExporterDispatcherTestCase, HealthIntervalTestCase, diff --git a/tests/unit/src/sqlbuild/runtime/event_exporting/test_sinks.py b/tests/unit/src/sqlbuild/runtime/event_exporting/test_sinks.py index b7a1e8da10..9153782600 100644 --- a/tests/unit/src/sqlbuild/runtime/event_exporting/test_sinks.py +++ b/tests/unit/src/sqlbuild/runtime/event_exporting/test_sinks.py @@ -5,6 +5,7 @@ import pytest +from sqlbuild._native import EventExportSeverity from sqlbuild.runtime.event_exporting.exceptions import EventExporterInputError from sqlbuild.sinks import ( CommandOutputSinkDefinition, @@ -16,7 +17,6 @@ get_lifecycle_event_sink_definition, lifecycle_event_sink, ) -from sqlbuild.spec.contracts.types import EventExportSeverity from tests.unit.src.sqlbuild.runtime.event_exporting._test_types import SinkApiTestCase diff --git a/tests/unit/src/sqlbuild/spec/contracts/main/_test_types.py b/tests/unit/src/sqlbuild/spec/contracts/main/_test_types.py index a9cb661203..28a6afdfd6 100644 --- a/tests/unit/src/sqlbuild/spec/contracts/main/_test_types.py +++ b/tests/unit/src/sqlbuild/spec/contracts/main/_test_types.py @@ -2,17 +2,17 @@ from dataclasses import dataclass, field -from sqlbuild._native import CollectionRendering -from sqlbuild.spec.contracts.models import ( +from sqlbuild._native import ( AuthoredTimeTravelRetention, + CollectionRendering, ExecutionLimitsConfig, LoaderDestinationParts, LocalConfig, + MissingMigrationOriginPolicy, ProjectConfig, TargetConfig, TargetWarehousesConfig, ) -from sqlbuild.spec.contracts.types import MissingMigrationOriginPolicy @dataclass(frozen=True) diff --git a/tests/unit/src/sqlbuild/spec/contracts/main/test_constants.py b/tests/unit/src/sqlbuild/spec/contracts/main/test_constants.py index 641d236bfd..485c0c1cb3 100644 --- a/tests/unit/src/sqlbuild/spec/contracts/main/test_constants.py +++ b/tests/unit/src/sqlbuild/spec/contracts/main/test_constants.py @@ -2,8 +2,12 @@ import pytest -from sqlbuild._native import CollectionRendering, resolve_effective_collection_rendering -from sqlbuild.spec.contracts.models import ConstantsConfig, ProjectConfig +from sqlbuild._native import ( + CollectionRendering, + ConstantsConfig, + ProjectConfig, + resolve_effective_collection_rendering, +) from tests.unit.src.sqlbuild.spec.contracts.main._test_types import ( EffectiveCollectionRenderingResolutionTestCase, ) diff --git a/tests/unit/src/sqlbuild/spec/contracts/main/test_loader_destination_parts.py b/tests/unit/src/sqlbuild/spec/contracts/main/test_loader_destination_parts.py index 7001a5e9e4..f9bc291b6e 100644 --- a/tests/unit/src/sqlbuild/spec/contracts/main/test_loader_destination_parts.py +++ b/tests/unit/src/sqlbuild/spec/contracts/main/test_loader_destination_parts.py @@ -4,8 +4,7 @@ import pytest -from sqlbuild._native import SpecConfigError, loader_destination_parts -from sqlbuild.spec.contracts.models import LoaderDestinationParts +from sqlbuild._native import LoaderDestinationParts, SpecConfigError, loader_destination_parts from tests.unit.src.sqlbuild.spec.contracts.main._test_types import ( InvalidLoaderDestinationTestCase, LoaderDestinationPartsTestCase, diff --git a/tests/unit/src/sqlbuild/spec/contracts/main/test_targets.py b/tests/unit/src/sqlbuild/spec/contracts/main/test_targets.py index 88594ac3da..b1fc909c0f 100644 --- a/tests/unit/src/sqlbuild/spec/contracts/main/test_targets.py +++ b/tests/unit/src/sqlbuild/spec/contracts/main/test_targets.py @@ -2,23 +2,22 @@ import pytest -from sqlbuild._native import SpecConfigError, resolve_target_config -from sqlbuild.spec.contracts.models import ( +from sqlbuild._native import ( AuthoredTimeTravelRetention, ClonePolicy, ExecutionLimitsConfig, LocalConfig, LocalTargetConfig, - ProjectConfig, - TargetConfig, - TargetDiffConfig, - TargetWarehousesConfig, -) -from sqlbuild.spec.contracts.types import ( MissingMigrationOriginPolicy, + ProjectConfig, RetentionDecreasePolicy, + SpecConfigError, TableType, TableTypeDowngradePolicy, + TargetConfig, + TargetDiffConfig, + TargetWarehousesConfig, + resolve_target_config, ) from tests.unit.src.sqlbuild.spec.contracts.main._test_types import ( AmbiguousTargetConnectionTestCase, diff --git a/tests/unit/src/sqlbuild/sql_values/test_normalization.py b/tests/unit/src/sqlbuild/sql_values/test_normalization.py index 3796531f8b..2be94bc7e1 100644 --- a/tests/unit/src/sqlbuild/sql_values/test_normalization.py +++ b/tests/unit/src/sqlbuild/sql_values/test_normalization.py @@ -7,11 +7,11 @@ from sqlbuild._native import ( AuthoredSqlSet, + SqlValue, SqlValueLimits, SqlValueValidationError, normalize_sql_value, ) -from sqlbuild.sql_values.models import SqlValue from tests.unit.src.sqlbuild.sql_values._test_types import ( NormalizeSqlValueErrorTestCase, NormalizeSqlValueTestCase, diff --git a/tests/unit/src/sqlbuild/sql_values/test_state_literal.py b/tests/unit/src/sqlbuild/sql_values/test_state_literal.py index 7f6f3a5932..75d1b3124e 100644 --- a/tests/unit/src/sqlbuild/sql_values/test_state_literal.py +++ b/tests/unit/src/sqlbuild/sql_values/test_state_literal.py @@ -6,8 +6,7 @@ import pytest -from sqlbuild._native import render_state_sql_literal -from sqlbuild.sql_values.types import StateSqlValueType +from sqlbuild._native import StateSqlValueType, render_state_sql_literal from tests.unit.src.sqlbuild.sql_values._test_types import StateLiteralGoldenTestCase From d6454857e59976ea7bc433815316f04bc8d99e8b Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Thu, 15 Oct 2026 16:16:03 +0100 Subject: [PATCH 03/48] refactor: host planner, migration and plan entry shapes natively with native decisions and cursor normalization --- .../src/hosted_shapes/constants.rs | 1637 +++++++++++++++++ .../src/hosted_shapes/errors.rs | 5 + .../hosted_shapes/main/hosted_native_name.rs | 28 + .../src/hosted_shapes/main/mod.rs | 1 + .../sqlbuild-cache/src/hosted_shapes/mod.rs | 8 + .../sqlbuild-cache/src/hosted_shapes/tests.rs | 6 + .../src/hosted_shapes/tests/helpers.rs | 10 + .../tests/test_hosted_shape_types.rs | 63 + .../src/hosted_shapes/tests/test_types.rs | 11 + crates/sqlbuild-cache/src/lib.rs | 1 + .../src/stage_capture/constants.rs | 1377 -------------- .../sqlbuild-cache/src/stage_capture/tests.rs | 2 - .../src/stage_capture/tests/helpers.rs | 10 - .../tests/test_hosted_shape_types.rs | 33 - .../src/stage_capture/tests/test_types.rs | 6 - crates/sqlbuild-cli/src/dispatch/constants.rs | 2 +- crates/sqlbuild-core/src/lib.rs | 1 + .../src/migration_values/constants.rs | 30 + .../main/column_decision_traits.rs | 21 + .../main/migration_decision_traits.rs | 15 + .../src/migration_values/main/mod.rs | 5 + .../main/model_migration_blocks_build.rs | 12 + .../src/migration_values/main/plan_label.rs | 7 + .../main/storage_transition.rs | 17 + .../sqlbuild-core/src/migration_values/mod.rs | 8 + .../src/migration_values/models.rs | 35 + .../src/migration_values/tests.rs | 4 + .../tests/test_migration_values.rs | 146 ++ .../src/migration_values/tests/test_types.rs | 25 + .../analysis_session/model_analyses.rs | 2 +- .../_helpers/build_command/execution.rs | 2 +- .../_helpers/compile_command/target_writer.rs | 2 +- .../compile_driver/assembly_analysis.rs | 2 +- .../_helpers/compile_driver/deferral.rs | 2 +- .../bindings/_helpers/compile_driver/diff.rs | 2 +- .../_helpers/compile_driver/phases.rs | 2 +- .../compile_driver/pipeline_entries.rs | 2 +- .../_helpers/compile_driver/targets.rs | 2 +- .../_helpers/compiled_project/loop_classes.rs | 2 +- .../_helpers/compiler_enums/constants.rs | 258 +++ .../_helpers/compiler_errors/constants.rs | 78 + .../compiler_records/shape_classes.rs | 33 +- .../bindings/_helpers/cursor/conversion.rs | 2 +- .../_helpers/dbt_interop/plan_output.rs | 12 +- .../_helpers/format_command/selection.rs | 2 +- .../src/bindings/_helpers/frontier/encoder.rs | 6 +- .../_helpers/graph/python_selection.rs | 2 +- .../_helpers/lineage/lineage_command.rs | 2 +- .../bindings/_helpers/load/load_command.rs | 2 +- .../bindings/_helpers/plan_output/display.rs | 4 +- .../_helpers/planner_changes/results.rs | 6 +- .../_helpers/planner_entries/values.rs | 8 +- .../_helpers/planner_identity/graph.rs | 2 +- .../_helpers/planner_identity/hashing.rs | 2 +- .../planner_migrations/compatibility.rs | 2 +- .../_helpers/planner_migrations/constants.rs | 4 +- .../planner_migrations/model_entries.rs | 7 +- .../planner_migrations/planning_origin.rs | 7 +- .../_helpers/planner_models/records.rs | 15 +- .../_helpers/planner_phases/constants.rs | 6 +- .../_helpers/planner_records/constants.rs | 738 ++++++++ .../planner_resolution/cursor_bounds.rs | 2 +- .../planner_resolution/cursor_model.rs | 8 +- .../planner_resolution/cursor_overrides.rs | 4 +- .../planner_resolution/cursor_ranges.rs | 2 +- .../planner_resolution/cursor_safety.rs | 4 +- .../planner_resolution/cursor_windows.rs | 8 +- .../planner_resolution/orchestration.rs | 2 +- .../bindings/_helpers/planner_scenario/cli.rs | 6 +- .../_helpers/planner_scenario/values.rs | 10 +- .../bindings/_helpers/planner_scope/values.rs | 2 +- .../_helpers/planner_selection/carrier.rs | 2 +- .../_helpers/planner_strategy/constants.rs | 4 +- .../_helpers/planner_strategy/values.rs | 2 +- .../record_behaviours/call_arguments.rs | 18 + .../_helpers/record_behaviours/constants.rs | 117 ++ .../_helpers/record_behaviours/cursors.rs | 184 ++ .../_helpers/record_behaviours/migrations.rs | 210 +++ .../_helpers/record_behaviours/mod.rs | 4 + .../_helpers/record_behaviours/planner.rs | 311 ++++ .../bindings/_helpers/records/constants.rs | 8 + .../_helpers/records/error_classes.rs | 9 +- .../src/bindings/_helpers/records/fields.rs | 1 + .../src/bindings/_helpers/records/models.rs | 6 +- .../src/bindings/_helpers/records/types.rs | 3 + .../_helpers/refactor/offline_compile.rs | 2 +- .../_helpers/scenario_command/conversions.rs | 2 +- .../_helpers/semantic_shapes/shapes.rs | 2 +- .../_helpers/sql_test_command/test_command.rs | 2 +- .../_helpers/sql_test_glue/constants.rs | 4 +- .../_helpers/sql_test_glue/plan_driver.rs | 8 +- .../_helpers/sql_test_glue/plan_errors.rs | 8 +- .../_helpers/sql_test_glue/plan_rendering.rs | 8 +- .../bindings/_helpers/state_rows/constants.rs | 4 +- python-boundary.toml | 6 +- src/sqlbuild/_native.pyi | 1374 +++++++++++++- .../adapter/contract/classes/snapshot_sql.py | 2 +- .../_helpers/freshness/source_freshness.py | 3 +- src/sqlbuild/compiler/migrations/models.py | 155 -- src/sqlbuild/compiler/migrations/types.py | 154 -- .../pipeline/_helpers/materializations.py | 2 +- src/sqlbuild/compiler/pipeline/models.py | 4 +- src/sqlbuild/compiler/planner/constants.py | 8 +- src/sqlbuild/compiler/planner/exceptions.py | 33 - src/sqlbuild/compiler/planner/models.py | 1043 ----------- src/sqlbuild/compiler/planner/types.py | 142 -- src/sqlbuild/cursor_algebra/constants.py | 2 +- .../executor/auditing/_helpers/execution.py | 2 +- .../auditing/_helpers/result_projection.py | 9 +- .../executor/auditing/main/_execute.py | 3 +- .../main/_execute_reporting_errors.py | 3 +- .../auditing/main/_project_results.py | 2 +- .../main/audit_result_publication_scope.py | 2 +- .../executor/build/_helpers/end_audits.py | 3 +- .../build/_helpers/external_source_loads.py | 3 +- .../executor/build/_helpers/indexes.py | 2 +- .../executor/build/_helpers/retention.py | 24 +- .../executor/build/_helpers/scheduler.py | 12 +- .../executor/build/_helpers/source_audits.py | 3 +- .../executor/build/_helpers/source_node.py | 3 +- .../executor/build/classes/build_scheduler.py | 31 +- src/sqlbuild/executor/build/constants.py | 2 +- src/sqlbuild/executor/build/main/_execute.py | 4 +- .../build/main/_external_source_loads.py | 2 +- src/sqlbuild/executor/build/models.py | 12 +- src/sqlbuild/executor/build/types.py | 2 +- .../clone/_helpers/execution_items.py | 12 +- .../executor/clone/_helpers/fingerprinting.py | 5 +- .../executor/clone/_helpers/inspection.py | 2 +- .../executor/clone/_helpers/lifecycle.py | 8 +- .../executor/clone/_helpers/operations.py | 2 +- .../executor/clone/_helpers/retention.py | 3 +- .../executor/clone/main/_fingerprinting.py | 2 +- src/sqlbuild/executor/clone/main/execute.py | 7 +- src/sqlbuild/executor/clone/models.py | 7 +- src/sqlbuild/executor/custom/models.py | 2 +- src/sqlbuild/executor/diff/_helpers/bounds.py | 3 +- .../executor/functions/main/_execute.py | 2 +- src/sqlbuild/executor/janitor/models.py | 3 +- src/sqlbuild/executor/load/_helpers/schema.py | 3 +- .../executor/load/_helpers/staging.py | 3 +- .../executor/migrations/_helpers/columns.py | 12 +- .../executor/migrations/_helpers/naming.py | 1 - .../migrations/_helpers/old_name_facts.py | 7 +- .../migrations/_helpers/old_name_views.py | 7 - .../executor/migrations/_helpers/promotion.py | 15 +- .../_helpers/relation_for_location.py | 2 +- .../executor/migrations/_helpers/staging.py | 2 +- .../executor/migrations/main/_apply.py | 10 +- .../migrations/main/_apply_columns.py | 2 +- .../main/_bound_old_name_view_guard.py | 2 +- .../main/_compatibility_view_sql.py | 2 +- .../main/_complete_old_name_views.py | 2 +- .../old_name_views/_helpers/janitor.py | 5 + .../executor/pipeline/_helpers/auditing.py | 3 +- .../executor/pipeline/_helpers/connections.py | 3 +- .../executor/pipeline/_helpers/graph_width.py | 2 +- .../executor/pipeline/_helpers/scenario.py | 2 +- .../pipeline/_helpers/schema_preparation.py | 3 +- .../executor/pipeline/_helpers/seeding.py | 2 +- .../executor/pipeline/_helpers/testing.py | 3 +- src/sqlbuild/executor/pipeline/main/run.py | 3 +- src/sqlbuild/executor/pipeline/models.py | 2 +- .../run/_helpers/execution/final_audits.py | 3 +- .../run/_helpers/execution/hook_phases.py | 2 +- .../run/_helpers/execution/results.py | 9 +- .../run/_helpers/execution/table_targets.py | 3 +- .../run/_helpers/materializations/custom.py | 4 +- .../_helpers/materializations/incremental.py | 6 +- .../_helpers/materializations/microbatch.py | 22 +- .../run/_helpers/materializations/snapshot.py | 19 +- .../_helpers/materializations/table_cursor.py | 3 +- .../run/_helpers/materializations/view.py | 3 +- .../executor/run/_helpers/reuse/audit.py | 3 +- .../_helpers/reuse/fingerprint_metadata.py | 8 +- .../run/_helpers/reuse/fingerprinting.py | 9 +- .../run/_helpers/validation/contracts.py | 3 +- .../run/_helpers/validation/cursor_bounds.py | 28 +- src/sqlbuild/executor/run/main/_execute.py | 3 +- .../executor/run/main/_table_targets.py | 2 +- src/sqlbuild/executor/run/models.py | 17 +- .../scenario/_helpers/capture/core.py | 2 +- .../scenario/_helpers/execution/interrupts.py | 2 +- .../_helpers/execution/model_execution.py | 2 +- .../scenario/_helpers/execution/run.py | 2 +- .../scenario/_helpers/lifecycle/cleanup.py | 15 +- .../_helpers/lifecycle/expectations.py | 4 +- .../scenario/_helpers/lifecycle/fixtures.py | 2 +- .../scenario/_helpers/local/execution.py | 19 +- .../scenario/_helpers/local/snapshots.py | 3 +- .../scenario/_helpers/snapshots/core.py | 7 +- .../classes/prepared_scenario_schemas.py | 2 +- .../scenario/classes/running_scenarios.py | 2 +- .../scenario/main/_assertion_expectations.py | 2 +- .../executor/scenario/main/_capture_steps.py | 2 +- .../executor/scenario/main/_cleanup.py | 3 +- .../scenario/main/_cleanup_interrupted.py | 2 +- .../executor/scenario/main/_execute.py | 3 +- .../scenario/main/_expected_expectations.py | 3 +- .../executor/scenario/main/_fixtures.py | 2 +- src/sqlbuild/executor/scenario/main/_local.py | 2 +- src/sqlbuild/executor/scenario/main/_run.py | 2 +- .../executor/scenario/main/_snapshots.py | 2 +- src/sqlbuild/executor/scenario/models.py | 13 +- .../executor/seed/_helpers/fingerprinting.py | 3 +- src/sqlbuild/executor/seed/main/_execute.py | 2 +- .../testing/_helpers/difference_samples.py | 3 +- .../testing/_helpers/expected_column_probe.py | 7 +- .../executor/testing/main/_execute.py | 3 +- src/sqlbuild/integrations/rivers/constants.py | 2 +- .../microbatches/classes/direct_store.py | 2 +- .../cli/commands/main/scenario/helpers.py | 2 +- .../snowflake/test_table_type_conversion.py | 3 +- .../sqlbuild/cli/commands/main/_test_types.py | 2 +- .../src/sqlbuild/cli/commands/main/helpers.py | 2 +- .../commands/main/old_name_views/helpers.py | 3 +- .../cli/commands/main/test_load_command.py | 4 +- .../main/test_startup_import_footprint.py | 2 - .../sqlbuild/compiler/contracts/helpers.py | 4 +- .../sqlbuild/compiler/pipeline/test_main.py | 2 +- .../pipeline/test_measurement_audits.py | 2 +- .../compiler/planner/_helpers/_test_types.py | 4 +- .../planner/_helpers/resolve/helpers.py | 6 +- .../_helpers/resolve/test_cursor_planning.py | 2 +- .../_helpers/resolve/test_orchestration.py | 2 +- .../planner/_helpers/resolve/test_resolve.py | 8 +- .../sql_test_assembly/test_execute_chain.py | 3 +- .../planner/_helpers/test_audit_entry.py | 9 +- .../planner/_helpers/test_fixture_planning.py | 2 +- .../_helpers/test_warehouse_cursor_reads.py | 6 +- .../_helpers/test_warehouse_snapshot.py | 8 +- .../compiler/planner/main/_test_types.py | 6 +- .../sqlbuild/compiler/planner/main/helpers.py | 2 +- .../planner/main/test_build_execution_plan.py | 18 +- .../main/test_snowflake_inspection_budget.py | 2 +- .../executor/build/concurrent/helpers.py | 8 +- .../concurrent/test_ordering_invariant.py | 8 +- .../executor/build/custom/_test_types.py | 3 +- .../sqlbuild/executor/build/custom/helpers.py | 11 +- .../test_advanced_custom_materialization.py | 3 +- .../custom/test_custom_materialization.py | 3 +- .../src/sqlbuild/executor/build/helpers.py | 3 +- .../executor/build/test_snapshot_execution.py | 9 +- .../executor/migrations/_helpers/helpers.py | 9 +- .../src/sqlbuild/executor/run/helpers.py | 13 +- .../executor/run/incremental/_test_types.py | 3 +- .../executor/run/incremental/helpers.py | 24 +- .../incremental/test_incremental_execution.py | 3 +- .../executor/run/microbatch/_test_types.py | 3 +- .../executor/run/microbatch/helpers.py | 24 +- .../microbatch/test_microbatch_execution.py | 8 +- .../test_microbatch_reconciliation.py | 12 +- .../executor/run/test_audit_helpers.py | 2 +- .../src/sqlbuild/executor/run/view/helpers.py | 13 +- .../executor/run/view/test_audit_helpers.py | 2 +- .../src/sqlbuild/executor/scenario/helpers.py | 24 +- .../executor/scenario/test_capture_types.py | 2 +- .../scenario/test_fixture_lifecycle.py | 6 +- .../src/sqlbuild/executor/testing/helpers.py | 3 +- .../testing/test_sql_test_build_parity.py | 3 +- .../testing/test_sql_test_execution.py | 8 +- .../sqlserver/test_migration_state_sql.py | 5 +- .../_helpers/build_planning/_test_types.py | 3 +- .../_helpers/build_planning/helpers.py | 20 +- .../build_planning/test_full_refresh.py | 2 +- .../build_planning/test_plan_policies.py | 2 +- .../build_planning/test_retention_decrease.py | 4 +- .../build_planning/test_table_type.py | 2 +- .../commands/_helpers/check/test_planning.py | 2 +- .../sqlbuild/cli/commands/_helpers/helpers.py | 8 +- .../_helpers/test_source_freshness.py | 2 +- .../cli/commands/main/compile/helpers.py | 20 +- .../compile/test_compile_target_writer.py | 4 +- .../cli/commands/main/entry/test_main.py | 2 +- .../commands/shared/_helpers/_test_types.py | 2 +- .../cli/commands/shared/_helpers/helpers.py | 13 +- .../commands/shared/_helpers/test_progress.py | 2 +- .../test_execution_result_document.py | 8 +- .../test_execution_protocol_v1.py | 3 +- .../cli/output/main/plan/_test_types.py | 6 +- .../sqlbuild/cli/output/main/plan/helpers.py | 30 +- .../cli/output/main/plan/test_plan_text.py | 18 +- .../cli/output/main/plan_json/_test_types.py | 3 +- .../output/main/plan_json/test_plan_json.py | 24 +- .../classes/test_build_sql_test_projection.py | 4 +- .../test_runtime_target_writer.py | 3 +- .../compiler/auditing/main/helpers.py | 3 +- .../compiler/auditing/main/test_identity.py | 2 +- .../_helpers/test_microbatch_redesign.py | 4 +- .../sqlbuild/compiler/manifest/_test_types.py | 2 +- .../src/sqlbuild/compiler/manifest/helpers.py | 22 +- .../sqlbuild/compiler/manifest/test_main.py | 5 +- .../compiler/migrations/main/_test_types.py | 3 +- .../compiler/migrations/main/helpers.py | 16 +- .../main/test_column_migration_events.py | 2 +- .../migrations/main/test_migration_events.py | 2 +- .../main/test_old_name_view_events.py | 10 +- .../migrations/main/test_pinned_event_ids.py | 4 +- .../migrations/main/test_stored_event_rows.py | 10 +- .../compiler/pipeline/main/_test_types.py | 3 +- .../compiler/pipeline/main/helpers.py | 12 +- .../main/test_compile_phase_timings.py | 2 +- .../compiler/pipeline/main/test_plan_work.py | 2 +- .../pipeline/main/test_relation_targets.py | 2 +- .../compiler/planner/_helpers/_test_types.py | 22 +- .../planner/_helpers/changes/_test_types.py | 12 +- .../planner/_helpers/changes/test_detect.py | 8 +- .../planner/_helpers/changes/test_schema.py | 2 +- .../compiler/planner/_helpers/helpers.py | 12 +- .../planner/_helpers/identity/_test_types.py | 4 +- .../planner/_helpers/identity/helpers.py | 4 +- .../identity/test_definition_hashes.py | 3 +- .../test_merge_recomputed_model_changes.py | 4 +- .../planner/_helpers/migrations/helpers.py | 4 +- .../test_column_migration_candidates.py | 2 +- .../planner/_helpers/planning/_test_types.py | 3 +- .../_helpers/planning/test_full_refresh.py | 6 +- .../_helpers/planning/test_retention.py | 13 +- .../planner/_helpers/resolve/_test_types.py | 3 +- .../planner/_helpers/resolve/test_cursor.py | 9 +- .../resolve/test_cursor_intrinsics.py | 8 +- .../resolve/test_cursor_override_types.py | 4 +- .../planner/_helpers/resolve/test_refs.py | 2 +- .../planner/_helpers/resolve/test_sources.py | 3 +- .../selection/test_unit_test_selectors.py | 2 +- .../_helpers/sql_test_assembly/helpers.py | 4 +- .../sql_test_assembly/test_plan_test.py | 6 +- .../planner/_helpers/test_audit_entry.py | 3 +- .../planner/_helpers/test_audit_scheduling.py | 2 +- .../test_cursor_override_resolution.py | 3 +- .../test_cursor_overrides_validation.py | 2 +- .../planner/_helpers/test_cursor_start.py | 3 +- .../_helpers/test_fixture_planning_work.py | 2 +- .../_helpers/test_function_fingerprints.py | 2 +- .../planner/_helpers/test_microbatch_limit.py | 18 +- .../_helpers/test_microbatch_lookback.py | 3 +- .../test_plan_entry_cursor_overrides.py | 10 +- .../_helpers/test_scenario_artifacts.py | 6 +- .../planner/_helpers/test_scenario_cli.py | 4 +- .../_helpers/test_scenario_relations.py | 10 +- .../planner/_helpers/test_source_columns.py | 4 +- .../_helpers/test_source_load_nodes.py | 2 +- .../planner/_helpers/test_strategy_action.py | 12 +- .../_helpers/test_strategy_schema_actions.py | 8 +- .../_helpers/test_strategy_warnings.py | 12 +- .../_helpers/warehouse/test_source_tables.py | 3 +- .../planner/classes/test_sql_test_planning.py | 2 +- .../compiler/planner/main/_test_types.py | 4 +- .../planner/main/execution/_test_types.py | 2 +- .../main/execution/test_display_plan.py | 12 +- .../sqlbuild/compiler/planner/main/helpers.py | 2 +- .../planner/main/pre_build/_test_types.py | 8 +- .../planner/main/pre_build/helpers.py | 16 +- .../pre_build/test_pre_build_work_display.py | 5 +- .../planner/main/test_advance_cursor_end.py | 2 +- .../planner/main/test_cursor_bound_display.py | 3 +- .../test_effective_microbatch_batch_size.py | 3 +- .../compiler/planner/main/test_execution.py | 14 +- .../planner/main/test_future_cursor_safety.py | 13 +- .../planner/main/test_inclusive_cursor_end.py | 3 +- .../planner/main/test_maximum_start_safety.py | 10 +- .../main/test_static_command_planning.py | 5 +- .../sqlbuild/compiler/planner/test_models.py | 8 +- .../cursor_algebra/main/_test_types.py | 2 +- .../main/test_cursor_algebra.py | 3 +- .../executor/auditing/main/helpers.py | 3 +- .../main/test_execute_reporting_errors.py | 2 +- .../auditing/main/test_project_results.py | 2 +- .../executor/build/_helpers/_test_types.py | 7 +- .../executor/build/_helpers/helpers.py | 25 +- .../_helpers/test_external_source_loads.py | 3 +- .../executor/build/_helpers/test_indexes.py | 2 +- .../executor/build/_helpers/test_retention.py | 24 +- .../executor/build/_helpers/test_scheduler.py | 13 +- .../build/_helpers/test_source_node.py | 2 +- .../test_build_scheduler_test_order.py | 2 +- .../executor/clone/_helpers/_test_types.py | 2 +- .../clone/_helpers/fingerprinting/helpers.py | 13 +- .../fingerprinting/test_fingerprinting.py | 10 +- .../executor/clone/_helpers/helpers.py | 7 +- .../clone/_helpers/test_operations.py | 2 +- .../executor/clone/_helpers/test_retention.py | 2 +- .../executor/clone/main/test_execute.py | 3 +- .../executor/migrations/_helpers/helpers.py | 8 +- .../migrations/_helpers/test_columns.py | 3 +- .../_helpers/test_old_name_views.py | 2 +- .../executor/pipeline/_helpers/helpers.py | 14 +- .../pipeline/_helpers/test_auditing.py | 3 +- .../pipeline/_helpers/test_connections.py | 2 +- .../pipeline/_helpers/test_scenario.py | 3 +- .../pipeline/_helpers/test_seeding.py | 3 +- .../pipeline/_helpers/test_testing.py | 2 +- .../executor/pipeline/main/helpers.py | 18 +- .../executor/pipeline/main/test_identity.py | 3 +- .../pipeline/main/test_schema_preflight.py | 3 +- .../executor/run/_helpers/_test_types.py | 3 +- .../sqlbuild/executor/run/_helpers/helpers.py | 12 +- .../executor/run/_helpers/test_contracts.py | 3 +- .../run/_helpers/test_cursor_bounds.py | 24 +- .../run/_helpers/test_custom_query_change.py | 2 +- .../run/_helpers/test_fingerprint_metadata.py | 3 +- .../executor/run/_helpers/test_hooks.py | 3 +- .../test_inclusive_end_batch_windows.py | 2 +- .../run/_helpers/test_lifecycle_progress.py | 2 +- .../executor/run/_helpers/test_reuse_audit.py | 2 +- .../test_snapshot_adapter_boundary.py | 10 +- .../run/_helpers/test_snapshot_lifecycle.py | 3 +- .../executor/scenario/_helpers/_test_types.py | 3 +- .../executor/scenario/_helpers/helpers.py | 22 +- .../scenario/_helpers/test_capture.py | 3 +- .../scenario/_helpers/test_failures.py | 2 +- .../scenario/_helpers/test_snapshots.py | 3 +- .../executor/scenario/main/helpers.py | 24 +- .../executor/scenario/main/test_capture.py | 3 +- .../executor/scenario/main/test_cleanup.py | 2 +- .../executor/scenario/main/test_fixtures.py | 3 +- .../executor/scenario/main/test_models.py | 3 +- .../sqlbuild/executor/seed/main/helpers.py | 3 +- .../sqlbuild/executor/testing/main/helpers.py | 3 +- .../testing/main/test_comparison_sql.py | 3 +- .../integrations/dagster/test_assets.py | 2 +- .../integrations/dagster/test_resource.py | 2 +- .../src/sqlbuild/integrations/dbt/helpers.py | 12 +- .../src/sqlbuild/microbatches/test_sql.py | 2 +- 424 files changed, 6347 insertions(+), 4169 deletions(-) create mode 100644 crates/sqlbuild-cache/src/hosted_shapes/constants.rs create mode 100644 crates/sqlbuild-cache/src/hosted_shapes/errors.rs create mode 100644 crates/sqlbuild-cache/src/hosted_shapes/main/hosted_native_name.rs create mode 100644 crates/sqlbuild-cache/src/hosted_shapes/main/mod.rs create mode 100644 crates/sqlbuild-cache/src/hosted_shapes/mod.rs create mode 100644 crates/sqlbuild-cache/src/hosted_shapes/tests.rs create mode 100644 crates/sqlbuild-cache/src/hosted_shapes/tests/helpers.rs create mode 100644 crates/sqlbuild-cache/src/hosted_shapes/tests/test_hosted_shape_types.rs create mode 100644 crates/sqlbuild-cache/src/hosted_shapes/tests/test_types.rs delete mode 100644 crates/sqlbuild-cache/src/stage_capture/tests/test_hosted_shape_types.rs create mode 100644 crates/sqlbuild-core/src/migration_values/constants.rs create mode 100644 crates/sqlbuild-core/src/migration_values/main/column_decision_traits.rs create mode 100644 crates/sqlbuild-core/src/migration_values/main/migration_decision_traits.rs create mode 100644 crates/sqlbuild-core/src/migration_values/main/mod.rs create mode 100644 crates/sqlbuild-core/src/migration_values/main/model_migration_blocks_build.rs create mode 100644 crates/sqlbuild-core/src/migration_values/main/plan_label.rs create mode 100644 crates/sqlbuild-core/src/migration_values/main/storage_transition.rs create mode 100644 crates/sqlbuild-core/src/migration_values/mod.rs create mode 100644 crates/sqlbuild-core/src/migration_values/models.rs create mode 100644 crates/sqlbuild-core/src/migration_values/tests.rs create mode 100644 crates/sqlbuild-core/src/migration_values/tests/test_migration_values.rs create mode 100644 crates/sqlbuild-core/src/migration_values/tests/test_types.rs create mode 100644 crates/sqlbuild-python/src/bindings/_helpers/record_behaviours/call_arguments.rs create mode 100644 crates/sqlbuild-python/src/bindings/_helpers/record_behaviours/cursors.rs create mode 100644 crates/sqlbuild-python/src/bindings/_helpers/record_behaviours/migrations.rs create mode 100644 crates/sqlbuild-python/src/bindings/_helpers/record_behaviours/planner.rs delete mode 100644 src/sqlbuild/compiler/migrations/models.py delete mode 100644 src/sqlbuild/compiler/migrations/types.py delete mode 100644 src/sqlbuild/compiler/planner/exceptions.py delete mode 100644 src/sqlbuild/compiler/planner/models.py delete mode 100644 src/sqlbuild/compiler/planner/types.py diff --git a/crates/sqlbuild-cache/src/hosted_shapes/constants.rs b/crates/sqlbuild-cache/src/hosted_shapes/constants.rs new file mode 100644 index 0000000000..451e84c100 --- /dev/null +++ b/crates/sqlbuild-cache/src/hosted_shapes/constants.rs @@ -0,0 +1,1637 @@ +//! The Python shape names of the natively hosted compiler shapes. + +/// The extension module hosting the shapes. +pub const NATIVE_MODULE: &str = "sqlbuild._native"; +/// The Python model type a natively hosted compiler shape is captured as. +pub const HOSTED_SHAPE_TYPES: [(&str, &str); 409] = [ + ( + "sqlbuild._native:AlignedInterval", + "sqlbuild.cursor_algebra.models:AlignedInterval", + ), + ( + "sqlbuild._native:AnalysisCacheContext", + "sqlbuild.compiler.compile.models:AnalysisCacheContext", + ), + ( + "sqlbuild._native:AttachedAuditGateEdge", + "sqlbuild.compiler.graph.models:AttachedAuditGateEdge", + ), + ( + "sqlbuild._native:AuditGateIdentity", + "sqlbuild.compiler.auditing.models:AuditGateIdentity", + ), + ( + "sqlbuild._native:AuditIdentity", + "sqlbuild.compiler.auditing.models:AuditIdentity", + ), + ( + "sqlbuild._native:AuditPlanEntry", + "sqlbuild.compiler.planner.models:AuditPlanEntry", + ), + ( + "sqlbuild._native:AuthoredTimeTravelRetention", + "sqlbuild.spec.contracts.models:AuthoredTimeTravelRetention", + ), + ( + "sqlbuild._native:BackfillAction", + "sqlbuild.compiler.planner.types:BackfillAction", + ), + ( + "sqlbuild._native:ChainStep", + "sqlbuild.compiler.planner.models:ChainStep", + ), + ( + "sqlbuild._native:ClonePipelineConnection", + "sqlbuild.compiler.pipeline.models:ClonePipelineConnection", + ), + ( + "sqlbuild._native:ClonePipelineOptions", + "sqlbuild.compiler.pipeline.models:ClonePipelineOptions", + ), + ( + "sqlbuild._native:ClonePipelineResult", + "sqlbuild.compiler.pipeline.models:ClonePipelineResult", + ), + ( + "sqlbuild._native:ClonePolicy", + "sqlbuild.spec.contracts.models:ClonePolicy", + ), + ( + "sqlbuild._native:CloneSourcePlanEntry", + "sqlbuild.compiler.planner.models:CloneSourcePlanEntry", + ), + ( + "sqlbuild._native:CollectionRendering", + "sqlbuild.sql_values.types:CollectionRendering", + ), + ( + "sqlbuild._native:ColumnContractMode", + "sqlbuild.spec.contracts.types:ColumnContractMode", + ), + ( + "sqlbuild._native:ColumnLineage", + "sqlbuild.compiler.lineage.models:ColumnLineage", + ), + ( + "sqlbuild._native:ColumnLineageConfidence", + "sqlbuild.compiler.lineage.types:ColumnLineageConfidence", + ), + ( + "sqlbuild._native:ColumnLineageEdge", + "sqlbuild.compiler.lineage.models:ColumnLineageEdge", + ), + ( + "sqlbuild._native:ColumnLineageMode", + "sqlbuild.compiler.lineage.types:ColumnLineageMode", + ), + ( + "sqlbuild._native:ColumnLineageSource", + "sqlbuild.compiler.lineage.models:ColumnLineageSource", + ), + ( + "sqlbuild._native:ColumnMigrationDecision", + "sqlbuild.compiler.migrations.types:ColumnMigrationDecision", + ), + ( + "sqlbuild._native:ColumnMigrationEvent", + "sqlbuild.compiler.migrations.models:ColumnMigrationEvent", + ), + ( + "sqlbuild._native:ColumnMigrationPlanEntry", + "sqlbuild.compiler.planner.models:ColumnMigrationPlanEntry", + ), + ( + "sqlbuild._native:ColumnTransformKind", + "sqlbuild.compiler.lineage.types:ColumnTransformKind", + ), + ( + "sqlbuild._native:CompileAdapterContext", + "sqlbuild.compiler.compile.models:CompileAdapterContext", + ), + ( + "sqlbuild._native:CompileAnalysisSelection", + "sqlbuild.compiler.compile.models:CompileAnalysisSelection", + ), + ( + "sqlbuild._native:CompileDirectLogicSqlTestInputPayload", + "sqlbuild.compiler.compile.models:CompileDirectLogicSqlTestInputPayload", + ), + ( + "sqlbuild._native:CompileFacts", + "sqlbuild.lint.models:CompileFacts", + ), + ( + "sqlbuild._native:CompileModelConfig", + "sqlbuild.compiler.compile.models:CompileModelConfig", + ), + ( + "sqlbuild._native:CompileModelSqlTestInputPayload", + "sqlbuild.compiler.compile.models:CompileModelSqlTestInputPayload", + ), + ( + "sqlbuild._native:CompilePipelineOptions", + "sqlbuild.compiler.pipeline.models:CompilePipelineOptions", + ), + ( + "sqlbuild._native:CompileProjectInputs", + "sqlbuild.compiler.compile.models:CompileProjectInputs", + ), + ( + "sqlbuild._native:CompileSqlScenarioCte", + "sqlbuild.compiler.compile.models:CompileSqlScenarioCte", + ), + ( + "sqlbuild._native:CompileSqlScenarioInput", + "sqlbuild.compiler.compile.models:CompileSqlScenarioInput", + ), + ( + "sqlbuild._native:CompileSqlTestCte", + "sqlbuild.compiler.compile.models:CompileSqlTestCte", + ), + ( + "sqlbuild._native:CompileSqlTestInput", + "sqlbuild.compiler.compile.models:CompileSqlTestInput", + ), + ( + "sqlbuild._native:CompiledDirectLogicSqlTestPayload", + "sqlbuild.compiler.compile.models:CompiledDirectLogicSqlTestPayload", + ), + ( + "sqlbuild._native:CompiledFunction", + "sqlbuild.compiler.compile.models:CompiledFunction", + ), + ( + "sqlbuild._native:CompiledLineageColumnFact", + "sqlbuild.compiler.compile.models:CompiledLineageColumnFact", + ), + ( + "sqlbuild._native:CompiledModelSqlTestPayload", + "sqlbuild.compiler.compile.models:CompiledModelSqlTestPayload", + ), + ( + "sqlbuild._native:CompiledProjectPhaseResult", + "sqlbuild.compiler.pipeline.models:CompiledProjectPhaseResult", + ), + ( + "sqlbuild._native:CompiledSeed", + "sqlbuild.compiler.compile.models:CompiledSeed", + ), + ( + "sqlbuild._native:CompiledSource", + "sqlbuild.compiler.compile.models:CompiledSource", + ), + ( + "sqlbuild._native:CompiledSqlExpansion", + "sqlbuild.compiler.compile.models:CompiledSqlExpansion", + ), + ( + "sqlbuild._native:CompiledSqlTestResource", + "sqlbuild.compiler.compile.models:CompiledSqlTestResource", + ), + ( + "sqlbuild._native:CompilerStage", + "sqlbuild.compiler.frontier.types:CompilerStage", + ), + ( + "sqlbuild._native:ConfigValueTypeError", + "sqlbuild.spec.contracts.exceptions:ConfigValueTypeError", + ), + ( + "sqlbuild._native:ConstantMetadata", + "sqlbuild.compiler.scopes.models:ConstantMetadata", + ), + ( + "sqlbuild._native:ConstantsConfig", + "sqlbuild.spec.contracts.models:ConstantsConfig", + ), + ( + "sqlbuild._native:ContractAdoptionError", + "sqlbuild.compiler.contract_adoption.exceptions:ContractAdoptionError", + ), + ( + "sqlbuild._native:ContractAdoptionResult", + "sqlbuild.compiler.contract_adoption.models:ContractAdoptionResult", + ), + ( + "sqlbuild._native:ContractEvidence", + "sqlbuild.compiler.contract_adoption.models:ContractEvidence", + ), + ( + "sqlbuild._native:ContractFinding", + "sqlbuild.compiler.contract_adoption.models:ContractFinding", + ), + ( + "sqlbuild._native:ContractFindingKind", + "sqlbuild.compiler.contract_adoption.types:ContractFindingKind", + ), + ( + "sqlbuild._native:ContractPolicy", + "sqlbuild.compiler.planner.types:ContractPolicy", + ), + ( + "sqlbuild._native:ContractValidationResult", + "sqlbuild.compiler.contracts.models:ContractValidationResult", + ), + ( + "sqlbuild._native:CostConfig", + "sqlbuild.spec.contracts.models:CostConfig", + ), + ( + "sqlbuild._native:CursorBounds", + "sqlbuild.compiler.planner.models:CursorBounds", + ), + ( + "sqlbuild._native:CursorGrain", + "sqlbuild.compiler.planner.types:CursorGrain", + ), + ( + "sqlbuild._native:CursorInputEvidence", + "sqlbuild.compiler.planner.models:CursorInputEvidence", + ), + ( + "sqlbuild._native:CursorInputRelation", + "sqlbuild.compiler.planner.models:CursorInputRelation", + ), + ( + "sqlbuild._native:CursorInputRoles", + "sqlbuild.compiler.compile.models:CursorInputRoles", + ), + ( + "sqlbuild._native:CursorOverrides", + "sqlbuild.compiler.planner.models:CursorOverrides", + ), + ( + "sqlbuild._native:CursorType", + "sqlbuild.compiler.planner.types:CursorType", + ), + ( + "sqlbuild._native:CursorsConfig", + "sqlbuild.spec.contracts.models:CursorsConfig", + ), + ( + "sqlbuild._native:CustomRulesOutcome", + "sqlbuild.rule_engine.models:CustomRulesOutcome", + ), + ( + "sqlbuild._native:DateValue", + "sqlbuild.cursor_algebra.models:DateValue", + ), + ( + "sqlbuild._native:DbtCliOptions", + "sqlbuild.integrations.dbt.models:DbtCliOptions", + ), + ( + "sqlbuild._native:DbtCommandResult", + "sqlbuild.integrations.dbt.models:DbtCommandResult", + ), + ( + "sqlbuild._native:DbtConfig", + "sqlbuild.spec.contracts.models:DbtConfig", + ), + ( + "sqlbuild._native:DbtInitProgressCallbacks", + "sqlbuild.integrations.dbt.models:DbtInitProgressCallbacks", + ), + ( + "sqlbuild._native:DbtInitRequest", + "sqlbuild.integrations.dbt.models:DbtInitRequest", + ), + ( + "sqlbuild._native:DbtInitResult", + "sqlbuild.integrations.dbt.models:DbtInitResult", + ), + ( + "sqlbuild._native:DbtInteropArgumentError", + "sqlbuild.integrations.dbt.exceptions:DbtInteropArgumentError", + ), + ( + "sqlbuild._native:DbtInteropConfigError", + "sqlbuild.integrations.dbt.exceptions:DbtInteropConfigError", + ), + ( + "sqlbuild._native:DbtInteropError", + "sqlbuild.integrations.dbt.exceptions:DbtInteropError", + ), + ( + "sqlbuild._native:DbtInteropRuntimeError", + "sqlbuild.integrations.dbt.exceptions:DbtInteropRuntimeError", + ), + ( + "sqlbuild._native:DbtLsResult", + "sqlbuild.integrations.dbt.models:DbtLsResult", + ), + ( + "sqlbuild._native:DbtNodeExecutionResult", + "sqlbuild.integrations.dbt.models:DbtNodeExecutionResult", + ), + ( + "sqlbuild._native:DeclarationExpansionContext", + "sqlbuild.compiler.compile.models:DeclarationExpansionContext", + ), + ( + "sqlbuild._native:DeclarationExplanation", + "sqlbuild.compiler.scopes.models:DeclarationExplanation", + ), + ( + "sqlbuild._native:DeclarationIdentity", + "sqlbuild.compiler.scopes.models:DeclarationIdentity", + ), + ( + "sqlbuild._native:DeclarationKind", + "sqlbuild.compiler.scopes.types:DeclarationKind", + ), + ( + "sqlbuild._native:DeclarationParseError", + "sqlbuild.compiler.discovery.exceptions:DeclarationParseError", + ), + ( + "sqlbuild._native:DeclarationRecord", + "sqlbuild.compiler.scopes.models:DeclarationRecord", + ), + ( + "sqlbuild._native:DeclarationReport", + "sqlbuild.compiler.scopes.models:DeclarationReport", + ), + ( + "sqlbuild._native:DeclarationRuntimeProjection", + "sqlbuild.compiler.compile.models:DeclarationRuntimeProjection", + ), + ( + "sqlbuild._native:DeclarationScopeBuild", + "sqlbuild.compiler.compile.models:DeclarationScopeBuild", + ), + ( + "sqlbuild._native:DeclarationScopeResolver", + "sqlbuild.compiler.compile.models:DeclarationScopeResolver", + ), + ( + "sqlbuild._native:DeclarationVisibility", + "sqlbuild.compiler.scopes.models:DeclarationVisibility", + ), + ( + "sqlbuild._native:DeclarationVisibilityIndex", + "sqlbuild.compiler.scopes.models:DeclarationVisibilityIndex", + ), + ( + "sqlbuild._native:DefaultsConfig", + "sqlbuild.spec.contracts.models:DefaultsConfig", + ), + ( + "sqlbuild._native:DeferralInputs", + "sqlbuild.compiler.planner.models:DeferralInputs", + ), + ( + "sqlbuild._native:DiagnosticPhase", + "sqlbuild.compiler.compile.types:DiagnosticPhase", + ), + ( + "sqlbuild._native:DiagnosticSeverity", + "sqlbuild.compiler.compile.types:DiagnosticSeverity", + ), + ( + "sqlbuild._native:DiffConfig", + "sqlbuild.spec.contracts.models:DiffConfig", + ), + ( + "sqlbuild._native:DirectSemanticColumnUse", + "sqlbuild.compiler.lineage.models:DirectSemanticColumnUse", + ), + ( + "sqlbuild._native:DirectSourceFreshnessPlanningResult", + "sqlbuild.compiler.source_freshness.models:DirectSourceFreshnessPlanningResult", + ), + ( + "sqlbuild._native:DiscoveredAdapterFile", + "sqlbuild.compiler.discovery.models:DiscoveredAdapterFile", + ), + ( + "sqlbuild._native:DiscoveredAuditBlock", + "sqlbuild.compiler.discovery.models:DiscoveredAuditBlock", + ), + ( + "sqlbuild._native:DiscoveredAuditFile", + "sqlbuild.compiler.discovery.models:DiscoveredAuditFile", + ), + ( + "sqlbuild._native:DiscoveredConstantFile", + "sqlbuild.compiler.discovery.models:DiscoveredConstantFile", + ), + ( + "sqlbuild._native:DiscoveredEnumFile", + "sqlbuild.compiler.discovery.models:DiscoveredEnumFile", + ), + ( + "sqlbuild._native:DiscoveredModelSchemaFile", + "sqlbuild.compiler.discovery.models:DiscoveredModelSchemaFile", + ), + ( + "sqlbuild._native:DiscoveredPythonAssetMetadata", + "sqlbuild.compiler.python_nodes.models:DiscoveredPythonAssetMetadata", + ), + ( + "sqlbuild._native:DiscoveredPythonCheckMetadata", + "sqlbuild.compiler.python_nodes.models:DiscoveredPythonCheckMetadata", + ), + ( + "sqlbuild._native:DiscoveredPythonFunctionFile", + "sqlbuild.compiler.discovery.models:DiscoveredPythonFunctionFile", + ), + ( + "sqlbuild._native:DiscoveredPythonLoaderMetadata", + "sqlbuild.compiler.python_nodes.models:DiscoveredPythonLoaderMetadata", + ), + ( + "sqlbuild._native:DiscoveredPythonNode", + "sqlbuild.compiler.python_nodes.models:DiscoveredPythonNode", + ), + ( + "sqlbuild._native:DiscoveredPythonTaskMetadata", + "sqlbuild.compiler.python_nodes.models:DiscoveredPythonTaskMetadata", + ), + ( + "sqlbuild._native:DiscoveredSchemaFile", + "sqlbuild.compiler.discovery.models:DiscoveredSchemaFile", + ), + ( + "sqlbuild._native:DiscoveredSeedFile", + "sqlbuild.compiler.discovery.models:DiscoveredSeedFile", + ), + ( + "sqlbuild._native:DiscoveredSqlFunctionFile", + "sqlbuild.compiler.discovery.models:DiscoveredSqlFunctionFile", + ), + ( + "sqlbuild._native:DiscoveredSqlHookFile", + "sqlbuild.compiler.discovery.models:DiscoveredSqlHookFile", + ), + ( + "sqlbuild._native:DiscoveredSqlTestBlock", + "sqlbuild.compiler.discovery.models:DiscoveredSqlTestBlock", + ), + ( + "sqlbuild._native:DiscoveredSqlTestCase", + "sqlbuild.compiler.discovery.models:DiscoveredSqlTestCase", + ), + ( + "sqlbuild._native:DiscoveryConflictError", + "sqlbuild.compiler.discovery.exceptions:DiscoveryConflictError", + ), + ( + "sqlbuild._native:DiscoveryError", + "sqlbuild.compiler.discovery.exceptions:DiscoveryError", + ), + ( + "sqlbuild._native:DiscoveryFileFault", + "sqlbuild.compiler.discovery.models:DiscoveryFileFault", + ), + ( + "sqlbuild._native:DisplayOptions", + "sqlbuild.presentation.models:DisplayOptions", + ), + ( + "sqlbuild._native:Duration", + "sqlbuild.cursor_algebra.models:Duration", + ), + ( + "sqlbuild._native:DynamicColumnContractProof", + "sqlbuild.compiler.compile.models:DynamicColumnContractProof", + ), + ( + "sqlbuild._native:DynamicColumnFamilyProof", + "sqlbuild.compiler.compile.models:DynamicColumnFamilyProof", + ), + ( + "sqlbuild._native:EmptyCursorInputDecision", + "sqlbuild.compiler.planner.models:EmptyCursorInputDecision", + ), + ( + "sqlbuild._native:EnumMember", + "sqlbuild.compiler.discovery.models:EnumMember", + ), + ( + "sqlbuild._native:EnumMemberMetadata", + "sqlbuild.compiler.scopes.models:EnumMemberMetadata", + ), + ( + "sqlbuild._native:EnumMetadata", + "sqlbuild.compiler.scopes.models:EnumMetadata", + ), + ( + "sqlbuild._native:EventExportSeverity", + "sqlbuild.spec.contracts.types:EventExportSeverity", + ), + ( + "sqlbuild._native:EventExporterDiscoveryError", + "sqlbuild.compiler.discovery.exceptions:EventExporterDiscoveryError", + ), + ( + "sqlbuild._native:ExecutionLimitsConfig", + "sqlbuild.spec.contracts.models:ExecutionLimitsConfig", + ), + ( + "sqlbuild._native:ExpandedLintFile", + "sqlbuild.lint.models:ExpandedLintFile", + ), + ( + "sqlbuild._native:ExpansionSpan", + "sqlbuild.compiler.compile.models:ExpansionSpan", + ), + ( + "sqlbuild._native:Fingerprint", + "sqlbuild.compiler.fingerprints.models:Fingerprint", + ), + ( + "sqlbuild._native:FingerprintInputError", + "sqlbuild.compiler.fingerprints.exceptions:FingerprintInputError", + ), + ( + "sqlbuild._native:FingerprintSet", + "sqlbuild.compiler.fingerprints.models:FingerprintSet", + ), + ( + "sqlbuild._native:FixVerdict", + "sqlbuild.lint.models:FixVerdict", + ), + ( + "sqlbuild._native:FixtureColumnInference", + "sqlbuild.compiler.compile.models:FixtureColumnInference", + ), + ( + "sqlbuild._native:FunctionArgument", + "sqlbuild.compiler.compile.models:FunctionArgument", + ), + ( + "sqlbuild._native:FunctionPlanEntry", + "sqlbuild.compiler.planner.models:FunctionPlanEntry", + ), + ( + "sqlbuild._native:FunctionReturnColumn", + "sqlbuild.compiler.compile.models:FunctionReturnColumn", + ), + ( + "sqlbuild._native:FutureCursorAction", + "sqlbuild.spec.contracts.types:FutureCursorAction", + ), + ( + "sqlbuild._native:FutureCursorSafetyError", + "sqlbuild.compiler.planner.exceptions:FutureCursorSafetyError", + ), + ( + "sqlbuild._native:FutureCursorSafetyEvidence", + "sqlbuild.compiler.planner.models:FutureCursorSafetyEvidence", + ), + ( + "sqlbuild._native:FutureCursorsConfig", + "sqlbuild.spec.contracts.models:FutureCursorsConfig", + ), + ( + "sqlbuild._native:GrantKind", + "sqlbuild.compiler.scopes.types:GrantKind", + ), + ( + "sqlbuild._native:GrantRecord", + "sqlbuild.compiler.scopes.models:GrantRecord", + ), + ( + "sqlbuild._native:GraphEdgeExclusion", + "sqlbuild.rule_engine.models:GraphEdgeExclusion", + ), + ( + "sqlbuild._native:HandBuiltAnalysis", + "sqlbuild.compiler.compile.models:HandBuiltAnalysis", + ), + ( + "sqlbuild._native:HeaderSpan", + "sqlbuild.lint.models:HeaderSpan", + ), + ( + "sqlbuild._native:HistoricalInput", + "sqlbuild.compiler.planner.types:HistoricalInput", + ), + ( + "sqlbuild._native:HookReadEdge", + "sqlbuild.compiler.graph.models:HookReadEdge", + ), + ( + "sqlbuild._native:HookReadType", + "sqlbuild.compiler.graph.types:HookReadType", + ), + ( + "sqlbuild._native:InaccessibleReason", + "sqlbuild.compiler.scopes.types:InaccessibleReason", + ), + ( + "sqlbuild._native:InaccessibleRecord", + "sqlbuild.compiler.scopes.models:InaccessibleRecord", + ), + ( + "sqlbuild._native:IncrementalMode", + "sqlbuild.compiler.planner.types:IncrementalMode", + ), + ( + "sqlbuild._native:IncrementalStrategy", + "sqlbuild.compiler.planner.types:IncrementalStrategy", + ), + ( + "sqlbuild._native:InitialValidFrom", + "sqlbuild.compiler.planner.types:InitialValidFrom", + ), + ( + "sqlbuild._native:IntegerValue", + "sqlbuild.cursor_algebra.models:IntegerValue", + ), + ( + "sqlbuild._native:IntegrationLoaderConfig", + "sqlbuild.spec.contracts.models:IntegrationLoaderConfig", + ), + ( + "sqlbuild._native:InterpolationSite", + "sqlbuild.lint.models:InterpolationSite", + ), + ( + "sqlbuild._native:InvalidQualifiedIdentityError", + "sqlbuild.compiler.scopes.exceptions:InvalidQualifiedIdentityError", + ), + ( + "sqlbuild._native:InvalidScopePathError", + "sqlbuild.compiler.scopes.exceptions:InvalidScopePathError", + ), + ( + "sqlbuild._native:InvalidSqlReferenceCall", + "sqlbuild.compiler.compile.models:InvalidSqlReferenceCall", + ), + ( + "sqlbuild._native:JanitorConfig", + "sqlbuild.spec.contracts.models:JanitorConfig", + ), + ( + "sqlbuild._native:LifecycleEventSinkFilterConfig", + "sqlbuild.spec.contracts.models:LifecycleEventSinkFilterConfig", + ), + ( + "sqlbuild._native:LifecycleEventSinksConfig", + "sqlbuild.spec.contracts.models:LifecycleEventSinksConfig", + ), + ( + "sqlbuild._native:LineageGraphViews", + "sqlbuild.compiler.graph.models:LineageGraphViews", + ), + ("sqlbuild._native:LintBody", "sqlbuild.lint.models:LintBody"), + ( + "sqlbuild._native:LintConfig", + "sqlbuild.lint.models:LintConfig", + ), + ("sqlbuild._native:LintEdit", "sqlbuild.lint.models:LintEdit"), + ( + "sqlbuild._native:LintError", + "sqlbuild.lint.exceptions:LintError", + ), + ( + "sqlbuild._native:LintFileRole", + "sqlbuild.lint.models:LintFileRole", + ), + ( + "sqlbuild._native:LintRunResult", + "sqlbuild.lint.models:LintRunResult", + ), + ( + "sqlbuild._native:LintSeverity", + "sqlbuild.lint.types:LintSeverity", + ), + ( + "sqlbuild._native:LintStoppedError", + "sqlbuild.lint.exceptions:LintStoppedError", + ), + ( + "sqlbuild._native:LintViolation", + "sqlbuild.lint.models:LintViolation", + ), + ( + "sqlbuild._native:LiteralSqlRelation", + "sqlbuild.compiler.references.models:LiteralSqlRelation", + ), + ( + "sqlbuild._native:LoaderDestinationParts", + "sqlbuild.spec.contracts.models:LoaderDestinationParts", + ), + ( + "sqlbuild._native:LocalClonePolicy", + "sqlbuild.spec.contracts.models:LocalClonePolicy", + ), + ( + "sqlbuild._native:LocalConfig", + "sqlbuild.spec.contracts.models:LocalConfig", + ), + ( + "sqlbuild._native:LocalDbtConfig", + "sqlbuild.spec.contracts.models:LocalDbtConfig", + ), + ( + "sqlbuild._native:LocalTargetConfig", + "sqlbuild.spec.contracts.models:LocalTargetConfig", + ), + ( + "sqlbuild._native:MacroExpansionResult", + "sqlbuild.compiler.compile.models:MacroExpansionResult", + ), + ( + "sqlbuild._native:MacroMetadata", + "sqlbuild.compiler.scopes.models:MacroMetadata", + ), + ( + "sqlbuild._native:ManualLocation", + "sqlbuild.compiler.refactoring.models:ManualLocation", + ), + ( + "sqlbuild._native:MappedOffset", + "sqlbuild.compiler.compile.models:MappedOffset", + ), + ( + "sqlbuild._native:MaterializationDefaultsConfig", + "sqlbuild.spec.contracts.models:MaterializationDefaultsConfig", + ), + ( + "sqlbuild._native:MaterializationRetentionDefaults", + "sqlbuild.spec.contracts.models:MaterializationRetentionDefaults", + ), + ( + "sqlbuild._native:MaterializationType", + "sqlbuild.compiler.planner.types:MaterializationType", + ), + ( + "sqlbuild._native:MaximumAutomaticStartError", + "sqlbuild.compiler.planner.exceptions:MaximumAutomaticStartError", + ), + ( + "sqlbuild._native:MaximumStartPolicyInputs", + "sqlbuild.compiler.planner.models:MaximumStartPolicyInputs", + ), + ( + "sqlbuild._native:MaximumStartSafetyEvidence", + "sqlbuild.compiler.planner.models:MaximumStartSafetyEvidence", + ), + ( + "sqlbuild._native:MeasurementAuditError", + "sqlbuild.compiler.auditing.exceptions:MeasurementAuditError", + ), + ( + "sqlbuild._native:MeasurementContract", + "sqlbuild.compiler.auditing.models:MeasurementContract", + ), + ( + "sqlbuild._native:MeasurementThresholdBound", + "sqlbuild.compiler.auditing.models:MeasurementThresholdBound", + ), + ( + "sqlbuild._native:MeasurementThresholds", + "sqlbuild.compiler.auditing.models:MeasurementThresholds", + ), + ( + "sqlbuild._native:MicrobatchLimitAction", + "sqlbuild.spec.contracts.types:MicrobatchLimitAction", + ), + ( + "sqlbuild._native:MicrobatchLimitsConfig", + "sqlbuild.spec.contracts.models:MicrobatchLimitsConfig", + ), + ( + "sqlbuild._native:MicrobatchStrategy", + "sqlbuild.compiler.planner.types:MicrobatchStrategy", + ), + ( + "sqlbuild._native:MicrobatchesConfig", + "sqlbuild.spec.contracts.models:MicrobatchesConfig", + ), + ( + "sqlbuild._native:MigrationCompatibility", + "sqlbuild.compiler.migrations.types:MigrationCompatibility", + ), + ( + "sqlbuild._native:MigrationDecision", + "sqlbuild.compiler.migrations.types:MigrationDecision", + ), + ( + "sqlbuild._native:MigrationDiscovery", + "sqlbuild.compiler.migrations.types:MigrationDiscovery", + ), + ( + "sqlbuild._native:MigrationEvent", + "sqlbuild.compiler.migrations.models:MigrationEvent", + ), + ( + "sqlbuild._native:MigrationPromotion", + "sqlbuild.compiler.migrations.types:MigrationPromotion", + ), + ( + "sqlbuild._native:MigrationRelation", + "sqlbuild.compiler.migrations.models:MigrationRelation", + ), + ( + "sqlbuild._native:MigrationsConfig", + "sqlbuild.spec.contracts.models:MigrationsConfig", + ), + ( + "sqlbuild._native:MissingMigrationOriginPolicy", + "sqlbuild.spec.contracts.types:MissingMigrationOriginPolicy", + ), + ( + "sqlbuild._native:ModelColumnLineage", + "sqlbuild.compiler.lineage.models:ModelColumnLineage", + ), + ( + "sqlbuild._native:ModelCursorSnapshot", + "sqlbuild.compiler.planner.models:ModelCursorSnapshot", + ), + ( + "sqlbuild._native:ModelMigrationPlanEntry", + "sqlbuild.compiler.planner.models:ModelMigrationPlanEntry", + ), + ( + "sqlbuild._native:ModelPlanEntry", + "sqlbuild.compiler.planner.models:ModelPlanEntry", + ), + ( + "sqlbuild._native:ModelSchemaDeclaration", + "sqlbuild.compiler.discovery.models:ModelSchemaDeclaration", + ), + ( + "sqlbuild._native:ModelSqlAnalysis", + "sqlbuild.compiler.compile.models:ModelSqlAnalysis", + ), + ( + "sqlbuild._native:ModelSqlParseError", + "sqlbuild.compiler.discovery.exceptions:ModelSqlParseError", + ), + ( + "sqlbuild._native:ModelValidationRequest", + "sqlbuild.compiler.compile.models:ModelValidationRequest", + ), + ( + "sqlbuild._native:MovePreview", + "sqlbuild.compiler.scopes.models:MovePreview", + ), + ( + "sqlbuild._native:NamedSqlHookEntry", + "sqlbuild.compiler.discovery.models:NamedSqlHookEntry", + ), + ( + "sqlbuild._native:NativeColumnQuery", + "sqlbuild.compiler.analysis_session.models:NativeColumnQuery", + ), + ( + "sqlbuild._native:NativeLintError", + "sqlbuild.lint.exceptions:NativeLintError", + ), + ( + "sqlbuild._native:NativeModelAnalyses", + "sqlbuild.compiler.analysis_session.models:NativeModelAnalyses", + ), + ( + "sqlbuild._native:NativeModelAnalysisRequest", + "sqlbuild.compiler.analysis_session.models:NativeModelAnalysisRequest", + ), + ( + "sqlbuild._native:NativeModelConfigInputs", + "sqlbuild.compiler.compile.models:NativeModelConfigInputs", + ), + ( + "sqlbuild._native:NativeModelConfigSession", + "sqlbuild.compiler.compile.models:NativeModelConfigSession", + ), + ( + "sqlbuild._native:NativePivotTables", + "sqlbuild.compiler.analysis_session.models:NativePivotTables", + ), + ( + "sqlbuild._native:NativeQueryColumns", + "sqlbuild.compiler.analysis_session.models:NativeQueryColumns", + ), + ( + "sqlbuild._native:NativeRulesEvaluation", + "sqlbuild.rule_engine.models:NativeRulesEvaluation", + ), + ( + "sqlbuild._native:NativeSqlTestAssembly", + "sqlbuild.compiler.sql_test_glue.models:NativeSqlTestAssembly", + ), + ( + "sqlbuild._native:NativeSqlTestChainRequest", + "sqlbuild.compiler.sql_test_glue.models:NativeSqlTestChainRequest", + ), + ( + "sqlbuild._native:NativeSqlTestFailure", + "sqlbuild.compiler.sql_test_glue.models:NativeSqlTestFailure", + ), + ( + "sqlbuild._native:NativeSqlTestPlanningError", + "sqlbuild.compiler.planner.exceptions:NativeSqlTestPlanningError", + ), + ( + "sqlbuild._native:NativeSqlTestPlanningRequest", + "sqlbuild.compiler.sql_test_glue.models:NativeSqlTestPlanningRequest", + ), + ( + "sqlbuild._native:NativeStageMismatchError", + "sqlbuild.compiler.frontier.exceptions:NativeStageMismatchError", + ), + ( + "sqlbuild._native:NormalizedDbtProfileConnection", + "sqlbuild.integrations.dbt.models:NormalizedDbtProfileConnection", + ), + ( + "sqlbuild._native:OldNameView", + "sqlbuild.compiler.planner.models:OldNameView", + ), + ( + "sqlbuild._native:OldNameViewAction", + "sqlbuild.compiler.migrations.types:OldNameViewAction", + ), + ( + "sqlbuild._native:OldNameViewDropReason", + "sqlbuild.compiler.migrations.types:OldNameViewDropReason", + ), + ( + "sqlbuild._native:OldNameViewEvent", + "sqlbuild.compiler.migrations.models:OldNameViewEvent", + ), + ( + "sqlbuild._native:OldNameViewEventType", + "sqlbuild.compiler.migrations.types:OldNameViewEventType", + ), + ( + "sqlbuild._native:OldNameViewHistory", + "sqlbuild.compiler.migrations.models:OldNameViewHistory", + ), + ( + "sqlbuild._native:OldNameViewPlanEntry", + "sqlbuild.compiler.planner.models:OldNameViewPlanEntry", + ), + ( + "sqlbuild._native:OldNameViewStatus", + "sqlbuild.compiler.migrations.types:OldNameViewStatus", + ), + ( + "sqlbuild._native:OnSchemaChange", + "sqlbuild.compiler.planner.types:OnSchemaChange", + ), + ( + "sqlbuild._native:OwnershipRoot", + "sqlbuild.compiler.scopes.models:OwnershipRoot", + ), + ( + "sqlbuild._native:OwnershipRootKind", + "sqlbuild.compiler.scopes.types:OwnershipRootKind", + ), + ( + "sqlbuild._native:ParsedSelector", + "sqlbuild.compiler.planner.models:ParsedSelector", + ), + ( + "sqlbuild._native:PathSelector", + "sqlbuild.compiler.planner.models:PathSelector", + ), + ( + "sqlbuild._native:PendingLintBody", + "sqlbuild.lint.models:PendingLintBody", + ), + ( + "sqlbuild._native:PlanAction", + "sqlbuild.compiler.planner.types:PlanAction", + ), + ( + "sqlbuild._native:PlanOutput", + "sqlbuild.compiler.planner.models:PlanOutput", + ), + ( + "sqlbuild._native:PlanReason", + "sqlbuild.compiler.planner.types:PlanReason", + ), + ( + "sqlbuild._native:PlannerInputError", + "sqlbuild.compiler.planner.exceptions:PlannerInputError", + ), + ( + "sqlbuild._native:PlannerOverrides", + "sqlbuild.compiler.planner.models:PlannerOverrides", + ), + ( + "sqlbuild._native:PlannerPolicies", + "sqlbuild.compiler.planner.models:PlannerPolicies", + ), + ( + "sqlbuild._native:PlannerRelationsContext", + "sqlbuild.compiler.planner.models:PlannerRelationsContext", + ), + ( + "sqlbuild._native:PlannerScope", + "sqlbuild.compiler.planner.models:PlannerScope", + ), + ( + "sqlbuild._native:PlannerSelection", + "sqlbuild.compiler.planner.models:PlannerSelection", + ), + ( + "sqlbuild._native:PolyglotAnalysisDialect", + "sqlbuild.compiler.lineage.types:PolyglotAnalysisDialect", + ), + ( + "sqlbuild._native:PolyglotAnalysisResult", + "sqlbuild.compiler.compile.models:PolyglotAnalysisResult", + ), + ( + "sqlbuild._native:PreparedLintFiles", + "sqlbuild.lint.models:PreparedLintFiles", + ), + ( + "sqlbuild._native:PreparedSqlLint", + "sqlbuild.rule_engine.models:PreparedSqlLint", + ), + ( + "sqlbuild._native:PreparedSqlLintResult", + "sqlbuild.rule_engine.models:PreparedSqlLintResult", + ), + ( + "sqlbuild._native:ProjectCompileError", + "sqlbuild.lint.exceptions:ProjectCompileError", + ), + ( + "sqlbuild._native:ProjectConfig", + "sqlbuild.spec.contracts.models:ProjectConfig", + ), + ( + "sqlbuild._native:ProjectConfigError", + "sqlbuild.compiler.discovery.exceptions:ProjectConfigError", + ), + ( + "sqlbuild._native:ProjectPathError", + "sqlbuild.compiler.discovery.exceptions:ProjectPathError", + ), + ( + "sqlbuild._native:ProjectPythonPathError", + "sqlbuild.compiler.discovery.exceptions:ProjectPythonPathError", + ), + ( + "sqlbuild._native:ProviderDiscoveryError", + "sqlbuild.compiler.discovery.exceptions:ProviderDiscoveryError", + ), + ( + "sqlbuild._native:PythonIdentityDependency", + "sqlbuild.compiler.python_nodes.models:PythonIdentityDependency", + ), + ( + "sqlbuild._native:PythonIdentityStatus", + "sqlbuild.compiler.python_nodes.types:PythonIdentityStatus", + ), + ( + "sqlbuild._native:PythonNodeDependencyEdge", + "sqlbuild.compiler.python_nodes.models:PythonNodeDependencyEdge", + ), + ( + "sqlbuild._native:PythonNodeDiscoveryError", + "sqlbuild.compiler.discovery.exceptions:PythonNodeDiscoveryError", + ), + ( + "sqlbuild._native:PythonNodeGraph", + "sqlbuild.compiler.python_nodes.models:PythonNodeGraph", + ), + ( + "sqlbuild._native:PythonNodeIdentity", + "sqlbuild.compiler.python_nodes.models:PythonNodeIdentity", + ), + ( + "sqlbuild._native:PythonPlanEntry", + "sqlbuild.compiler.pipeline.models:PythonPlanEntry", + ), + ( + "sqlbuild._native:PythonRunPhase", + "sqlbuild.compiler.python_nodes.types:PythonRunPhase", + ), + ( + "sqlbuild._native:PythonRunPlanOutputs", + "sqlbuild.compiler.pipeline.models:PythonRunPlanOutputs", + ), + ( + "sqlbuild._native:PythonSqlReferenceReport", + "sqlbuild.compiler.compile.models:PythonSqlReferenceReport", + ), + ( + "sqlbuild._native:PythonSqlRunLifecyclePlan", + "sqlbuild.compiler.python_nodes.models:PythonSqlRunLifecyclePlan", + ), + ( + "sqlbuild._native:PythonSqlRunSelection", + "sqlbuild.compiler.python_nodes.models:PythonSqlRunSelection", + ), + ( + "sqlbuild._native:QualifiedLineageColumn", + "sqlbuild.compiler.lineage.models:QualifiedLineageColumn", + ), + ( + "sqlbuild._native:QueryFingerprintError", + "sqlbuild.compiler.fingerprints.exceptions:QueryFingerprintError", + ), + ( + "sqlbuild._native:RefactorIOError", + "sqlbuild.compiler.refactoring.exceptions:RefactorIOError", + ), + ( + "sqlbuild._native:RefactorProject", + "sqlbuild.compiler.refactoring.models:RefactorProject", + ), + ( + "sqlbuild._native:RefactorValueError", + "sqlbuild.compiler.refactoring.exceptions:RefactorValueError", + ), + ( + "sqlbuild._native:ReferencesConfig", + "sqlbuild.spec.contracts.models:ReferencesConfig", + ), + ( + "sqlbuild._native:RelatedLocation", + "sqlbuild.compiler.compile.models:RelatedLocation", + ), + ( + "sqlbuild._native:ResolvedDbtProfileOutput", + "sqlbuild.integrations.dbt.models:ResolvedDbtProfileOutput", + ), + ( + "sqlbuild._native:ResolvedTableType", + "sqlbuild.spec.contracts.models:ResolvedTableType", + ), + ( + "sqlbuild._native:ResolvedTimeTravelRetention", + "sqlbuild.spec.contracts.models:ResolvedTimeTravelRetention", + ), + ( + "sqlbuild._native:ResourceIdentity", + "sqlbuild.compiler.scopes.models:ResourceIdentity", + ), + ( + "sqlbuild._native:ResourceIdentityError", + "sqlbuild.compiler.resource_names.exceptions:ResourceIdentityError", + ), + ( + "sqlbuild._native:ResourceKind", + "sqlbuild.compiler.scopes.types:ResourceKind", + ), + ( + "sqlbuild._native:ResourceRecord", + "sqlbuild.compiler.scopes.models:ResourceRecord", + ), + ( + "sqlbuild._native:RetentionDecreasePolicy", + "sqlbuild.spec.contracts.types:RetentionDecreasePolicy", + ), + ( + "sqlbuild._native:RetentionDirection", + "sqlbuild.compiler.planner.types:RetentionDirection", + ), + ( + "sqlbuild._native:RetentionPlanEntry", + "sqlbuild.compiler.planner.models:RetentionPlanEntry", + ), + ( + "sqlbuild._native:RetentionPlanPhase", + "sqlbuild.compiler.planner.types:RetentionPlanPhase", + ), + ( + "sqlbuild._native:RuleExemption", + "sqlbuild.rule_engine.models:RuleExemption", + ), + ( + "sqlbuild._native:RuleFixResult", + "sqlbuild.lint.models:RuleFixResult", + ), + ( + "sqlbuild._native:RuleGuidance", + "sqlbuild.rule_engine.models:RuleGuidance", + ), + ( + "sqlbuild._native:RuleIgnore", + "sqlbuild.rule_engine.models:RuleIgnore", + ), + ( + "sqlbuild._native:RuleRunPlan", + "sqlbuild.rule_engine.models:RuleRunPlan", + ), + ( + "sqlbuild._native:RulesCacheConfig", + "sqlbuild.rule_engine.models:RulesCacheConfig", + ), + ( + "sqlbuild._native:RulesRunResult", + "sqlbuild.rule_engine.models:RulesRunResult", + ), + ( + "sqlbuild._native:ScenarioArtifactKind", + "sqlbuild.compiler.planner.types:ScenarioArtifactKind", + ), + ( + "sqlbuild._native:ScenarioAssertionExpectationPlan", + "sqlbuild.compiler.planner.models:ScenarioAssertionExpectationPlan", + ), + ( + "sqlbuild._native:ScenarioConfig", + "sqlbuild.spec.contracts.models:ScenarioConfig", + ), + ( + "sqlbuild._native:ScenarioExecutionPlan", + "sqlbuild.compiler.planner.models:ScenarioExecutionPlan", + ), + ( + "sqlbuild._native:ScenarioExpectedExpectationPlan", + "sqlbuild.compiler.planner.models:ScenarioExpectedExpectationPlan", + ), + ( + "sqlbuild._native:ScenarioFixturePlan", + "sqlbuild.compiler.planner.models:ScenarioFixturePlan", + ), + ( + "sqlbuild._native:ScenarioRelationMap", + "sqlbuild.compiler.planner.models:ScenarioRelationMap", + ), + ( + "sqlbuild._native:ScenarioRelationPlan", + "sqlbuild.compiler.planner.models:ScenarioRelationPlan", + ), + ( + "sqlbuild._native:ScenarioSnapshotLimitsConfig", + "sqlbuild.spec.contracts.models:ScenarioSnapshotLimitsConfig", + ), + ( + "sqlbuild._native:SchemaAuditInstance", + "sqlbuild.spec.contracts.models:SchemaAuditInstance", + ), + ( + "sqlbuild._native:SchemaColumn", + "sqlbuild.spec.contracts.models:SchemaColumn", + ), + ( + "sqlbuild._native:SchemaDynamicColumnFamily", + "sqlbuild.spec.contracts.models:SchemaDynamicColumnFamily", + ), + ( + "sqlbuild._native:SchemaFinding", + "sqlbuild.compiler.planner.models:SchemaFinding", + ), + ( + "sqlbuild._native:SchemaModelEntry", + "sqlbuild.spec.contracts.models:SchemaModelEntry", + ), + ( + "sqlbuild._native:SchemaParseError", + "sqlbuild.compiler.discovery.exceptions:SchemaParseError", + ), + ( + "sqlbuild._native:SchemaSeedEntry", + "sqlbuild.spec.contracts.models:SchemaSeedEntry", + ), + ( + "sqlbuild._native:ScopeBrowseResult", + "sqlbuild.compiler.scopes.models:ScopeBrowseResult", + ), + ( + "sqlbuild._native:ScopeChainEntry", + "sqlbuild.compiler.scopes.models:ScopeChainEntry", + ), + ( + "sqlbuild._native:ScopeCompleteness", + "sqlbuild.compiler.scopes.models:ScopeCompleteness", + ), + ( + "sqlbuild._native:ScopeDiagnostic", + "sqlbuild.compiler.scopes.models:ScopeDiagnostic", + ), + ( + "sqlbuild._native:ScopeDiagnosticCode", + "sqlbuild.compiler.scopes.types:ScopeDiagnosticCode", + ), + ( + "sqlbuild._native:ScopeDiagnosticSeverity", + "sqlbuild.compiler.scopes.types:DiagnosticSeverity", + ), + ( + "sqlbuild._native:ScopeError", + "sqlbuild.compiler.scopes.exceptions:ScopeError", + ), + ( + "sqlbuild._native:ScopeFolder", + "sqlbuild.compiler.scopes.models:ScopeFolder", + ), + ( + "sqlbuild._native:ScopeIndex", + "sqlbuild.compiler.scopes.models:ScopeIndex", + ), + ( + "sqlbuild._native:ScopeKind", + "sqlbuild.compiler.scopes.types:ScopeKind", + ), + ( + "sqlbuild._native:ScopeListResult", + "sqlbuild.compiler.scopes.models:ScopeListResult", + ), + ( + "sqlbuild._native:ScopeLookup", + "sqlbuild.compiler.scopes.models:ScopeLookup", + ), + ( + "sqlbuild._native:ScopeReport", + "sqlbuild.compiler.scopes.models:ScopeReport", + ), + ( + "sqlbuild._native:ScopeReportFilters", + "sqlbuild.compiler.scopes.models:ScopeReportFilters", + ), + ( + "sqlbuild._native:ScopeResourceReport", + "sqlbuild.compiler.scopes.models:ScopeResourceReport", + ), + ( + "sqlbuild._native:ScopeSection", + "sqlbuild.compiler.scopes.models:ScopeSection", + ), + ( + "sqlbuild._native:ScopeSourceLocation", + "sqlbuild.compiler.scopes.models:SourceLocation", + ), + ( + "sqlbuild._native:ScopeTargetQuery", + "sqlbuild.compiler.scopes.models:ScopeTargetQuery", + ), + ( + "sqlbuild._native:ScopeValidationError", + "sqlbuild.compiler.scopes.exceptions:ScopeValidationError", + ), + ( + "sqlbuild._native:ScopesConfig", + "sqlbuild.spec.contracts.models:ScopesConfig", + ), + ( + "sqlbuild._native:SeedCsvSettings", + "sqlbuild.spec.contracts.models:SeedCsvSettings", + ), + ( + "sqlbuild._native:SeedDiscoveryError", + "sqlbuild.compiler.discovery.exceptions:SeedDiscoveryError", + ), + ( + "sqlbuild._native:SeedPlanEntry", + "sqlbuild.compiler.planner.models:SeedPlanEntry", + ), + ( + "sqlbuild._native:SelectStarAllow", + "sqlbuild.rule_engine.models:SelectStarAllow", + ), + ( + "sqlbuild._native:SelectedDbtProfileOutput", + "sqlbuild.integrations.dbt.models:SelectedDbtProfileOutput", + ), + ( + "sqlbuild._native:SelectorKind", + "sqlbuild.compiler.planner.types:SelectorKind", + ), + ( + "sqlbuild._native:SettingsConfig", + "sqlbuild.spec.contracts.models:SettingsConfig", + ), + ( + "sqlbuild._native:SinksConfig", + "sqlbuild.spec.contracts.models:SinksConfig", + ), + ( + "sqlbuild._native:SnapshotSchemaChangePolicy", + "sqlbuild.compiler.planner.types:SnapshotSchemaChangePolicy", + ), + ( + "sqlbuild._native:SnapshotStrategy", + "sqlbuild.compiler.planner.types:SnapshotStrategy", + ), + ( + "sqlbuild._native:SnapshotsConfig", + "sqlbuild.spec.contracts.models:SnapshotsConfig", + ), + ( + "sqlbuild._native:SourceColumnEntry", + "sqlbuild.spec.contracts.models:SourceColumnEntry", + ), + ( + "sqlbuild._native:SourceEntry", + "sqlbuild.spec.contracts.models:SourceEntry", + ), + ( + "sqlbuild._native:SourceFreshnessAgePolicy", + "sqlbuild.spec.contracts.models:SourceFreshnessAgePolicy", + ), + ( + "sqlbuild._native:SourceFreshnessConfig", + "sqlbuild.spec.contracts.models:SourceFreshnessConfig", + ), + ( + "sqlbuild._native:SourceFreshnessRecord", + "sqlbuild.compiler.source_freshness.models:SourceFreshnessRecord", + ), + ( + "sqlbuild._native:SourceFreshnessStrategy", + "sqlbuild.spec.contracts.types:SourceFreshnessStrategy", + ), + ( + "sqlbuild._native:SourceFreshnessValueKind", + "sqlbuild.spec.contracts.types:SourceFreshnessValueKind", + ), + ( + "sqlbuild._native:SourceLoadPlanEntry", + "sqlbuild.compiler.planner.models:SourceLoadPlanEntry", + ), + ( + "sqlbuild._native:SourceLocation", + "sqlbuild.spec.contracts.models:SourceLocation", + ), + ( + "sqlbuild._native:SourceParseError", + "sqlbuild.compiler.discovery.exceptions:SourceParseError", + ), + ( + "sqlbuild._native:SourceWriteStrategy", + "sqlbuild.spec.contracts.types:SourceWriteStrategy", + ), + ( + "sqlbuild._native:SpecConfigError", + "sqlbuild.spec.contracts.exceptions:SpecConfigError", + ), + ( + "sqlbuild._native:SqlAuditParseError", + "sqlbuild.compiler.discovery.exceptions:SqlAuditParseError", + ), + ( + "sqlbuild._native:SqlBindingDiagnostic", + "sqlbuild.compiler.sql_analysis.models:SqlBindingDiagnostic", + ), + ( + "sqlbuild._native:SqlBindingResult", + "sqlbuild.compiler.sql_analysis.models:SqlBindingResult", + ), + ( + "sqlbuild._native:SqlExpansionContext", + "sqlbuild.compiler.compile.models:SqlExpansionContext", + ), + ( + "sqlbuild._native:SqlExpansionReuse", + "sqlbuild.rule_engine.models:SqlExpansionReuse", + ), + ( + "sqlbuild._native:SqlHookParseError", + "sqlbuild.compiler.discovery.exceptions:SqlHookParseError", + ), + ( + "sqlbuild._native:SqlLexicalSyntax", + "sqlbuild.compiler.sql_analysis.models:SqlLexicalSyntax", + ), + ( + "sqlbuild._native:SqlLogicalType", + "sqlbuild.sql_values.models:SqlLogicalType", + ), + ( + "sqlbuild._native:SqlReferenceScan", + "sqlbuild.compiler.compile.models:SqlReferenceScan", + ), + ( + "sqlbuild._native:SqlRulesEvaluation", + "sqlbuild.rule_engine.models:SqlRulesEvaluation", + ), + ( + "sqlbuild._native:SqlScenarioParseError", + "sqlbuild.compiler.discovery.exceptions:SqlScenarioParseError", + ), + ( + "sqlbuild._native:SqlSchemaValidationRequest", + "sqlbuild.compiler.sql_analysis.models:SqlSchemaValidationRequest", + ), + ( + "sqlbuild._native:SqlTestAssertionStep", + "sqlbuild.compiler.planner.models:SqlTestAssertionStep", + ), + ( + "sqlbuild._native:SqlTestParseError", + "sqlbuild.compiler.discovery.exceptions:SqlTestParseError", + ), + ( + "sqlbuild._native:SqlTestPlanEntry", + "sqlbuild.compiler.planner.models:SqlTestPlanEntry", + ), + ( + "sqlbuild._native:SqlTestRulesConfig", + "sqlbuild.rule_engine.models:SqlTestRulesConfig", + ), + ( + "sqlbuild._native:SqlTestSelection", + "sqlbuild.compiler.planner.models:SqlTestSelection", + ), + ( + "sqlbuild._native:SqlValue", + "sqlbuild.sql_values.models:SqlValue", + ), + ( + "sqlbuild._native:SqlValueKind", + "sqlbuild.sql_values.types:SqlValueKind", + ), + ( + "sqlbuild._native:StartCursorsConfig", + "sqlbuild.spec.contracts.models:StartCursorsConfig", + ), + ( + "sqlbuild._native:StateSqlValueType", + "sqlbuild.sql_values.types:StateSqlValueType", + ), + ( + "sqlbuild._native:StaticCommandContext", + "sqlbuild.compiler.pipeline.models:StaticCommandContext", + ), + ( + "sqlbuild._native:TableType", + "sqlbuild.spec.contracts.types:TableType", + ), + ( + "sqlbuild._native:TableTypeDowngradePolicy", + "sqlbuild.spec.contracts.types:TableTypeDowngradePolicy", + ), + ( + "sqlbuild._native:TableTypePlanEntry", + "sqlbuild.compiler.planner.models:TableTypePlanEntry", + ), + ( + "sqlbuild._native:TableTypeSource", + "sqlbuild.spec.contracts.types:TableTypeSource", + ), + ( + "sqlbuild._native:TargetConfig", + "sqlbuild.spec.contracts.models:TargetConfig", + ), + ( + "sqlbuild._native:TargetDiffConfig", + "sqlbuild.spec.contracts.models:TargetDiffConfig", + ), + ( + "sqlbuild._native:TargetWarehousesConfig", + "sqlbuild.spec.contracts.models:TargetWarehousesConfig", + ), + ( + "sqlbuild._native:ThresholdOverride", + "sqlbuild.rule_engine.models:ThresholdOverride", + ), + ( + "sqlbuild._native:TimeTravelRetentionSource", + "sqlbuild.spec.contracts.types:TimeTravelRetentionSource", + ), + ( + "sqlbuild._native:TimestampValue", + "sqlbuild.cursor_algebra.models:TimestampValue", + ), + ( + "sqlbuild._native:TolerantScopeDiscovery", + "sqlbuild.compiler.discovery.models:TolerantScopeDiscovery", + ), + ( + "sqlbuild._native:UnindexableScopeError", + "sqlbuild.compiler.scopes.exceptions:UnindexableScopeError", + ), + ( + "sqlbuild._native:UnknownLineageDialectError", + "sqlbuild.compiler.lineage.exceptions:UnknownLineageDialectError", + ), + ( + "sqlbuild._native:UnsupportedLexicalSyntaxError", + "sqlbuild.compiler.sql_analysis.exceptions:UnsupportedLexicalSyntaxError", + ), + ( + "sqlbuild._native:UsageKind", + "sqlbuild.compiler.scopes.types:UsageKind", + ), + ( + "sqlbuild._native:UsageRecord", + "sqlbuild.compiler.scopes.models:UsageRecord", + ), + ( + "sqlbuild._native:VisibilityProvenance", + "sqlbuild.compiler.scopes.models:VisibilityProvenance", + ), + ( + "sqlbuild._native:VisibilityReason", + "sqlbuild.compiler.scopes.types:VisibilityReason", + ), + ( + "sqlbuild._native:VisibilityRecord", + "sqlbuild.compiler.scopes.models:VisibilityRecord", + ), + ( + "sqlbuild._native:VisibilityResolution", + "sqlbuild.compiler.scopes.models:VisibilityResolution", + ), + ( + "sqlbuild._native:VisibleModelDeclarations", + "sqlbuild.compiler.compile.models:VisibleModelDeclarations", + ), + ( + "sqlbuild._native:WarehouseGroup", + "sqlbuild.spec.contracts.types:WarehouseGroup", + ), +]; diff --git a/crates/sqlbuild-cache/src/hosted_shapes/errors.rs b/crates/sqlbuild-cache/src/hosted_shapes/errors.rs new file mode 100644 index 0000000000..0f1f6ce65b --- /dev/null +++ b/crates/sqlbuild-cache/src/hosted_shapes/errors.rs @@ -0,0 +1,5 @@ +//! Why a Python shape name could not be resolved to one native class. + +/// More than one natively hosted class claims the same Python shape name. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub struct AmbiguousShapeName; diff --git a/crates/sqlbuild-cache/src/hosted_shapes/main/hosted_native_name.rs b/crates/sqlbuild-cache/src/hosted_shapes/main/hosted_native_name.rs new file mode 100644 index 0000000000..62473e0934 --- /dev/null +++ b/crates/sqlbuild-cache/src/hosted_shapes/main/hosted_native_name.rs @@ -0,0 +1,28 @@ +//! The native class hosting one Python shape, found by its module and class name. + +use crate::hosted_shapes::constants::{HOSTED_SHAPE_TYPES, NATIVE_MODULE}; +use crate::hosted_shapes::errors::AmbiguousShapeName; + +/// The native class name hosting `module:class`, or `None` when the shape is still Python. +pub fn hosted_native_name( + module: &str, + class: &str, +) -> Result, AmbiguousShapeName> { + let mut found: Option<&'static str> = None; + for (native, python) in HOSTED_SHAPE_TYPES { + if python.split_once(':') != Some((module, class)) { + continue; + } + let Some(native_name) = native + .strip_prefix(NATIVE_MODULE) + .and_then(|rest| rest.strip_prefix(':')) + else { + continue; + }; + if found.is_some_and(|existing| existing != native_name) { + return Err(AmbiguousShapeName); + } + found = Some(native_name); + } + Ok(found) +} diff --git a/crates/sqlbuild-cache/src/hosted_shapes/main/mod.rs b/crates/sqlbuild-cache/src/hosted_shapes/main/mod.rs new file mode 100644 index 0000000000..99b510965b --- /dev/null +++ b/crates/sqlbuild-cache/src/hosted_shapes/main/mod.rs @@ -0,0 +1 @@ +pub mod hosted_native_name; diff --git a/crates/sqlbuild-cache/src/hosted_shapes/mod.rs b/crates/sqlbuild-cache/src/hosted_shapes/mod.rs new file mode 100644 index 0000000000..45be7c1b09 --- /dev/null +++ b/crates/sqlbuild-cache/src/hosted_shapes/mod.rs @@ -0,0 +1,8 @@ +//! Natively hosted compiler shapes and the Python names they are captured and resolved as. + +pub mod constants; +pub mod errors; +pub mod main; + +#[cfg(test)] +mod tests; diff --git a/crates/sqlbuild-cache/src/hosted_shapes/tests.rs b/crates/sqlbuild-cache/src/hosted_shapes/tests.rs new file mode 100644 index 0000000000..94b69ddd59 --- /dev/null +++ b/crates/sqlbuild-cache/src/hosted_shapes/tests.rs @@ -0,0 +1,6 @@ +#[path = "tests/helpers.rs"] +mod helpers; +#[path = "tests/test_hosted_shape_types.rs"] +mod hosted_shape_types; +#[path = "tests/test_types.rs"] +mod test_types; diff --git a/crates/sqlbuild-cache/src/hosted_shapes/tests/helpers.rs b/crates/sqlbuild-cache/src/hosted_shapes/tests/helpers.rs new file mode 100644 index 0000000000..a6bf6a941c --- /dev/null +++ b/crates/sqlbuild-cache/src/hosted_shapes/tests/helpers.rs @@ -0,0 +1,10 @@ +use std::collections::HashSet; + +/// The keys of `keys` seen more than once, in first-repeat order. +pub(super) fn duplicates(keys: &[&'static str]) -> Vec<&'static str> { + let mut seen: HashSet<&str> = HashSet::new(); + keys.iter() + .copied() + .filter(|key| !seen.insert(key)) + .collect() +} diff --git a/crates/sqlbuild-cache/src/hosted_shapes/tests/test_hosted_shape_types.rs b/crates/sqlbuild-cache/src/hosted_shapes/tests/test_hosted_shape_types.rs new file mode 100644 index 0000000000..8070b1c832 --- /dev/null +++ b/crates/sqlbuild-cache/src/hosted_shapes/tests/test_hosted_shape_types.rs @@ -0,0 +1,63 @@ +use crate::hosted_shapes::constants::HOSTED_SHAPE_TYPES; +use crate::hosted_shapes::main::hosted_native_name::hosted_native_name; +use crate::hosted_shapes::tests::helpers::duplicates; +use crate::hosted_shapes::tests::test_types::{HostedKeyTestCase, NativeNameTestCase}; + +#[test] +fn given_hosted_shape_table_when_keyed_then_every_native_and_python_name_is_unique() { + let test_cases = [ + HostedKeyTestCase { + description: "native class names", + keys: HOSTED_SHAPE_TYPES + .iter() + .map(|(native, _)| *native) + .collect(), + expected_duplicates: Vec::new(), + }, + HostedKeyTestCase { + description: "qualified Python shape names", + keys: HOSTED_SHAPE_TYPES + .iter() + .map(|(_, python)| *python) + .collect(), + expected_duplicates: Vec::new(), + }, + ]; + for test_case in test_cases { + assert_eq!( + duplicates(&test_case.keys), + test_case.expected_duplicates, + "{}", + test_case.description + ); + } +} + +#[test] +fn given_python_shape_when_resolving_then_only_its_own_module_selects_the_native_class() { + let test_cases = [ + NativeNameTestCase { + description: "a scope severity resolves to the scope enum", + shape: ("sqlbuild.compiler.scopes.types", "DiagnosticSeverity"), + expected_native: Some("ScopeDiagnosticSeverity"), + }, + NativeNameTestCase { + description: "a shape name from another module is not hosted", + shape: ("sqlbuild.compiler.compile.models", "DiagnosticSeverity"), + expected_native: None, + }, + NativeNameTestCase { + description: "a hosted planner record resolves by its own name", + shape: ("sqlbuild.compiler.planner.models", "PlanOutput"), + expected_native: Some("PlanOutput"), + }, + ]; + for test_case in test_cases { + assert_eq!( + hosted_native_name(test_case.shape.0, test_case.shape.1), + Ok(test_case.expected_native), + "{}", + test_case.description + ); + } +} diff --git a/crates/sqlbuild-cache/src/hosted_shapes/tests/test_types.rs b/crates/sqlbuild-cache/src/hosted_shapes/tests/test_types.rs new file mode 100644 index 0000000000..68ad02324d --- /dev/null +++ b/crates/sqlbuild-cache/src/hosted_shapes/tests/test_types.rs @@ -0,0 +1,11 @@ +pub(crate) struct HostedKeyTestCase { + pub(crate) description: &'static str, + pub(crate) keys: Vec<&'static str>, + pub(crate) expected_duplicates: Vec<&'static str>, +} + +pub(crate) struct NativeNameTestCase { + pub(crate) description: &'static str, + pub(crate) shape: (&'static str, &'static str), + pub(crate) expected_native: Option<&'static str>, +} diff --git a/crates/sqlbuild-cache/src/lib.rs b/crates/sqlbuild-cache/src/lib.rs index 679ef09814..c11728d69f 100644 --- a/crates/sqlbuild-cache/src/lib.rs +++ b/crates/sqlbuild-cache/src/lib.rs @@ -4,6 +4,7 @@ pub mod artifacts; pub mod digest; +pub mod hosted_shapes; pub mod macro_store_keys; pub mod module_digests; pub mod project_reuse; diff --git a/crates/sqlbuild-cache/src/stage_capture/constants.rs b/crates/sqlbuild-cache/src/stage_capture/constants.rs index bb0c1a1f70..a3877a527a 100644 --- a/crates/sqlbuild-cache/src/stage_capture/constants.rs +++ b/crates/sqlbuild-cache/src/stage_capture/constants.rs @@ -382,1380 +382,3 @@ pub const CANONICAL_TYPES: [(&str, &str); 11] = [ ]; /// Sequence types captured as their items. pub const DECODED_SEQUENCES: [&str; 1] = ["sqlbuild.compiler.compile.models:CompactLineageFacts"]; -/// The Python model type a natively hosted compiler shape is captured as. -pub const HOSTED_SHAPE_TYPES: [(&str, &str); 345] = [ - ( - "sqlbuild._native:AlignedInterval", - "sqlbuild.cursor_algebra.models:AlignedInterval", - ), - ( - "sqlbuild._native:AnalysisCacheContext", - "sqlbuild.compiler.compile.models:AnalysisCacheContext", - ), - ( - "sqlbuild._native:AttachedAuditGateEdge", - "sqlbuild.compiler.graph.models:AttachedAuditGateEdge", - ), - ( - "sqlbuild._native:AuditGateIdentity", - "sqlbuild.compiler.auditing.models:AuditGateIdentity", - ), - ( - "sqlbuild._native:AuditIdentity", - "sqlbuild.compiler.auditing.models:AuditIdentity", - ), - ( - "sqlbuild._native:AuthoredTimeTravelRetention", - "sqlbuild.spec.contracts.models:AuthoredTimeTravelRetention", - ), - ( - "sqlbuild._native:ClonePipelineConnection", - "sqlbuild.compiler.pipeline.models:ClonePipelineConnection", - ), - ( - "sqlbuild._native:ClonePipelineOptions", - "sqlbuild.compiler.pipeline.models:ClonePipelineOptions", - ), - ( - "sqlbuild._native:ClonePipelineResult", - "sqlbuild.compiler.pipeline.models:ClonePipelineResult", - ), - ( - "sqlbuild._native:ClonePolicy", - "sqlbuild.spec.contracts.models:ClonePolicy", - ), - ( - "sqlbuild._native:CollectionRendering", - "sqlbuild.sql_values.types:CollectionRendering", - ), - ( - "sqlbuild._native:ColumnContractMode", - "sqlbuild.spec.contracts.types:ColumnContractMode", - ), - ( - "sqlbuild._native:ColumnLineage", - "sqlbuild.compiler.lineage.models:ColumnLineage", - ), - ( - "sqlbuild._native:ColumnLineageConfidence", - "sqlbuild.compiler.lineage.types:ColumnLineageConfidence", - ), - ( - "sqlbuild._native:ColumnLineageEdge", - "sqlbuild.compiler.lineage.models:ColumnLineageEdge", - ), - ( - "sqlbuild._native:ColumnLineageMode", - "sqlbuild.compiler.lineage.types:ColumnLineageMode", - ), - ( - "sqlbuild._native:ColumnLineageSource", - "sqlbuild.compiler.lineage.models:ColumnLineageSource", - ), - ( - "sqlbuild._native:ColumnTransformKind", - "sqlbuild.compiler.lineage.types:ColumnTransformKind", - ), - ( - "sqlbuild._native:CompileAdapterContext", - "sqlbuild.compiler.compile.models:CompileAdapterContext", - ), - ( - "sqlbuild._native:CompileAnalysisSelection", - "sqlbuild.compiler.compile.models:CompileAnalysisSelection", - ), - ( - "sqlbuild._native:CompileDirectLogicSqlTestInputPayload", - "sqlbuild.compiler.compile.models:CompileDirectLogicSqlTestInputPayload", - ), - ( - "sqlbuild._native:CompileFacts", - "sqlbuild.lint.models:CompileFacts", - ), - ( - "sqlbuild._native:CompileModelConfig", - "sqlbuild.compiler.compile.models:CompileModelConfig", - ), - ( - "sqlbuild._native:CompileModelSqlTestInputPayload", - "sqlbuild.compiler.compile.models:CompileModelSqlTestInputPayload", - ), - ( - "sqlbuild._native:CompilePipelineOptions", - "sqlbuild.compiler.pipeline.models:CompilePipelineOptions", - ), - ( - "sqlbuild._native:CompileProjectInputs", - "sqlbuild.compiler.compile.models:CompileProjectInputs", - ), - ( - "sqlbuild._native:CompileSqlScenarioCte", - "sqlbuild.compiler.compile.models:CompileSqlScenarioCte", - ), - ( - "sqlbuild._native:CompileSqlScenarioInput", - "sqlbuild.compiler.compile.models:CompileSqlScenarioInput", - ), - ( - "sqlbuild._native:CompileSqlTestCte", - "sqlbuild.compiler.compile.models:CompileSqlTestCte", - ), - ( - "sqlbuild._native:CompileSqlTestInput", - "sqlbuild.compiler.compile.models:CompileSqlTestInput", - ), - ( - "sqlbuild._native:CompiledDirectLogicSqlTestPayload", - "sqlbuild.compiler.compile.models:CompiledDirectLogicSqlTestPayload", - ), - ( - "sqlbuild._native:CompiledFunction", - "sqlbuild.compiler.compile.models:CompiledFunction", - ), - ( - "sqlbuild._native:CompiledLineageColumnFact", - "sqlbuild.compiler.compile.models:CompiledLineageColumnFact", - ), - ( - "sqlbuild._native:CompiledModelSqlTestPayload", - "sqlbuild.compiler.compile.models:CompiledModelSqlTestPayload", - ), - ( - "sqlbuild._native:CompiledProjectPhaseResult", - "sqlbuild.compiler.pipeline.models:CompiledProjectPhaseResult", - ), - ( - "sqlbuild._native:CompiledSeed", - "sqlbuild.compiler.compile.models:CompiledSeed", - ), - ( - "sqlbuild._native:CompiledSource", - "sqlbuild.compiler.compile.models:CompiledSource", - ), - ( - "sqlbuild._native:CompiledSqlExpansion", - "sqlbuild.compiler.compile.models:CompiledSqlExpansion", - ), - ( - "sqlbuild._native:CompiledSqlTestResource", - "sqlbuild.compiler.compile.models:CompiledSqlTestResource", - ), - ( - "sqlbuild._native:CompilerStage", - "sqlbuild.compiler.frontier.types:CompilerStage", - ), - ( - "sqlbuild._native:ConfigValueTypeError", - "sqlbuild.spec.contracts.exceptions:ConfigValueTypeError", - ), - ( - "sqlbuild._native:ConstantMetadata", - "sqlbuild.compiler.scopes.models:ConstantMetadata", - ), - ( - "sqlbuild._native:ConstantsConfig", - "sqlbuild.spec.contracts.models:ConstantsConfig", - ), - ( - "sqlbuild._native:ContractAdoptionError", - "sqlbuild.compiler.contract_adoption.exceptions:ContractAdoptionError", - ), - ( - "sqlbuild._native:ContractAdoptionResult", - "sqlbuild.compiler.contract_adoption.models:ContractAdoptionResult", - ), - ( - "sqlbuild._native:ContractEvidence", - "sqlbuild.compiler.contract_adoption.models:ContractEvidence", - ), - ( - "sqlbuild._native:ContractFinding", - "sqlbuild.compiler.contract_adoption.models:ContractFinding", - ), - ( - "sqlbuild._native:ContractFindingKind", - "sqlbuild.compiler.contract_adoption.types:ContractFindingKind", - ), - ( - "sqlbuild._native:ContractValidationResult", - "sqlbuild.compiler.contracts.models:ContractValidationResult", - ), - ( - "sqlbuild._native:CostConfig", - "sqlbuild.spec.contracts.models:CostConfig", - ), - ( - "sqlbuild._native:CursorInputRoles", - "sqlbuild.compiler.compile.models:CursorInputRoles", - ), - ( - "sqlbuild._native:CursorsConfig", - "sqlbuild.spec.contracts.models:CursorsConfig", - ), - ( - "sqlbuild._native:CustomRulesOutcome", - "sqlbuild.rule_engine.models:CustomRulesOutcome", - ), - ( - "sqlbuild._native:DateValue", - "sqlbuild.cursor_algebra.models:DateValue", - ), - ( - "sqlbuild._native:DbtCliOptions", - "sqlbuild.integrations.dbt.models:DbtCliOptions", - ), - ( - "sqlbuild._native:DbtCommandResult", - "sqlbuild.integrations.dbt.models:DbtCommandResult", - ), - ( - "sqlbuild._native:DbtConfig", - "sqlbuild.spec.contracts.models:DbtConfig", - ), - ( - "sqlbuild._native:DbtInitProgressCallbacks", - "sqlbuild.integrations.dbt.models:DbtInitProgressCallbacks", - ), - ( - "sqlbuild._native:DbtInitRequest", - "sqlbuild.integrations.dbt.models:DbtInitRequest", - ), - ( - "sqlbuild._native:DbtInitResult", - "sqlbuild.integrations.dbt.models:DbtInitResult", - ), - ( - "sqlbuild._native:DbtInteropArgumentError", - "sqlbuild.integrations.dbt.exceptions:DbtInteropArgumentError", - ), - ( - "sqlbuild._native:DbtInteropConfigError", - "sqlbuild.integrations.dbt.exceptions:DbtInteropConfigError", - ), - ( - "sqlbuild._native:DbtInteropError", - "sqlbuild.integrations.dbt.exceptions:DbtInteropError", - ), - ( - "sqlbuild._native:DbtInteropRuntimeError", - "sqlbuild.integrations.dbt.exceptions:DbtInteropRuntimeError", - ), - ( - "sqlbuild._native:DbtLsResult", - "sqlbuild.integrations.dbt.models:DbtLsResult", - ), - ( - "sqlbuild._native:DbtNodeExecutionResult", - "sqlbuild.integrations.dbt.models:DbtNodeExecutionResult", - ), - ( - "sqlbuild._native:DeclarationExpansionContext", - "sqlbuild.compiler.compile.models:DeclarationExpansionContext", - ), - ( - "sqlbuild._native:DeclarationExplanation", - "sqlbuild.compiler.scopes.models:DeclarationExplanation", - ), - ( - "sqlbuild._native:DeclarationIdentity", - "sqlbuild.compiler.scopes.models:DeclarationIdentity", - ), - ( - "sqlbuild._native:DeclarationKind", - "sqlbuild.compiler.scopes.types:DeclarationKind", - ), - ( - "sqlbuild._native:DeclarationParseError", - "sqlbuild.compiler.discovery.exceptions:DeclarationParseError", - ), - ( - "sqlbuild._native:DeclarationRecord", - "sqlbuild.compiler.scopes.models:DeclarationRecord", - ), - ( - "sqlbuild._native:DeclarationReport", - "sqlbuild.compiler.scopes.models:DeclarationReport", - ), - ( - "sqlbuild._native:DeclarationRuntimeProjection", - "sqlbuild.compiler.compile.models:DeclarationRuntimeProjection", - ), - ( - "sqlbuild._native:DeclarationScopeBuild", - "sqlbuild.compiler.compile.models:DeclarationScopeBuild", - ), - ( - "sqlbuild._native:DeclarationScopeResolver", - "sqlbuild.compiler.compile.models:DeclarationScopeResolver", - ), - ( - "sqlbuild._native:DeclarationVisibility", - "sqlbuild.compiler.scopes.models:DeclarationVisibility", - ), - ( - "sqlbuild._native:DeclarationVisibilityIndex", - "sqlbuild.compiler.scopes.models:DeclarationVisibilityIndex", - ), - ( - "sqlbuild._native:DefaultsConfig", - "sqlbuild.spec.contracts.models:DefaultsConfig", - ), - ( - "sqlbuild._native:DeferralInputs", - "sqlbuild.compiler.planner.models:DeferralInputs", - ), - ( - "sqlbuild._native:DiagnosticPhase", - "sqlbuild.compiler.compile.types:DiagnosticPhase", - ), - ( - "sqlbuild._native:DiagnosticSeverity", - "sqlbuild.compiler.compile.types:DiagnosticSeverity", - ), - ( - "sqlbuild._native:DiffConfig", - "sqlbuild.spec.contracts.models:DiffConfig", - ), - ( - "sqlbuild._native:DirectSemanticColumnUse", - "sqlbuild.compiler.lineage.models:DirectSemanticColumnUse", - ), - ( - "sqlbuild._native:DirectSourceFreshnessPlanningResult", - "sqlbuild.compiler.source_freshness.models:DirectSourceFreshnessPlanningResult", - ), - ( - "sqlbuild._native:DiscoveredAdapterFile", - "sqlbuild.compiler.discovery.models:DiscoveredAdapterFile", - ), - ( - "sqlbuild._native:DiscoveredAuditBlock", - "sqlbuild.compiler.discovery.models:DiscoveredAuditBlock", - ), - ( - "sqlbuild._native:DiscoveredAuditFile", - "sqlbuild.compiler.discovery.models:DiscoveredAuditFile", - ), - ( - "sqlbuild._native:DiscoveredConstantFile", - "sqlbuild.compiler.discovery.models:DiscoveredConstantFile", - ), - ( - "sqlbuild._native:DiscoveredEnumFile", - "sqlbuild.compiler.discovery.models:DiscoveredEnumFile", - ), - ( - "sqlbuild._native:DiscoveredModelSchemaFile", - "sqlbuild.compiler.discovery.models:DiscoveredModelSchemaFile", - ), - ( - "sqlbuild._native:DiscoveredPythonAssetMetadata", - "sqlbuild.compiler.python_nodes.models:DiscoveredPythonAssetMetadata", - ), - ( - "sqlbuild._native:DiscoveredPythonCheckMetadata", - "sqlbuild.compiler.python_nodes.models:DiscoveredPythonCheckMetadata", - ), - ( - "sqlbuild._native:DiscoveredPythonFunctionFile", - "sqlbuild.compiler.discovery.models:DiscoveredPythonFunctionFile", - ), - ( - "sqlbuild._native:DiscoveredPythonLoaderMetadata", - "sqlbuild.compiler.python_nodes.models:DiscoveredPythonLoaderMetadata", - ), - ( - "sqlbuild._native:DiscoveredPythonNode", - "sqlbuild.compiler.python_nodes.models:DiscoveredPythonNode", - ), - ( - "sqlbuild._native:DiscoveredPythonTaskMetadata", - "sqlbuild.compiler.python_nodes.models:DiscoveredPythonTaskMetadata", - ), - ( - "sqlbuild._native:DiscoveredSchemaFile", - "sqlbuild.compiler.discovery.models:DiscoveredSchemaFile", - ), - ( - "sqlbuild._native:DiscoveredSeedFile", - "sqlbuild.compiler.discovery.models:DiscoveredSeedFile", - ), - ( - "sqlbuild._native:DiscoveredSqlFunctionFile", - "sqlbuild.compiler.discovery.models:DiscoveredSqlFunctionFile", - ), - ( - "sqlbuild._native:DiscoveredSqlHookFile", - "sqlbuild.compiler.discovery.models:DiscoveredSqlHookFile", - ), - ( - "sqlbuild._native:DiscoveredSqlTestBlock", - "sqlbuild.compiler.discovery.models:DiscoveredSqlTestBlock", - ), - ( - "sqlbuild._native:DiscoveredSqlTestCase", - "sqlbuild.compiler.discovery.models:DiscoveredSqlTestCase", - ), - ( - "sqlbuild._native:DiscoveryConflictError", - "sqlbuild.compiler.discovery.exceptions:DiscoveryConflictError", - ), - ( - "sqlbuild._native:DiscoveryError", - "sqlbuild.compiler.discovery.exceptions:DiscoveryError", - ), - ( - "sqlbuild._native:DiscoveryFileFault", - "sqlbuild.compiler.discovery.models:DiscoveryFileFault", - ), - ( - "sqlbuild._native:DisplayOptions", - "sqlbuild.presentation.models:DisplayOptions", - ), - ( - "sqlbuild._native:Duration", - "sqlbuild.cursor_algebra.models:Duration", - ), - ( - "sqlbuild._native:DynamicColumnContractProof", - "sqlbuild.compiler.compile.models:DynamicColumnContractProof", - ), - ( - "sqlbuild._native:DynamicColumnFamilyProof", - "sqlbuild.compiler.compile.models:DynamicColumnFamilyProof", - ), - ( - "sqlbuild._native:EnumMember", - "sqlbuild.compiler.discovery.models:EnumMember", - ), - ( - "sqlbuild._native:EnumMemberMetadata", - "sqlbuild.compiler.scopes.models:EnumMemberMetadata", - ), - ( - "sqlbuild._native:EnumMetadata", - "sqlbuild.compiler.scopes.models:EnumMetadata", - ), - ( - "sqlbuild._native:EventExportSeverity", - "sqlbuild.spec.contracts.types:EventExportSeverity", - ), - ( - "sqlbuild._native:EventExporterDiscoveryError", - "sqlbuild.compiler.discovery.exceptions:EventExporterDiscoveryError", - ), - ( - "sqlbuild._native:ExecutionLimitsConfig", - "sqlbuild.spec.contracts.models:ExecutionLimitsConfig", - ), - ( - "sqlbuild._native:ExpandedLintFile", - "sqlbuild.lint.models:ExpandedLintFile", - ), - ( - "sqlbuild._native:ExpansionSpan", - "sqlbuild.compiler.compile.models:ExpansionSpan", - ), - ( - "sqlbuild._native:Fingerprint", - "sqlbuild.compiler.fingerprints.models:Fingerprint", - ), - ( - "sqlbuild._native:FingerprintInputError", - "sqlbuild.compiler.fingerprints.exceptions:FingerprintInputError", - ), - ( - "sqlbuild._native:FingerprintSet", - "sqlbuild.compiler.fingerprints.models:FingerprintSet", - ), - ( - "sqlbuild._native:FixVerdict", - "sqlbuild.lint.models:FixVerdict", - ), - ( - "sqlbuild._native:FixtureColumnInference", - "sqlbuild.compiler.compile.models:FixtureColumnInference", - ), - ( - "sqlbuild._native:FunctionArgument", - "sqlbuild.compiler.compile.models:FunctionArgument", - ), - ( - "sqlbuild._native:FunctionReturnColumn", - "sqlbuild.compiler.compile.models:FunctionReturnColumn", - ), - ( - "sqlbuild._native:FutureCursorAction", - "sqlbuild.spec.contracts.types:FutureCursorAction", - ), - ( - "sqlbuild._native:FutureCursorsConfig", - "sqlbuild.spec.contracts.models:FutureCursorsConfig", - ), - ( - "sqlbuild._native:GrantKind", - "sqlbuild.compiler.scopes.types:GrantKind", - ), - ( - "sqlbuild._native:GrantRecord", - "sqlbuild.compiler.scopes.models:GrantRecord", - ), - ( - "sqlbuild._native:GraphEdgeExclusion", - "sqlbuild.rule_engine.models:GraphEdgeExclusion", - ), - ( - "sqlbuild._native:HandBuiltAnalysis", - "sqlbuild.compiler.compile.models:HandBuiltAnalysis", - ), - ( - "sqlbuild._native:HeaderSpan", - "sqlbuild.lint.models:HeaderSpan", - ), - ( - "sqlbuild._native:HookReadEdge", - "sqlbuild.compiler.graph.models:HookReadEdge", - ), - ( - "sqlbuild._native:HookReadType", - "sqlbuild.compiler.graph.types:HookReadType", - ), - ( - "sqlbuild._native:InaccessibleReason", - "sqlbuild.compiler.scopes.types:InaccessibleReason", - ), - ( - "sqlbuild._native:InaccessibleRecord", - "sqlbuild.compiler.scopes.models:InaccessibleRecord", - ), - ( - "sqlbuild._native:IntegerValue", - "sqlbuild.cursor_algebra.models:IntegerValue", - ), - ( - "sqlbuild._native:IntegrationLoaderConfig", - "sqlbuild.spec.contracts.models:IntegrationLoaderConfig", - ), - ( - "sqlbuild._native:InterpolationSite", - "sqlbuild.lint.models:InterpolationSite", - ), - ( - "sqlbuild._native:InvalidQualifiedIdentityError", - "sqlbuild.compiler.scopes.exceptions:InvalidQualifiedIdentityError", - ), - ( - "sqlbuild._native:InvalidScopePathError", - "sqlbuild.compiler.scopes.exceptions:InvalidScopePathError", - ), - ( - "sqlbuild._native:InvalidSqlReferenceCall", - "sqlbuild.compiler.compile.models:InvalidSqlReferenceCall", - ), - ( - "sqlbuild._native:JanitorConfig", - "sqlbuild.spec.contracts.models:JanitorConfig", - ), - ( - "sqlbuild._native:LifecycleEventSinkFilterConfig", - "sqlbuild.spec.contracts.models:LifecycleEventSinkFilterConfig", - ), - ( - "sqlbuild._native:LifecycleEventSinksConfig", - "sqlbuild.spec.contracts.models:LifecycleEventSinksConfig", - ), - ( - "sqlbuild._native:LineageGraphViews", - "sqlbuild.compiler.graph.models:LineageGraphViews", - ), - ("sqlbuild._native:LintBody", "sqlbuild.lint.models:LintBody"), - ( - "sqlbuild._native:LintConfig", - "sqlbuild.lint.models:LintConfig", - ), - ("sqlbuild._native:LintEdit", "sqlbuild.lint.models:LintEdit"), - ( - "sqlbuild._native:LintError", - "sqlbuild.lint.exceptions:LintError", - ), - ( - "sqlbuild._native:LintFileRole", - "sqlbuild.lint.models:LintFileRole", - ), - ( - "sqlbuild._native:LintRunResult", - "sqlbuild.lint.models:LintRunResult", - ), - ( - "sqlbuild._native:LintSeverity", - "sqlbuild.lint.types:LintSeverity", - ), - ( - "sqlbuild._native:LintStoppedError", - "sqlbuild.lint.exceptions:LintStoppedError", - ), - ( - "sqlbuild._native:LintViolation", - "sqlbuild.lint.models:LintViolation", - ), - ( - "sqlbuild._native:LiteralSqlRelation", - "sqlbuild.compiler.references.models:LiteralSqlRelation", - ), - ( - "sqlbuild._native:LoaderDestinationParts", - "sqlbuild.spec.contracts.models:LoaderDestinationParts", - ), - ( - "sqlbuild._native:LocalClonePolicy", - "sqlbuild.spec.contracts.models:LocalClonePolicy", - ), - ( - "sqlbuild._native:LocalConfig", - "sqlbuild.spec.contracts.models:LocalConfig", - ), - ( - "sqlbuild._native:LocalDbtConfig", - "sqlbuild.spec.contracts.models:LocalDbtConfig", - ), - ( - "sqlbuild._native:LocalTargetConfig", - "sqlbuild.spec.contracts.models:LocalTargetConfig", - ), - ( - "sqlbuild._native:MacroExpansionResult", - "sqlbuild.compiler.compile.models:MacroExpansionResult", - ), - ( - "sqlbuild._native:MacroMetadata", - "sqlbuild.compiler.scopes.models:MacroMetadata", - ), - ( - "sqlbuild._native:ManualLocation", - "sqlbuild.compiler.refactoring.models:ManualLocation", - ), - ( - "sqlbuild._native:MappedOffset", - "sqlbuild.compiler.compile.models:MappedOffset", - ), - ( - "sqlbuild._native:MaterializationDefaultsConfig", - "sqlbuild.spec.contracts.models:MaterializationDefaultsConfig", - ), - ( - "sqlbuild._native:MaterializationRetentionDefaults", - "sqlbuild.spec.contracts.models:MaterializationRetentionDefaults", - ), - ( - "sqlbuild._native:MeasurementAuditError", - "sqlbuild.compiler.auditing.exceptions:MeasurementAuditError", - ), - ( - "sqlbuild._native:MeasurementContract", - "sqlbuild.compiler.auditing.models:MeasurementContract", - ), - ( - "sqlbuild._native:MeasurementThresholdBound", - "sqlbuild.compiler.auditing.models:MeasurementThresholdBound", - ), - ( - "sqlbuild._native:MeasurementThresholds", - "sqlbuild.compiler.auditing.models:MeasurementThresholds", - ), - ( - "sqlbuild._native:MicrobatchLimitAction", - "sqlbuild.spec.contracts.types:MicrobatchLimitAction", - ), - ( - "sqlbuild._native:MicrobatchLimitsConfig", - "sqlbuild.spec.contracts.models:MicrobatchLimitsConfig", - ), - ( - "sqlbuild._native:MicrobatchesConfig", - "sqlbuild.spec.contracts.models:MicrobatchesConfig", - ), - ( - "sqlbuild._native:MigrationCompatibility", - "sqlbuild.compiler.migrations.types:MigrationCompatibility", - ), - ( - "sqlbuild._native:MigrationDiscovery", - "sqlbuild.compiler.migrations.types:MigrationDiscovery", - ), - ( - "sqlbuild._native:MigrationsConfig", - "sqlbuild.spec.contracts.models:MigrationsConfig", - ), - ( - "sqlbuild._native:MissingMigrationOriginPolicy", - "sqlbuild.spec.contracts.types:MissingMigrationOriginPolicy", - ), - ( - "sqlbuild._native:ModelColumnLineage", - "sqlbuild.compiler.lineage.models:ModelColumnLineage", - ), - ( - "sqlbuild._native:ModelSchemaDeclaration", - "sqlbuild.compiler.discovery.models:ModelSchemaDeclaration", - ), - ( - "sqlbuild._native:ModelSqlAnalysis", - "sqlbuild.compiler.compile.models:ModelSqlAnalysis", - ), - ( - "sqlbuild._native:ModelSqlParseError", - "sqlbuild.compiler.discovery.exceptions:ModelSqlParseError", - ), - ( - "sqlbuild._native:ModelValidationRequest", - "sqlbuild.compiler.compile.models:ModelValidationRequest", - ), - ( - "sqlbuild._native:MovePreview", - "sqlbuild.compiler.scopes.models:MovePreview", - ), - ( - "sqlbuild._native:NamedSqlHookEntry", - "sqlbuild.compiler.discovery.models:NamedSqlHookEntry", - ), - ( - "sqlbuild._native:NativeColumnQuery", - "sqlbuild.compiler.analysis_session.models:NativeColumnQuery", - ), - ( - "sqlbuild._native:NativeLintError", - "sqlbuild.lint.exceptions:NativeLintError", - ), - ( - "sqlbuild._native:NativeModelAnalyses", - "sqlbuild.compiler.analysis_session.models:NativeModelAnalyses", - ), - ( - "sqlbuild._native:NativeModelAnalysisRequest", - "sqlbuild.compiler.analysis_session.models:NativeModelAnalysisRequest", - ), - ( - "sqlbuild._native:NativeModelConfigInputs", - "sqlbuild.compiler.compile.models:NativeModelConfigInputs", - ), - ( - "sqlbuild._native:NativeModelConfigSession", - "sqlbuild.compiler.compile.models:NativeModelConfigSession", - ), - ( - "sqlbuild._native:NativePivotTables", - "sqlbuild.compiler.analysis_session.models:NativePivotTables", - ), - ( - "sqlbuild._native:NativeQueryColumns", - "sqlbuild.compiler.analysis_session.models:NativeQueryColumns", - ), - ( - "sqlbuild._native:NativeRulesEvaluation", - "sqlbuild.rule_engine.models:NativeRulesEvaluation", - ), - ( - "sqlbuild._native:NativeSqlTestAssembly", - "sqlbuild.compiler.sql_test_glue.models:NativeSqlTestAssembly", - ), - ( - "sqlbuild._native:NativeSqlTestChainRequest", - "sqlbuild.compiler.sql_test_glue.models:NativeSqlTestChainRequest", - ), - ( - "sqlbuild._native:NativeSqlTestFailure", - "sqlbuild.compiler.sql_test_glue.models:NativeSqlTestFailure", - ), - ( - "sqlbuild._native:NativeSqlTestPlanningRequest", - "sqlbuild.compiler.sql_test_glue.models:NativeSqlTestPlanningRequest", - ), - ( - "sqlbuild._native:NativeStageMismatchError", - "sqlbuild.compiler.frontier.exceptions:NativeStageMismatchError", - ), - ( - "sqlbuild._native:NormalizedDbtProfileConnection", - "sqlbuild.integrations.dbt.models:NormalizedDbtProfileConnection", - ), - ( - "sqlbuild._native:OldNameViewAction", - "sqlbuild.compiler.migrations.types:OldNameViewAction", - ), - ( - "sqlbuild._native:OwnershipRoot", - "sqlbuild.compiler.scopes.models:OwnershipRoot", - ), - ( - "sqlbuild._native:OwnershipRootKind", - "sqlbuild.compiler.scopes.types:OwnershipRootKind", - ), - ( - "sqlbuild._native:ParsedSelector", - "sqlbuild.compiler.planner.models:ParsedSelector", - ), - ( - "sqlbuild._native:PathSelector", - "sqlbuild.compiler.planner.models:PathSelector", - ), - ( - "sqlbuild._native:PendingLintBody", - "sqlbuild.lint.models:PendingLintBody", - ), - ( - "sqlbuild._native:PlannerOverrides", - "sqlbuild.compiler.planner.models:PlannerOverrides", - ), - ( - "sqlbuild._native:PlannerPolicies", - "sqlbuild.compiler.planner.models:PlannerPolicies", - ), - ( - "sqlbuild._native:PlannerRelationsContext", - "sqlbuild.compiler.planner.models:PlannerRelationsContext", - ), - ( - "sqlbuild._native:PlannerScope", - "sqlbuild.compiler.planner.models:PlannerScope", - ), - ( - "sqlbuild._native:PlannerSelection", - "sqlbuild.compiler.planner.models:PlannerSelection", - ), - ( - "sqlbuild._native:PolyglotAnalysisDialect", - "sqlbuild.compiler.lineage.types:PolyglotAnalysisDialect", - ), - ( - "sqlbuild._native:PolyglotAnalysisResult", - "sqlbuild.compiler.compile.models:PolyglotAnalysisResult", - ), - ( - "sqlbuild._native:PreparedLintFiles", - "sqlbuild.lint.models:PreparedLintFiles", - ), - ( - "sqlbuild._native:PreparedSqlLint", - "sqlbuild.rule_engine.models:PreparedSqlLint", - ), - ( - "sqlbuild._native:PreparedSqlLintResult", - "sqlbuild.rule_engine.models:PreparedSqlLintResult", - ), - ( - "sqlbuild._native:ProjectCompileError", - "sqlbuild.lint.exceptions:ProjectCompileError", - ), - ( - "sqlbuild._native:ProjectConfig", - "sqlbuild.spec.contracts.models:ProjectConfig", - ), - ( - "sqlbuild._native:ProjectConfigError", - "sqlbuild.compiler.discovery.exceptions:ProjectConfigError", - ), - ( - "sqlbuild._native:ProjectPathError", - "sqlbuild.compiler.discovery.exceptions:ProjectPathError", - ), - ( - "sqlbuild._native:ProjectPythonPathError", - "sqlbuild.compiler.discovery.exceptions:ProjectPythonPathError", - ), - ( - "sqlbuild._native:ProviderDiscoveryError", - "sqlbuild.compiler.discovery.exceptions:ProviderDiscoveryError", - ), - ( - "sqlbuild._native:PythonIdentityDependency", - "sqlbuild.compiler.python_nodes.models:PythonIdentityDependency", - ), - ( - "sqlbuild._native:PythonIdentityStatus", - "sqlbuild.compiler.python_nodes.types:PythonIdentityStatus", - ), - ( - "sqlbuild._native:PythonNodeDependencyEdge", - "sqlbuild.compiler.python_nodes.models:PythonNodeDependencyEdge", - ), - ( - "sqlbuild._native:PythonNodeDiscoveryError", - "sqlbuild.compiler.discovery.exceptions:PythonNodeDiscoveryError", - ), - ( - "sqlbuild._native:PythonNodeGraph", - "sqlbuild.compiler.python_nodes.models:PythonNodeGraph", - ), - ( - "sqlbuild._native:PythonNodeIdentity", - "sqlbuild.compiler.python_nodes.models:PythonNodeIdentity", - ), - ( - "sqlbuild._native:PythonPlanEntry", - "sqlbuild.compiler.pipeline.models:PythonPlanEntry", - ), - ( - "sqlbuild._native:PythonRunPhase", - "sqlbuild.compiler.python_nodes.types:PythonRunPhase", - ), - ( - "sqlbuild._native:PythonRunPlanOutputs", - "sqlbuild.compiler.pipeline.models:PythonRunPlanOutputs", - ), - ( - "sqlbuild._native:PythonSqlReferenceReport", - "sqlbuild.compiler.compile.models:PythonSqlReferenceReport", - ), - ( - "sqlbuild._native:PythonSqlRunLifecyclePlan", - "sqlbuild.compiler.python_nodes.models:PythonSqlRunLifecyclePlan", - ), - ( - "sqlbuild._native:PythonSqlRunSelection", - "sqlbuild.compiler.python_nodes.models:PythonSqlRunSelection", - ), - ( - "sqlbuild._native:QualifiedLineageColumn", - "sqlbuild.compiler.lineage.models:QualifiedLineageColumn", - ), - ( - "sqlbuild._native:QueryFingerprintError", - "sqlbuild.compiler.fingerprints.exceptions:QueryFingerprintError", - ), - ( - "sqlbuild._native:RefactorIOError", - "sqlbuild.compiler.refactoring.exceptions:RefactorIOError", - ), - ( - "sqlbuild._native:RefactorProject", - "sqlbuild.compiler.refactoring.models:RefactorProject", - ), - ( - "sqlbuild._native:RefactorValueError", - "sqlbuild.compiler.refactoring.exceptions:RefactorValueError", - ), - ( - "sqlbuild._native:ReferencesConfig", - "sqlbuild.spec.contracts.models:ReferencesConfig", - ), - ( - "sqlbuild._native:RelatedLocation", - "sqlbuild.compiler.compile.models:RelatedLocation", - ), - ( - "sqlbuild._native:ResolvedDbtProfileOutput", - "sqlbuild.integrations.dbt.models:ResolvedDbtProfileOutput", - ), - ( - "sqlbuild._native:ResolvedTableType", - "sqlbuild.spec.contracts.models:ResolvedTableType", - ), - ( - "sqlbuild._native:ResolvedTimeTravelRetention", - "sqlbuild.spec.contracts.models:ResolvedTimeTravelRetention", - ), - ( - "sqlbuild._native:ResourceIdentity", - "sqlbuild.compiler.scopes.models:ResourceIdentity", - ), - ( - "sqlbuild._native:ResourceIdentityError", - "sqlbuild.compiler.resource_names.exceptions:ResourceIdentityError", - ), - ( - "sqlbuild._native:ResourceKind", - "sqlbuild.compiler.scopes.types:ResourceKind", - ), - ( - "sqlbuild._native:ResourceRecord", - "sqlbuild.compiler.scopes.models:ResourceRecord", - ), - ( - "sqlbuild._native:RetentionDecreasePolicy", - "sqlbuild.spec.contracts.types:RetentionDecreasePolicy", - ), - ( - "sqlbuild._native:RuleExemption", - "sqlbuild.rule_engine.models:RuleExemption", - ), - ( - "sqlbuild._native:RuleFixResult", - "sqlbuild.lint.models:RuleFixResult", - ), - ( - "sqlbuild._native:RuleGuidance", - "sqlbuild.rule_engine.models:RuleGuidance", - ), - ( - "sqlbuild._native:RuleIgnore", - "sqlbuild.rule_engine.models:RuleIgnore", - ), - ( - "sqlbuild._native:RuleRunPlan", - "sqlbuild.rule_engine.models:RuleRunPlan", - ), - ( - "sqlbuild._native:RulesCacheConfig", - "sqlbuild.rule_engine.models:RulesCacheConfig", - ), - ( - "sqlbuild._native:RulesRunResult", - "sqlbuild.rule_engine.models:RulesRunResult", - ), - ( - "sqlbuild._native:ScenarioConfig", - "sqlbuild.spec.contracts.models:ScenarioConfig", - ), - ( - "sqlbuild._native:ScenarioSnapshotLimitsConfig", - "sqlbuild.spec.contracts.models:ScenarioSnapshotLimitsConfig", - ), - ( - "sqlbuild._native:SchemaAuditInstance", - "sqlbuild.spec.contracts.models:SchemaAuditInstance", - ), - ( - "sqlbuild._native:SchemaColumn", - "sqlbuild.spec.contracts.models:SchemaColumn", - ), - ( - "sqlbuild._native:SchemaDynamicColumnFamily", - "sqlbuild.spec.contracts.models:SchemaDynamicColumnFamily", - ), - ( - "sqlbuild._native:SchemaModelEntry", - "sqlbuild.spec.contracts.models:SchemaModelEntry", - ), - ( - "sqlbuild._native:SchemaParseError", - "sqlbuild.compiler.discovery.exceptions:SchemaParseError", - ), - ( - "sqlbuild._native:SchemaSeedEntry", - "sqlbuild.spec.contracts.models:SchemaSeedEntry", - ), - ( - "sqlbuild._native:ScopeBrowseResult", - "sqlbuild.compiler.scopes.models:ScopeBrowseResult", - ), - ( - "sqlbuild._native:ScopeChainEntry", - "sqlbuild.compiler.scopes.models:ScopeChainEntry", - ), - ( - "sqlbuild._native:ScopeCompleteness", - "sqlbuild.compiler.scopes.models:ScopeCompleteness", - ), - ( - "sqlbuild._native:ScopeDiagnostic", - "sqlbuild.compiler.scopes.models:ScopeDiagnostic", - ), - ( - "sqlbuild._native:ScopeDiagnosticCode", - "sqlbuild.compiler.scopes.types:ScopeDiagnosticCode", - ), - ( - "sqlbuild._native:ScopeDiagnosticSeverity", - "sqlbuild.compiler.scopes.types:DiagnosticSeverity", - ), - ( - "sqlbuild._native:ScopeError", - "sqlbuild.compiler.scopes.exceptions:ScopeError", - ), - ( - "sqlbuild._native:ScopeFolder", - "sqlbuild.compiler.scopes.models:ScopeFolder", - ), - ( - "sqlbuild._native:ScopeIndex", - "sqlbuild.compiler.scopes.models:ScopeIndex", - ), - ( - "sqlbuild._native:ScopeKind", - "sqlbuild.compiler.scopes.types:ScopeKind", - ), - ( - "sqlbuild._native:ScopeListResult", - "sqlbuild.compiler.scopes.models:ScopeListResult", - ), - ( - "sqlbuild._native:ScopeLookup", - "sqlbuild.compiler.scopes.models:ScopeLookup", - ), - ( - "sqlbuild._native:ScopeReport", - "sqlbuild.compiler.scopes.models:ScopeReport", - ), - ( - "sqlbuild._native:ScopeReportFilters", - "sqlbuild.compiler.scopes.models:ScopeReportFilters", - ), - ( - "sqlbuild._native:ScopeResourceReport", - "sqlbuild.compiler.scopes.models:ScopeResourceReport", - ), - ( - "sqlbuild._native:ScopeSection", - "sqlbuild.compiler.scopes.models:ScopeSection", - ), - ( - "sqlbuild._native:ScopeSourceLocation", - "sqlbuild.compiler.scopes.models:SourceLocation", - ), - ( - "sqlbuild._native:ScopeTargetQuery", - "sqlbuild.compiler.scopes.models:ScopeTargetQuery", - ), - ( - "sqlbuild._native:ScopeValidationError", - "sqlbuild.compiler.scopes.exceptions:ScopeValidationError", - ), - ( - "sqlbuild._native:ScopesConfig", - "sqlbuild.spec.contracts.models:ScopesConfig", - ), - ( - "sqlbuild._native:SeedCsvSettings", - "sqlbuild.spec.contracts.models:SeedCsvSettings", - ), - ( - "sqlbuild._native:SeedDiscoveryError", - "sqlbuild.compiler.discovery.exceptions:SeedDiscoveryError", - ), - ( - "sqlbuild._native:SelectStarAllow", - "sqlbuild.rule_engine.models:SelectStarAllow", - ), - ( - "sqlbuild._native:SelectedDbtProfileOutput", - "sqlbuild.integrations.dbt.models:SelectedDbtProfileOutput", - ), - ( - "sqlbuild._native:SelectorKind", - "sqlbuild.compiler.planner.types:SelectorKind", - ), - ( - "sqlbuild._native:SettingsConfig", - "sqlbuild.spec.contracts.models:SettingsConfig", - ), - ( - "sqlbuild._native:SinksConfig", - "sqlbuild.spec.contracts.models:SinksConfig", - ), - ( - "sqlbuild._native:SnapshotsConfig", - "sqlbuild.spec.contracts.models:SnapshotsConfig", - ), - ( - "sqlbuild._native:SourceColumnEntry", - "sqlbuild.spec.contracts.models:SourceColumnEntry", - ), - ( - "sqlbuild._native:SourceEntry", - "sqlbuild.spec.contracts.models:SourceEntry", - ), - ( - "sqlbuild._native:SourceFreshnessAgePolicy", - "sqlbuild.spec.contracts.models:SourceFreshnessAgePolicy", - ), - ( - "sqlbuild._native:SourceFreshnessConfig", - "sqlbuild.spec.contracts.models:SourceFreshnessConfig", - ), - ( - "sqlbuild._native:SourceFreshnessRecord", - "sqlbuild.compiler.source_freshness.models:SourceFreshnessRecord", - ), - ( - "sqlbuild._native:SourceFreshnessStrategy", - "sqlbuild.spec.contracts.types:SourceFreshnessStrategy", - ), - ( - "sqlbuild._native:SourceFreshnessValueKind", - "sqlbuild.spec.contracts.types:SourceFreshnessValueKind", - ), - ( - "sqlbuild._native:SourceLocation", - "sqlbuild.spec.contracts.models:SourceLocation", - ), - ( - "sqlbuild._native:SourceParseError", - "sqlbuild.compiler.discovery.exceptions:SourceParseError", - ), - ( - "sqlbuild._native:SourceWriteStrategy", - "sqlbuild.spec.contracts.types:SourceWriteStrategy", - ), - ( - "sqlbuild._native:SpecConfigError", - "sqlbuild.spec.contracts.exceptions:SpecConfigError", - ), - ( - "sqlbuild._native:SqlAuditParseError", - "sqlbuild.compiler.discovery.exceptions:SqlAuditParseError", - ), - ( - "sqlbuild._native:SqlBindingDiagnostic", - "sqlbuild.compiler.sql_analysis.models:SqlBindingDiagnostic", - ), - ( - "sqlbuild._native:SqlBindingResult", - "sqlbuild.compiler.sql_analysis.models:SqlBindingResult", - ), - ( - "sqlbuild._native:SqlExpansionContext", - "sqlbuild.compiler.compile.models:SqlExpansionContext", - ), - ( - "sqlbuild._native:SqlExpansionReuse", - "sqlbuild.rule_engine.models:SqlExpansionReuse", - ), - ( - "sqlbuild._native:SqlHookParseError", - "sqlbuild.compiler.discovery.exceptions:SqlHookParseError", - ), - ( - "sqlbuild._native:SqlLexicalSyntax", - "sqlbuild.compiler.sql_analysis.models:SqlLexicalSyntax", - ), - ( - "sqlbuild._native:SqlLogicalType", - "sqlbuild.sql_values.models:SqlLogicalType", - ), - ( - "sqlbuild._native:SqlReferenceScan", - "sqlbuild.compiler.compile.models:SqlReferenceScan", - ), - ( - "sqlbuild._native:SqlRulesEvaluation", - "sqlbuild.rule_engine.models:SqlRulesEvaluation", - ), - ( - "sqlbuild._native:SqlScenarioParseError", - "sqlbuild.compiler.discovery.exceptions:SqlScenarioParseError", - ), - ( - "sqlbuild._native:SqlSchemaValidationRequest", - "sqlbuild.compiler.sql_analysis.models:SqlSchemaValidationRequest", - ), - ( - "sqlbuild._native:SqlTestAssertionStep", - "sqlbuild.compiler.planner.models:SqlTestAssertionStep", - ), - ( - "sqlbuild._native:SqlTestParseError", - "sqlbuild.compiler.discovery.exceptions:SqlTestParseError", - ), - ( - "sqlbuild._native:SqlTestRulesConfig", - "sqlbuild.rule_engine.models:SqlTestRulesConfig", - ), - ( - "sqlbuild._native:SqlTestSelection", - "sqlbuild.compiler.planner.models:SqlTestSelection", - ), - ( - "sqlbuild._native:SqlValue", - "sqlbuild.sql_values.models:SqlValue", - ), - ( - "sqlbuild._native:SqlValueKind", - "sqlbuild.sql_values.types:SqlValueKind", - ), - ( - "sqlbuild._native:StartCursorsConfig", - "sqlbuild.spec.contracts.models:StartCursorsConfig", - ), - ( - "sqlbuild._native:StateSqlValueType", - "sqlbuild.sql_values.types:StateSqlValueType", - ), - ( - "sqlbuild._native:StaticCommandContext", - "sqlbuild.compiler.pipeline.models:StaticCommandContext", - ), - ( - "sqlbuild._native:TableType", - "sqlbuild.spec.contracts.types:TableType", - ), - ( - "sqlbuild._native:TableTypeDowngradePolicy", - "sqlbuild.spec.contracts.types:TableTypeDowngradePolicy", - ), - ( - "sqlbuild._native:TableTypeSource", - "sqlbuild.spec.contracts.types:TableTypeSource", - ), - ( - "sqlbuild._native:TargetConfig", - "sqlbuild.spec.contracts.models:TargetConfig", - ), - ( - "sqlbuild._native:TargetDiffConfig", - "sqlbuild.spec.contracts.models:TargetDiffConfig", - ), - ( - "sqlbuild._native:TargetWarehousesConfig", - "sqlbuild.spec.contracts.models:TargetWarehousesConfig", - ), - ( - "sqlbuild._native:ThresholdOverride", - "sqlbuild.rule_engine.models:ThresholdOverride", - ), - ( - "sqlbuild._native:TimeTravelRetentionSource", - "sqlbuild.spec.contracts.types:TimeTravelRetentionSource", - ), - ( - "sqlbuild._native:TimestampValue", - "sqlbuild.cursor_algebra.models:TimestampValue", - ), - ( - "sqlbuild._native:TolerantScopeDiscovery", - "sqlbuild.compiler.discovery.models:TolerantScopeDiscovery", - ), - ( - "sqlbuild._native:UnindexableScopeError", - "sqlbuild.compiler.scopes.exceptions:UnindexableScopeError", - ), - ( - "sqlbuild._native:UnknownLineageDialectError", - "sqlbuild.compiler.lineage.exceptions:UnknownLineageDialectError", - ), - ( - "sqlbuild._native:UnsupportedLexicalSyntaxError", - "sqlbuild.compiler.sql_analysis.exceptions:UnsupportedLexicalSyntaxError", - ), - ( - "sqlbuild._native:UsageKind", - "sqlbuild.compiler.scopes.types:UsageKind", - ), - ( - "sqlbuild._native:UsageRecord", - "sqlbuild.compiler.scopes.models:UsageRecord", - ), - ( - "sqlbuild._native:VisibilityProvenance", - "sqlbuild.compiler.scopes.models:VisibilityProvenance", - ), - ( - "sqlbuild._native:VisibilityReason", - "sqlbuild.compiler.scopes.types:VisibilityReason", - ), - ( - "sqlbuild._native:VisibilityRecord", - "sqlbuild.compiler.scopes.models:VisibilityRecord", - ), - ( - "sqlbuild._native:VisibilityResolution", - "sqlbuild.compiler.scopes.models:VisibilityResolution", - ), - ( - "sqlbuild._native:VisibleModelDeclarations", - "sqlbuild.compiler.compile.models:VisibleModelDeclarations", - ), - ( - "sqlbuild._native:WarehouseGroup", - "sqlbuild.spec.contracts.types:WarehouseGroup", - ), -]; diff --git a/crates/sqlbuild-cache/src/stage_capture/tests.rs b/crates/sqlbuild-cache/src/stage_capture/tests.rs index 86958e472c..80c6c333f9 100644 --- a/crates/sqlbuild-cache/src/stage_capture/tests.rs +++ b/crates/sqlbuild-cache/src/stage_capture/tests.rs @@ -2,8 +2,6 @@ mod capture_document; #[path = "tests/helpers.rs"] mod helpers; -#[path = "tests/test_hosted_shape_types.rs"] -mod hosted_shape_types; #[path = "tests/test_mask_capture_noise.rs"] mod mask_capture_noise; #[path = "tests/test_types.rs"] diff --git a/crates/sqlbuild-cache/src/stage_capture/tests/helpers.rs b/crates/sqlbuild-cache/src/stage_capture/tests/helpers.rs index cbc968651b..59924b9967 100644 --- a/crates/sqlbuild-cache/src/stage_capture/tests/helpers.rs +++ b/crates/sqlbuild-cache/src/stage_capture/tests/helpers.rs @@ -1,5 +1,4 @@ use sqlbuild_core::json::models::{JsonInteger, JsonValue}; -use std::collections::HashSet; pub(crate) fn text(value: &str) -> JsonValue { JsonValue::String(value.to_owned()) @@ -8,12 +7,3 @@ pub(crate) fn text(value: &str) -> JsonValue { pub(crate) fn integer(value: i64) -> JsonValue { JsonValue::Integer(JsonInteger::from(value)) } - -/// The keys of `keys` seen more than once, in first-repeat order. -pub(super) fn duplicates(keys: &[&'static str]) -> Vec<&'static str> { - let mut seen: HashSet<&str> = HashSet::new(); - keys.iter() - .copied() - .filter(|key| !seen.insert(key)) - .collect() -} diff --git a/crates/sqlbuild-cache/src/stage_capture/tests/test_hosted_shape_types.rs b/crates/sqlbuild-cache/src/stage_capture/tests/test_hosted_shape_types.rs deleted file mode 100644 index e5d828e9c7..0000000000 --- a/crates/sqlbuild-cache/src/stage_capture/tests/test_hosted_shape_types.rs +++ /dev/null @@ -1,33 +0,0 @@ -use crate::stage_capture::constants::HOSTED_SHAPE_TYPES; -use crate::stage_capture::tests::helpers::duplicates; -use crate::stage_capture::tests::test_types::HostedKeyTestCase; - -#[test] -fn given_hosted_shape_table_when_keyed_then_every_native_and_python_name_is_unique() { - let test_cases = [ - HostedKeyTestCase { - description: "native class names", - keys: HOSTED_SHAPE_TYPES - .iter() - .map(|(native, _)| *native) - .collect(), - expected_duplicates: Vec::new(), - }, - HostedKeyTestCase { - description: "qualified Python shape names", - keys: HOSTED_SHAPE_TYPES - .iter() - .map(|(_, python)| *python) - .collect(), - expected_duplicates: Vec::new(), - }, - ]; - for test_case in test_cases { - assert_eq!( - duplicates(&test_case.keys), - test_case.expected_duplicates, - "{}", - test_case.description - ); - } -} diff --git a/crates/sqlbuild-cache/src/stage_capture/tests/test_types.rs b/crates/sqlbuild-cache/src/stage_capture/tests/test_types.rs index 25806f4d1b..9962247a4d 100644 --- a/crates/sqlbuild-cache/src/stage_capture/tests/test_types.rs +++ b/crates/sqlbuild-cache/src/stage_capture/tests/test_types.rs @@ -18,9 +18,3 @@ pub(crate) struct SortKeyTestCase { pub(crate) item: JsonValue, pub(crate) expected_key: &'static str, } - -pub(crate) struct HostedKeyTestCase { - pub(crate) description: &'static str, - pub(crate) keys: Vec<&'static str>, - pub(crate) expected_duplicates: Vec<&'static str>, -} diff --git a/crates/sqlbuild-cli/src/dispatch/constants.rs b/crates/sqlbuild-cli/src/dispatch/constants.rs index d61be461e5..e375fd4a63 100644 --- a/crates/sqlbuild-cli/src/dispatch/constants.rs +++ b/crates/sqlbuild-cli/src/dispatch/constants.rs @@ -13,7 +13,7 @@ pub(crate) const DIFF_COMMAND_NAME: &str = "diff"; const NATIVE_RECORDS: &str = "sqlbuild._native"; const ENTRY_MODELS: &str = "sqlbuild._native"; -const PLANNER_MODELS: &str = "sqlbuild.compiler.planner.models"; +const PLANNER_MODELS: &str = "sqlbuild._native"; const PROJECT_DIR: Field = field("project_dir", Source::ProjectDir); const SELECT: Field = field("select", Source::Select); diff --git a/crates/sqlbuild-core/src/lib.rs b/crates/sqlbuild-core/src/lib.rs index 4d3832c03d..a77c5ead45 100644 --- a/crates/sqlbuild-core/src/lib.rs +++ b/crates/sqlbuild-core/src/lib.rs @@ -32,6 +32,7 @@ pub mod integration_results; pub mod janitor_text; pub mod json; pub mod load_command; +pub mod migration_values; pub mod models; pub mod nested_progress; pub mod panics; diff --git a/crates/sqlbuild-core/src/migration_values/constants.rs b/crates/sqlbuild-core/src/migration_values/constants.rs new file mode 100644 index 0000000000..e1bcf6c806 --- /dev/null +++ b/crates/sqlbuild-core/src/migration_values/constants.rs @@ -0,0 +1,30 @@ +//! Migration decision vocabularies. + +/// Model decisions that clone the origin over the destination. +pub(crate) const MOVING_DECISIONS: [&str; 4] = + ["migrate", "redo", "superseded_replace", "forced_replace"]; +/// Model decisions that stop a build regardless of compatibility. +pub(crate) const BLOCKING_DECISIONS: [&str; 2] = ["conflict", "origin_missing"]; +pub(crate) const CONFLICT_DECISION: &str = "conflict"; +/// Column decisions that append a column migration event. +pub(crate) const RECORDING_COLUMN_DECISIONS: [&str; 2] = ["rename", "record"]; +pub(crate) const RENAME_COLUMN_DECISION: &str = "rename"; +/// Column decisions that stop a build before any execution. +pub(crate) const BLOCKING_COLUMN_DECISIONS: [&str; 4] = [ + "conflict", + "source_missing", + "still_produced", + "unsupported", +]; +/// Plan labels of the column decisions. +pub(crate) const COLUMN_DECISION_LABELS: [(&str, &str); 7] = [ + ("rename", "rename in place"), + ("record", "already renamed, record"), + ("done", "done"), + ("conflict", "conflict"), + ("source_missing", "source missing"), + ("still_produced", "source still produced"), + ("unsupported", "unsupported"), +]; +/// Snowflake table types by whether the table is transient. +pub(crate) const STORAGE_LABELS: [&str; 2] = ["permanent", "transient"]; diff --git a/crates/sqlbuild-core/src/migration_values/main/column_decision_traits.rs b/crates/sqlbuild-core/src/migration_values/main/column_decision_traits.rs new file mode 100644 index 0000000000..358d9fe467 --- /dev/null +++ b/crates/sqlbuild-core/src/migration_values/main/column_decision_traits.rs @@ -0,0 +1,21 @@ +//! What one column migration decision does. + +use crate::migration_values::constants::{ + BLOCKING_COLUMN_DECISIONS, COLUMN_DECISION_LABELS, RECORDING_COLUMN_DECISIONS, + RENAME_COLUMN_DECISION, +}; +use crate::migration_values::models::ColumnDecisionTraits; + +/// The behaviour of the column migration decision whose value is `decision`. +#[must_use] +pub fn column_decision_traits(decision: &str) -> ColumnDecisionTraits { + ColumnDecisionTraits { + renames: decision == RENAME_COLUMN_DECISION, + records_event: RECORDING_COLUMN_DECISIONS.contains(&decision), + blocks_build: BLOCKING_COLUMN_DECISIONS.contains(&decision), + label: COLUMN_DECISION_LABELS + .iter() + .find(|(value, _)| *value == decision) + .map(|(_, label)| *label), + } +} diff --git a/crates/sqlbuild-core/src/migration_values/main/migration_decision_traits.rs b/crates/sqlbuild-core/src/migration_values/main/migration_decision_traits.rs new file mode 100644 index 0000000000..75976c0ff1 --- /dev/null +++ b/crates/sqlbuild-core/src/migration_values/main/migration_decision_traits.rs @@ -0,0 +1,15 @@ +//! What one model migration decision does. + +use crate::migration_values::constants::{BLOCKING_DECISIONS, CONFLICT_DECISION, MOVING_DECISIONS}; +use crate::migration_values::models::DecisionTraits; + +/// The behaviour of the model migration decision whose value is `decision`. +#[must_use] +pub fn migration_decision_traits(decision: &str) -> DecisionTraits { + let moves_data: bool = MOVING_DECISIONS.contains(&decision); + DecisionTraits { + moves_data, + blocks_build: BLOCKING_DECISIONS.contains(&decision), + checks_compatibility: moves_data || decision == CONFLICT_DECISION, + } +} diff --git a/crates/sqlbuild-core/src/migration_values/main/mod.rs b/crates/sqlbuild-core/src/migration_values/main/mod.rs new file mode 100644 index 0000000000..c8c8b9703d --- /dev/null +++ b/crates/sqlbuild-core/src/migration_values/main/mod.rs @@ -0,0 +1,5 @@ +pub mod column_decision_traits; +pub mod migration_decision_traits; +pub mod model_migration_blocks_build; +pub mod plan_label; +pub mod storage_transition; diff --git a/crates/sqlbuild-core/src/migration_values/main/model_migration_blocks_build.rs b/crates/sqlbuild-core/src/migration_values/main/model_migration_blocks_build.rs new file mode 100644 index 0000000000..3e11b5c365 --- /dev/null +++ b/crates/sqlbuild-core/src/migration_values/main/model_migration_blocks_build.rs @@ -0,0 +1,12 @@ +//! Whether one planned model migration stops a build before any execution. + +use crate::migration_values::models::ModelMigrationFacts; + +/// A missing origin blocks when tracked or denied; otherwise the decision or an incompatible move blocks. +#[must_use] +pub fn model_migration_blocks_build(facts: &ModelMigrationFacts) -> bool { + if facts.origin_missing { + return facts.origin_tracked || facts.deny_missing_origin; + } + facts.decision.blocks_build || (facts.decision.moves_data && facts.incompatible) +} diff --git a/crates/sqlbuild-core/src/migration_values/main/plan_label.rs b/crates/sqlbuild-core/src/migration_values/main/plan_label.rs new file mode 100644 index 0000000000..83701478df --- /dev/null +++ b/crates/sqlbuild-core/src/migration_values/main/plan_label.rs @@ -0,0 +1,7 @@ +//! The plan label of a model migration decision or promotion. + +/// `value` with underscores shown as spaces. +#[must_use] +pub fn plan_label(value: &str) -> String { + value.replace('_', " ") +} diff --git a/crates/sqlbuild-core/src/migration_values/main/storage_transition.rs b/crates/sqlbuild-core/src/migration_values/main/storage_transition.rs new file mode 100644 index 0000000000..dfd8cf5aad --- /dev/null +++ b/crates/sqlbuild-core/src/migration_values/main/storage_transition.rs @@ -0,0 +1,17 @@ +//! The Snowflake table-type transition one model migration makes. + +use crate::migration_values::constants::STORAGE_LABELS; + +/// `origin -> stage` table types, or `None` when the stage's type is unknown. +#[must_use] +pub fn storage_transition( + origin_is_transient: bool, + stage_is_transient: Option, +) -> Option { + let stage: bool = stage_is_transient?; + Some(format!( + "{} -> {}", + STORAGE_LABELS[usize::from(origin_is_transient)], + STORAGE_LABELS[usize::from(stage)] + )) +} diff --git a/crates/sqlbuild-core/src/migration_values/mod.rs b/crates/sqlbuild-core/src/migration_values/mod.rs new file mode 100644 index 0000000000..19c466d2dd --- /dev/null +++ b/crates/sqlbuild-core/src/migration_values/mod.rs @@ -0,0 +1,8 @@ +//! Model and column migration decisions: what each decision does and when it blocks a build. + +pub(crate) mod constants; +pub mod main; +pub mod models; + +#[cfg(test)] +mod tests; diff --git a/crates/sqlbuild-core/src/migration_values/models.rs b/crates/sqlbuild-core/src/migration_values/models.rs new file mode 100644 index 0000000000..409ef75c16 --- /dev/null +++ b/crates/sqlbuild-core/src/migration_values/models.rs @@ -0,0 +1,35 @@ +//! What one migration decision does. + +/// The behaviour of one model migration decision. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub struct DecisionTraits { + /// The decision clones the origin over the destination. + pub moves_data: bool, + /// The decision stops a build regardless of compatibility. + pub blocks_build: bool, + /// The decision evaluated the origin against the destination. + pub checks_compatibility: bool, +} + +/// The behaviour of one column migration decision. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub struct ColumnDecisionTraits { + pub renames: bool, + pub records_event: bool, + pub blocks_build: bool, + /// The plan label, or `None` for an unknown decision. + pub label: Option<&'static str>, +} + +/// The facts of one planned model migration that decide whether it blocks a build. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub struct ModelMigrationFacts { + pub origin_missing: bool, + /// The missing origin has build history in this target. + pub origin_tracked: bool, + /// `missing_migration_origin` is `deny`. + pub deny_missing_origin: bool, + pub decision: DecisionTraits, + /// The origin is incompatible with the destination. + pub incompatible: bool, +} diff --git a/crates/sqlbuild-core/src/migration_values/tests.rs b/crates/sqlbuild-core/src/migration_values/tests.rs new file mode 100644 index 0000000000..04bacf2250 --- /dev/null +++ b/crates/sqlbuild-core/src/migration_values/tests.rs @@ -0,0 +1,4 @@ +#[path = "tests/test_migration_values.rs"] +mod migration_values; +#[path = "tests/test_types.rs"] +mod test_types; diff --git a/crates/sqlbuild-core/src/migration_values/tests/test_migration_values.rs b/crates/sqlbuild-core/src/migration_values/tests/test_migration_values.rs new file mode 100644 index 0000000000..3ee22cd32a --- /dev/null +++ b/crates/sqlbuild-core/src/migration_values/tests/test_migration_values.rs @@ -0,0 +1,146 @@ +use crate::migration_values::main::column_decision_traits::column_decision_traits; +use crate::migration_values::main::migration_decision_traits::migration_decision_traits; +use crate::migration_values::main::model_migration_blocks_build::model_migration_blocks_build; +use crate::migration_values::main::storage_transition::storage_transition; +use crate::migration_values::models::{ColumnDecisionTraits, DecisionTraits, ModelMigrationFacts}; +use crate::migration_values::tests::test_types::{ + BlocksBuildTestCase, ColumnDecisionTestCase, DecisionTestCase, TransitionTestCase, +}; + +const MOVING: DecisionTraits = DecisionTraits { + moves_data: true, + blocks_build: false, + checks_compatibility: true, +}; + +#[test] +fn given_model_decision_when_classifying_then_matches_python_properties() { + let test_cases = [ + DecisionTestCase { + description: "migrate moves data and checks compatibility", + decision: "migrate", + expected_traits: MOVING, + }, + DecisionTestCase { + description: "conflict blocks and checks compatibility", + decision: "conflict", + expected_traits: DecisionTraits { + moves_data: false, + blocks_build: true, + checks_compatibility: true, + }, + }, + DecisionTestCase { + description: "done does nothing", + decision: "done", + expected_traits: DecisionTraits { + moves_data: false, + blocks_build: false, + checks_compatibility: false, + }, + }, + ]; + for test_case in test_cases { + assert_eq!( + migration_decision_traits(test_case.decision), + test_case.expected_traits, + "{}", + test_case.description + ); + } +} + +#[test] +fn given_column_decision_when_classifying_then_matches_python_properties() { + let test_cases = [ + ColumnDecisionTestCase { + description: "rename renames and records", + decision: "rename", + expected_traits: ColumnDecisionTraits { + renames: true, + records_event: true, + blocks_build: false, + label: Some("rename in place"), + }, + }, + ColumnDecisionTestCase { + description: "a produced source blocks", + decision: "still_produced", + expected_traits: ColumnDecisionTraits { + renames: false, + records_event: false, + blocks_build: true, + label: Some("source still produced"), + }, + }, + ]; + for test_case in test_cases { + assert_eq!( + column_decision_traits(test_case.decision), + test_case.expected_traits, + "{}", + test_case.description + ); + } +} + +#[test] +fn given_migration_facts_when_checking_then_blocks_as_python_did() { + let facts = + |origin_missing, origin_tracked, deny_missing_origin, incompatible| ModelMigrationFacts { + origin_missing, + origin_tracked, + deny_missing_origin, + decision: MOVING, + incompatible, + }; + let test_cases = [ + BlocksBuildTestCase { + description: "a tracked missing origin blocks", + facts: facts(true, true, false, false), + expected_blocks: true, + }, + BlocksBuildTestCase { + description: "an untracked allowed missing origin passes", + facts: facts(true, false, false, true), + expected_blocks: false, + }, + BlocksBuildTestCase { + description: "an incompatible move blocks", + facts: facts(false, false, false, true), + expected_blocks: true, + }, + ]; + for test_case in test_cases { + assert_eq!( + model_migration_blocks_build(&test_case.facts), + test_case.expected_blocks, + "{}", + test_case.description + ); + } +} + +#[test] +fn given_table_types_when_labelling_then_shows_the_transition() { + let test_cases = [ + TransitionTestCase { + description: "a permanent origin staged as transient", + types: (false, Some(true)), + expected_transition: Some("permanent -> transient"), + }, + TransitionTestCase { + description: "an unknown stage type has no label", + types: (true, None), + expected_transition: None, + }, + ]; + for test_case in test_cases { + assert_eq!( + storage_transition(test_case.types.0, test_case.types.1).as_deref(), + test_case.expected_transition, + "{}", + test_case.description + ); + } +} diff --git a/crates/sqlbuild-core/src/migration_values/tests/test_types.rs b/crates/sqlbuild-core/src/migration_values/tests/test_types.rs new file mode 100644 index 0000000000..871d0a9408 --- /dev/null +++ b/crates/sqlbuild-core/src/migration_values/tests/test_types.rs @@ -0,0 +1,25 @@ +use crate::migration_values::models::{ColumnDecisionTraits, DecisionTraits, ModelMigrationFacts}; + +pub(super) struct DecisionTestCase { + pub(super) description: &'static str, + pub(super) decision: &'static str, + pub(super) expected_traits: DecisionTraits, +} + +pub(super) struct ColumnDecisionTestCase { + pub(super) description: &'static str, + pub(super) decision: &'static str, + pub(super) expected_traits: ColumnDecisionTraits, +} + +pub(super) struct BlocksBuildTestCase { + pub(super) description: &'static str, + pub(super) facts: ModelMigrationFacts, + pub(super) expected_blocks: bool, +} + +pub(super) struct TransitionTestCase { + pub(super) description: &'static str, + pub(super) types: (bool, Option), + pub(super) expected_transition: Option<&'static str>, +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/analysis_session/model_analyses.rs b/crates/sqlbuild-python/src/bindings/_helpers/analysis_session/model_analyses.rs index 9dadfd103b..69ce5af6c6 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/analysis_session/model_analyses.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/analysis_session/model_analyses.rs @@ -16,7 +16,7 @@ use crate::bindings::_helpers::compile_driver::analysis_cache::record_analysis_c const NATIVE_MODULE: &str = "sqlbuild._native"; const ANALYSIS_MODELS_MODULE: &str = "sqlbuild._native"; -const PLANNER_TYPES_MODULE: &str = "sqlbuild.compiler.planner.types"; +const PLANNER_TYPES_MODULE: &str = "sqlbuild._native"; const LINEAGE_TYPES_MODULE: &str = "sqlbuild.compiler.lineage.types"; const LOG_DEBUG_MODULE: &str = "sqlbuild.diagnostics.main.log_debug_event"; const LOGGING_MODULE: &str = "logging"; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/build_command/execution.rs b/crates/sqlbuild-python/src/bindings/_helpers/build_command/execution.rs index 7b59a30828..2cdade0728 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/build_command/execution.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/build_command/execution.rs @@ -24,7 +24,7 @@ use crate::bindings::_helpers::graph::python_node_graph::python_node_graph; const NATIVE_MODULE: &str = "sqlbuild._native"; const CURSOR_OVERRIDES_MODULE: &str = "sqlbuild._native"; -const PLANNER_MODELS: &str = "sqlbuild.compiler.planner.models"; +const PLANNER_MODELS: &str = "sqlbuild._native"; const BUILD_MODELS: &str = "sqlbuild.executor.build.models"; const PIPELINE_RUN_MODULE: &str = "sqlbuild.executor.pipeline.main.run"; const FRESHNESS_MODULE: &str = "sqlbuild.cli.commands._helpers.freshness.source_freshness"; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compile_command/target_writer.rs b/crates/sqlbuild-python/src/bindings/_helpers/compile_command/target_writer.rs index a71989a2db..532e6d8a22 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compile_command/target_writer.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compile_command/target_writer.rs @@ -49,7 +49,7 @@ const PUBLISHED_TREE: &str = "tree"; const OUTPUT_MODELS_MODULE: &str = "sqlbuild._native"; const CLI_EXCEPTIONS_MODULE: &str = "sqlbuild._native"; const COMPILE_MODELS_MODULE: &str = "sqlbuild.compiler.compile.models"; -const PLANNER_EXCEPTIONS_MODULE: &str = "sqlbuild.compiler.planner.exceptions"; +const PLANNER_EXCEPTIONS_MODULE: &str = "sqlbuild._native"; const MODEL_CHAIN_MODULE: &str = "sqlbuild._native"; const PLAN_ARTIFACTS_MODULE: &str = "sqlbuild._native"; const LIFECYCLE_MODULE: &str = "sqlbuild.runtime.observability.classes.operation_lifecycle"; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/assembly_analysis.rs b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/assembly_analysis.rs index e58b1509b1..fbe498fcd0 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/assembly_analysis.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/assembly_analysis.rs @@ -19,7 +19,7 @@ const NATIVE_MODULE: &str = "sqlbuild._native"; const ADAPTER_MODELS_MODULE: &str = "sqlbuild.adapter.contract.models"; pub(crate) const ANALYSIS_MODELS_MODULE: &str = "sqlbuild._native"; const LINEAGE_TYPES_MODULE: &str = "sqlbuild.compiler.lineage.types"; -const PLANNER_TYPES_MODULE: &str = "sqlbuild.compiler.planner.types"; +const PLANNER_TYPES_MODULE: &str = "sqlbuild._native"; const GENERIC_DIALECT: &str = "generic"; const UDF_PREFIX: &str = "__sqlbuild_udf_"; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/deferral.rs b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/deferral.rs index f3b8062bda..6385633870 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/deferral.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/deferral.rs @@ -15,7 +15,7 @@ use crate::bindings::_helpers::compile_driver::python_calls::{ const NATIVE_RECORDS_MODULE: &str = "sqlbuild._native"; -const PLANNER_EXCEPTIONS_MODULE: &str = "sqlbuild.compiler.planner.exceptions"; +const PLANNER_EXCEPTIONS_MODULE: &str = "sqlbuild._native"; const COMPILE_MODELS_MODULE: &str = "sqlbuild.compiler.compile.models"; const PROJECT_VARS_MODULE: &str = "sqlbuild._native"; const PRESERVE_TARGET_VALUE: &str = "preserve"; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/diff.rs b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/diff.rs index b45acacb9a..0787d960c5 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/diff.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/diff.rs @@ -18,7 +18,7 @@ use crate::bindings::_helpers::compile_driver::python_calls::{ use crate::bindings::_helpers::graph::views::project_lineage_views; const NATIVE_MODULE: &str = "sqlbuild._native"; -const PLANNER_EXCEPTIONS_MODULE: &str = "sqlbuild.compiler.planner.exceptions"; +const PLANNER_EXCEPTIONS_MODULE: &str = "sqlbuild._native"; const COMPILE_TYPES_MODULE: &str = "sqlbuild.compiler.compile.types"; /// Indexes of the `CompiledObjectKey`s a diff graph names, assigned on first sight. diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/phases.rs b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/phases.rs index 075ad1d7af..255780afe9 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/phases.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/phases.rs @@ -19,7 +19,7 @@ const NATIVE_MODULE: &str = "sqlbuild._native"; const ADAPTER_NAME_MODULE: &str = "sqlbuild._native"; const OUTPUT_MODELS_MODULE: &str = "sqlbuild._native"; const COMPILE_EXCEPTIONS_MODULE: &str = "sqlbuild.compiler.compile.exceptions"; -const PLANNER_EXCEPTIONS_MODULE: &str = "sqlbuild.compiler.planner.exceptions"; +const PLANNER_EXCEPTIONS_MODULE: &str = "sqlbuild._native"; const COMPILE_MODELS_MODULE: &str = "sqlbuild.compiler.compile.models"; const EMPTY_DAG_PATH: &str = ""; const TARGET_DIRECTORY: &str = "target"; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/pipeline_entries.rs b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/pipeline_entries.rs index 2a9f54957d..ce5ca3fe7d 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/pipeline_entries.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/pipeline_entries.rs @@ -11,7 +11,7 @@ use crate::bindings::_helpers::compile_driver::python_calls::{ const NATIVE_RECORDS_MODULE: &str = "sqlbuild._native"; const PIPELINE_MODELS_MODULE: &str = "sqlbuild.compiler.pipeline.models"; -const PLANNER_MODELS_MODULE: &str = "sqlbuild.compiler.planner.models"; +const PLANNER_MODELS_MODULE: &str = "sqlbuild._native"; const NATIVE_MODULE: &str = "sqlbuild._native"; /// Discovered Python node functions whose selectors need the Python graph. const PYTHON_NODE_FUNCTIONS: [&str; 4] = [ diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/targets.rs b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/targets.rs index d64171676c..f0b8d9983b 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/targets.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/targets.rs @@ -25,7 +25,7 @@ use crate::bindings::_helpers::compile_driver::python_calls::{ }; use crate::bindings::_helpers::model_config::template_data::expanded_template_data; -const PLANNER_EXCEPTIONS_MODULE: &str = "sqlbuild.compiler.planner.exceptions"; +const PLANNER_EXCEPTIONS_MODULE: &str = "sqlbuild._native"; const TARGET_NAME_MODULE: &str = "sqlbuild._native"; const TARGET_CONFIG_MODULE: &str = "sqlbuild._native"; const EFFECTIVE_RUNTIME_MODULE: &str = "sqlbuild._native"; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/loop_classes.rs b/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/loop_classes.rs index c07b76c9c6..6021852ebf 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/loop_classes.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/loop_classes.rs @@ -14,7 +14,7 @@ const COMPILE_ERRORS_MODULE: &str = "sqlbuild.compiler.compile.exceptions"; const AUDITING_TYPES_MODULE: &str = "sqlbuild.compiler.auditing.types"; const DISCOVERY_MODELS_MODULE: &str = "sqlbuild.compiler.discovery.models"; const LINEAGE_TYPES_MODULE: &str = "sqlbuild.compiler.lineage.types"; -const PLANNER_TYPES_MODULE: &str = "sqlbuild.compiler.planner.types"; +const PLANNER_TYPES_MODULE: &str = "sqlbuild._native"; const REFERENCE_TYPES_MODULE: &str = "sqlbuild.compiler.references.types"; const SCOPE_ERRORS_MODULE: &str = "sqlbuild._native"; const SCOPE_MODELS_MODULE: &str = "sqlbuild._native"; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compiler_enums/constants.rs b/crates/sqlbuild-python/src/bindings/_helpers/compiler_enums/constants.rs index 98b0933030..80c9460645 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compiler_enums/constants.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compiler_enums/constants.rs @@ -442,6 +442,240 @@ const E_STATE_SQL_VALUE_TYPE: EnumSpec = EnumSpec { ("JSON", "json"), ], }; +const E_MIGRATION_DECISION: EnumSpec = EnumSpec { + name: "MigrationDecision", + doc: "Per-run decision for one declared or discovered model migration.", + members: &[ + ("DONE", "done"), + ("MIGRATE", "migrate"), + ("REDO", "redo"), + ("SUPERSEDED_REPLACE", "superseded_replace"), + ("FORCED_REPLACE", "forced_replace"), + ("CONFLICT", "conflict"), + ("ORIGIN_MISSING", "origin_missing"), + ("RENAMED", "renamed"), + ], +}; +const E_MIGRATION_PROMOTION: EnumSpec = EnumSpec { + name: "MigrationPromotion", + doc: "How a completed migration stage replaces or creates the destination.", + members: &[ + ("SWAP", "swap"), + ("RENAME", "rename"), + ("TRANSACTIONAL_RENAME", "transactional_rename"), + ], +}; +const E_COLUMN_MIGRATION_DECISION: EnumSpec = EnumSpec { + name: "ColumnMigrationDecision", + doc: "Per-run decision for one declared or detected column rename.", + members: &[ + ("RENAME", "rename"), + ("RECORD", "record"), + ("DONE", "done"), + ("CONFLICT", "conflict"), + ("SOURCE_MISSING", "source_missing"), + ("STILL_PRODUCED", "still_produced"), + ("UNSUPPORTED", "unsupported"), + ], +}; +const E_OLD_NAME_VIEW_EVENT_TYPE: EnumSpec = EnumSpec { + name: "OldNameViewEventType", + doc: "Immutable facts about what happened at a migrated relation's old name.", + members: &[ + ("REQUIRED", "required"), + ("ORIGIN_ARCHIVED", "origin_archived"), + ("VIEW_CREATED", "view_created"), + ("VIEW_DROPPED", "view_dropped"), + ], +}; +const E_OLD_NAME_VIEW_DROP_REASON: EnumSpec = EnumSpec { + name: "OldNameViewDropReason", + doc: "Why a compatibility view at an old name was dropped.", + members: &[ + ("EXPIRED", "expired"), + ("EARLY", "early"), + ("MISSING", "missing"), + ], +}; +const E_OLD_NAME_VIEW_STATUS: EnumSpec = EnumSpec { + name: "OldNameViewStatus", + doc: "Projected state of the old-name steps of one recorded move.", + members: &[ + ("PENDING_ARCHIVE", "pending_archive"), + ("PENDING_VIEW", "pending_view"), + ("LIVE", "live"), + ("EXPIRED", "expired"), + ("DROPPED", "dropped"), + ], +}; +const E_BACKFILL_ACTION: EnumSpec = EnumSpec { + name: "BackfillAction", + doc: "", + members: &[ + ("FULL", "full"), + ("BOUNDED", "bounded"), + ("FORWARD_ONLY", "forward"), + ], +}; +const E_ON_SCHEMA_CHANGE: EnumSpec = EnumSpec { + name: "OnSchemaChange", + doc: "", + members: &[ + ("IGNORE", "ignore"), + ("FAIL", "fail"), + ("APPEND_NEW_COLUMNS", "append_new_columns"), + ("SYNC_ALL_COLUMNS", "sync_all_columns"), + ], +}; +const E_CONTRACT_POLICY: EnumSpec = EnumSpec { + name: "ContractPolicy", + doc: "", + members: &[("NONE", "none"), ("ENFORCED", "enforced")], +}; +const E_RETENTION_DIRECTION: EnumSpec = EnumSpec { + name: "RetentionDirection", + doc: "", + members: &[ + ("INCREASE", "increase"), + ("DECREASE", "decrease"), + ("MIXED", "mixed"), + ("MATCH", "match"), + ("APPLY_AFTER_CREATE", "apply_after_create"), + ], +}; +const E_RETENTION_PLAN_PHASE: EnumSpec = EnumSpec { + name: "RetentionPlanPhase", + doc: "", + members: &[ + ("PRE", "pre"), + ("POST", "post"), + ("AFTER_CREATE", "after_create"), + ("NONE", "none"), + ], +}; +const E_PLAN_ACTION: EnumSpec = EnumSpec { + name: "PlanAction", + doc: "", + members: &[ + ("CREATE_VIEW", "create_view"), + ("CREATE_TABLE", "create_table"), + ("INCREMENTAL_APPEND", "incremental_append"), + ("INCREMENTAL_DELETE_INSERT", "incremental_delete_insert"), + ("INCREMENTAL_MERGE", "incremental_merge"), + ("SNAPSHOT", "snapshot"), + ("LOAD_SEED", "load_seed"), + ("SKIP", "skip"), + ("CUSTOM", "custom"), + ], +}; +const E_PLAN_REASON: EnumSpec = EnumSpec { + name: "PlanReason", + doc: "", + members: &[ + ("FIRST_RUN", "first_run"), + ("RENAMED", "renamed"), + ("FULL_REFRESH", "full_refresh"), + ("QUERY_CHANGED", "query_changed"), + ("FUNCTION_CHANGED", "function_changed"), + ("CONFIG_CHANGED", "config_changed"), + ("SCHEMA_CHANGED", "schema_changed"), + ("NORMAL_INCREMENTAL", "normal_incremental"), + ("NO_CHANGE", "no_change"), + ("DISABLED", "disabled"), + ("SOURCE_FRESHNESS_ERROR", "source_freshness_error"), + ("EXTERNAL_UPSTREAM_FAILED", "external_upstream_failed"), + ], +}; +const E_INCREMENTAL_STRATEGY: EnumSpec = EnumSpec { + name: "IncrementalStrategy", + doc: "", + members: &[ + ("APPEND", "append"), + ("DELETE_INSERT", "delete_insert"), + ("MERGE", "merge"), + ], +}; +const E_MICROBATCH_STRATEGY: EnumSpec = EnumSpec { + name: "MicrobatchStrategy", + doc: "How an ordinary microbatch range is selected.", + members: &[ + ("ROLLING_WINDOW", "rolling_window"), + ("WATERMARK", "watermark"), + ], +}; +const E_INCREMENTAL_MODE: EnumSpec = EnumSpec { + name: "IncrementalMode", + doc: "", + members: &[("FULL", "full"), ("MICROBATCH", "microbatch")], +}; +const E_SNAPSHOT_STRATEGY: EnumSpec = EnumSpec { + name: "SnapshotStrategy", + doc: "", + members: &[("TIMESTAMP", "timestamp"), ("CHECK", "check")], +}; +const E_HISTORICAL_INPUT: EnumSpec = EnumSpec { + name: "HistoricalInput", + doc: "", + members: &[("SNAPSHOT", "snapshot"), ("CHANGES", "changes")], +}; +const E_INITIAL_VALID_FROM: EnumSpec = EnumSpec { + name: "InitialValidFrom", + doc: "", + members: &[ + ("UPDATED_AT", "updated_at"), + ("OBSERVED_AT", "observed_at"), + ("EXECUTION_TIME", "execution_time"), + ], +}; +const E_SNAPSHOT_SCHEMA_CHANGE_POLICY: EnumSpec = EnumSpec { + name: "SnapshotSchemaChangePolicy", + doc: "", + members: &[ + ("DENY", "deny"), + ("REQUIRE_CONFIRMATION", "require_confirmation"), + ("APPEND_NEW_COLUMNS", "append_new_columns"), + ], +}; +const E_CURSOR_TYPE: EnumSpec = EnumSpec { + name: "CursorType", + doc: "", + members: &[("TIMESTAMP", "timestamp"), ("INTEGER", "integer")], +}; +const E_CURSOR_GRAIN: EnumSpec = EnumSpec { + name: "CursorGrain", + doc: "", + members: &[ + ("SECOND", "second"), + ("MINUTE", "minute"), + ("HOUR", "hour"), + ("DAY", "day"), + ("MONTH", "month"), + ("YEAR", "year"), + ], +}; +const E_MATERIALIZATION_TYPE: EnumSpec = EnumSpec { + name: "MaterializationType", + doc: "", + members: &[ + ("VIEW", "view"), + ("TABLE", "table"), + ("INCREMENTAL", "incremental"), + ("SNAPSHOT", "snapshot"), + ("SEED", "seed"), + ("CUSTOM", "custom"), + ], +}; +const E_SCENARIO_ARTIFACT_KIND: EnumSpec = EnumSpec { + name: "ScenarioArtifactKind", + doc: "", + members: &[ + ("SOURCE", "source"), + ("REF", "ref"), + ("SEED", "seed"), + ("DBT_REF", "dbt_ref"), + ("MODEL", "model"), + ], +}; /// Compiler, rule, lint and config string enums. pub(crate) const COMPILER_ENUMS: &[EnumSpec] = &[ @@ -488,4 +722,28 @@ pub(crate) const COMPILER_ENUMS: &[EnumSpec] = &[ E_RETENTION_DECREASE_POLICY, E_SQL_VALUE_KIND, E_STATE_SQL_VALUE_TYPE, + E_MIGRATION_DECISION, + E_MIGRATION_PROMOTION, + E_COLUMN_MIGRATION_DECISION, + E_OLD_NAME_VIEW_EVENT_TYPE, + E_OLD_NAME_VIEW_DROP_REASON, + E_OLD_NAME_VIEW_STATUS, + E_BACKFILL_ACTION, + E_ON_SCHEMA_CHANGE, + E_CONTRACT_POLICY, + E_RETENTION_DIRECTION, + E_RETENTION_PLAN_PHASE, + E_PLAN_ACTION, + E_PLAN_REASON, + E_INCREMENTAL_STRATEGY, + E_MICROBATCH_STRATEGY, + E_INCREMENTAL_MODE, + E_SNAPSHOT_STRATEGY, + E_HISTORICAL_INPUT, + E_INITIAL_VALID_FROM, + E_SNAPSHOT_SCHEMA_CHANGE_POLICY, + E_CURSOR_TYPE, + E_CURSOR_GRAIN, + E_MATERIALIZATION_TYPE, + E_SCENARIO_ARTIFACT_KIND, ]; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compiler_errors/constants.rs b/crates/sqlbuild-python/src/bindings/_helpers/compiler_errors/constants.rs index 78d9a96fef..fc73ecff02 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compiler_errors/constants.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compiler_errors/constants.rs @@ -11,6 +11,7 @@ pub(crate) const COMPILER_ERRORS: &[ErrorSpec] = &[ doc: "Base error for expected discovery failures.", class_attributes: &[("code", ClassValue::Text("D000"))], required: &[], + optional: &[], fixed: &[], }, ErrorSpec { @@ -19,6 +20,7 @@ pub(crate) const COMPILER_ERRORS: &[ErrorSpec] = &[ doc: "Raised when shared or local project config files are invalid.", class_attributes: &[("code", ClassValue::Text("D001"))], required: &[], + optional: &[], fixed: &[], }, ErrorSpec { @@ -27,6 +29,7 @@ pub(crate) const COMPILER_ERRORS: &[ErrorSpec] = &[ doc: "Raised when an authored SQL model file has an invalid shape.", class_attributes: &[("code", ClassValue::Text("D002"))], required: &[], + optional: &[], fixed: &[], }, ErrorSpec { @@ -35,6 +38,7 @@ pub(crate) const COMPILER_ERRORS: &[ErrorSpec] = &[ doc: "Raised when an authored SQL-native test file has an invalid shape.", class_attributes: &[("code", ClassValue::Text("D003"))], required: &[], + optional: &[], fixed: &[], }, ErrorSpec { @@ -43,6 +47,7 @@ pub(crate) const COMPILER_ERRORS: &[ErrorSpec] = &[ doc: "Raised when an authored SQL audit file has an invalid shape.", class_attributes: &[("code", ClassValue::Text("D004"))], required: &[], + optional: &[], fixed: &[], }, ErrorSpec { @@ -51,6 +56,7 @@ pub(crate) const COMPILER_ERRORS: &[ErrorSpec] = &[ doc: "Raised when a schema.yml file has an invalid shape.", class_attributes: &[("code", ClassValue::Text("D005"))], required: &[], + optional: &[], fixed: &[], }, ErrorSpec { @@ -59,6 +65,7 @@ pub(crate) const COMPILER_ERRORS: &[ErrorSpec] = &[ doc: "Raised when a sources/*.yml file has an invalid shape.", class_attributes: &[("code", ClassValue::Text("D006"))], required: &[], + optional: &[], fixed: &[], }, ErrorSpec { @@ -67,6 +74,7 @@ pub(crate) const COMPILER_ERRORS: &[ErrorSpec] = &[ doc: "Raised when discovered project inputs conflict across files.", class_attributes: &[("code", ClassValue::Text("D007"))], required: &[], + optional: &[], fixed: &[], }, ErrorSpec { @@ -75,6 +83,7 @@ pub(crate) const COMPILER_ERRORS: &[ErrorSpec] = &[ doc: "Raised when declared seed metadata does not match local seed files.", class_attributes: &[("code", ClassValue::Text("D008"))], required: &[], + optional: &[], fixed: &[], }, ErrorSpec { @@ -83,6 +92,7 @@ pub(crate) const COMPILER_ERRORS: &[ErrorSpec] = &[ doc: "Raised when an authored SQL scenario file has an invalid shape.", class_attributes: &[("code", ClassValue::Text("D009"))], required: &[], + optional: &[], fixed: &[], }, ErrorSpec { @@ -91,6 +101,7 @@ pub(crate) const COMPILER_ERRORS: &[ErrorSpec] = &[ doc: "Raised when project Python nodes cannot be discovered.", class_attributes: &[("code", ClassValue::Text("D011"))], required: &[], + optional: &[], fixed: &[], }, ErrorSpec { @@ -99,6 +110,7 @@ pub(crate) const COMPILER_ERRORS: &[ErrorSpec] = &[ doc: "Raised when project providers cannot be discovered.", class_attributes: &[("code", ClassValue::Text("D012"))], required: &[], + optional: &[], fixed: &[], }, ErrorSpec { @@ -107,6 +119,7 @@ pub(crate) const COMPILER_ERRORS: &[ErrorSpec] = &[ doc: "Raised when an authored public SQL declaration is invalid.", class_attributes: &[("code", ClassValue::Text("D013"))], required: &[], + optional: &[], fixed: &[], }, ErrorSpec { @@ -115,6 +128,7 @@ pub(crate) const COMPILER_ERRORS: &[ErrorSpec] = &[ doc: "Raised when an authored SQL hook resource has an invalid shape.", class_attributes: &[("code", ClassValue::Text("D014"))], required: &[], + optional: &[], fixed: &[], }, ErrorSpec { @@ -123,6 +137,7 @@ pub(crate) const COMPILER_ERRORS: &[ErrorSpec] = &[ doc: "Raised when project lifecycle event exporters are invalid.", class_attributes: &[("code", ClassValue::Text("D015"))], required: &[], + optional: &[], fixed: &[], }, ErrorSpec { @@ -131,6 +146,7 @@ pub(crate) const COMPILER_ERRORS: &[ErrorSpec] = &[ doc: "Raised when project-owned Python lives outside a supported extension root.", class_attributes: &[("code", ClassValue::Text("D016"))], required: &[], + optional: &[], fixed: &[], }, ErrorSpec { @@ -139,6 +155,7 @@ pub(crate) const COMPILER_ERRORS: &[ErrorSpec] = &[ doc: "Raised when an authored project path cannot be read as valid UTF-8.", class_attributes: &[("code", ClassValue::Text("D016"))], required: &[], + optional: &[], fixed: &[], }, ErrorSpec { @@ -147,6 +164,7 @@ pub(crate) const COMPILER_ERRORS: &[ErrorSpec] = &[ doc: "Raised when an authored resource identity is not canonical snake_case.", class_attributes: &[("code", ClassValue::Text("D016"))], required: &[], + optional: &[], fixed: &[], }, ErrorSpec { @@ -155,6 +173,7 @@ pub(crate) const COMPILER_ERRORS: &[ErrorSpec] = &[ doc: "Raised when a measurement audit contract or value is invalid.", class_attributes: &[], required: &[], + optional: &[], fixed: &[], }, ErrorSpec { @@ -163,6 +182,7 @@ pub(crate) const COMPILER_ERRORS: &[ErrorSpec] = &[ doc: "Raised when safe contract source editing cannot continue.", class_attributes: &[], required: &[], + optional: &[], fixed: &[], }, ErrorSpec { @@ -171,6 +191,7 @@ pub(crate) const COMPILER_ERRORS: &[ErrorSpec] = &[ doc: "Raised when fingerprint storage inputs or rows are invalid.", class_attributes: &[("code", ClassValue::Text("F000"))], required: &[], + optional: &[], fixed: &[], }, ErrorSpec { @@ -179,6 +200,7 @@ pub(crate) const COMPILER_ERRORS: &[ErrorSpec] = &[ doc: "Raised when SQL cannot be tokenized into a change fingerprint.", class_attributes: &[], required: &[], + optional: &[], fixed: &[], }, ErrorSpec { @@ -187,6 +209,7 @@ pub(crate) const COMPILER_ERRORS: &[ErrorSpec] = &[ doc: "Raised when native state the compiler replays is inconsistent with its inputs.", class_attributes: &[], required: &[], + optional: &[], fixed: &[], }, ErrorSpec { @@ -195,6 +218,7 @@ pub(crate) const COMPILER_ERRORS: &[ErrorSpec] = &[ doc: "Raised, as Python's parser raised `ValueError`, for a dialect Polyglot does not know.", class_attributes: &[], required: &[], + optional: &[], fixed: &[], }, ErrorSpec { @@ -203,6 +227,7 @@ pub(crate) const COMPILER_ERRORS: &[ErrorSpec] = &[ doc: "An unhandled value failure the Python planner also leaves to the caller.", class_attributes: &[], required: &[], + optional: &[], fixed: &[], }, ErrorSpec { @@ -211,6 +236,7 @@ pub(crate) const COMPILER_ERRORS: &[ErrorSpec] = &[ doc: "A file read or write failure the Python planner also leaves to the caller.", class_attributes: &[], required: &[], + optional: &[], fixed: &[], }, ErrorSpec { @@ -219,6 +245,7 @@ pub(crate) const COMPILER_ERRORS: &[ErrorSpec] = &[ doc: "Base error for invalid scope-domain input.", class_attributes: &[("code", ClassValue::Text("S000"))], required: &[], + optional: &[], fixed: &[], }, ErrorSpec { @@ -227,6 +254,7 @@ pub(crate) const COMPILER_ERRORS: &[ErrorSpec] = &[ doc: "Raised when a project-relative scope path is invalid.", class_attributes: &[("code", ClassValue::Text("S011"))], required: &[], + optional: &[], fixed: &[], }, ErrorSpec { @@ -235,6 +263,7 @@ pub(crate) const COMPILER_ERRORS: &[ErrorSpec] = &[ doc: "Raised when a qualified scope identity cannot be parsed.", class_attributes: &[("code", ClassValue::Text("S002"))], required: &[], + optional: &[], fixed: &[], }, ErrorSpec { @@ -243,6 +272,7 @@ pub(crate) const COMPILER_ERRORS: &[ErrorSpec] = &[ doc: "Raised when the native scope index cannot place a discovered path or resource kind.", class_attributes: &[], required: &[], + optional: &[], fixed: &[], }, ErrorSpec { @@ -251,6 +281,7 @@ pub(crate) const COMPILER_ERRORS: &[ErrorSpec] = &[ doc: "Aggregate error raised only when strict scope validation is requested.", class_attributes: &[], required: &["diagnostics"], + optional: &[], fixed: &[], }, ErrorSpec { @@ -259,6 +290,7 @@ pub(crate) const COMPILER_ERRORS: &[ErrorSpec] = &[ doc: "Raised when an adapter declares lexical rules the native SQL scanners do not support.", class_attributes: &[], required: &[], + optional: &[], fixed: &[], }, ErrorSpec { @@ -267,6 +299,7 @@ pub(crate) const COMPILER_ERRORS: &[ErrorSpec] = &[ doc: "A lint or format run could not proceed.", class_attributes: &[("code", ClassValue::Text("L001"))], required: &[], + optional: &[], fixed: &[], }, ErrorSpec { @@ -275,6 +308,7 @@ pub(crate) const COMPILER_ERRORS: &[ErrorSpec] = &[ doc: "The SQLBuild-owned native lint engine rejected a request or response.", class_attributes: &[("code", ClassValue::Text("L003"))], required: &[], + optional: &[], fixed: &[], }, ErrorSpec { @@ -283,6 +317,7 @@ pub(crate) const COMPILER_ERRORS: &[ErrorSpec] = &[ doc: "The project could not be compiled, so its SQL cannot be linted.", class_attributes: &[("code", ClassValue::Text("L005"))], required: &[], + optional: &[], fixed: &[], }, ErrorSpec { @@ -291,6 +326,7 @@ pub(crate) const COMPILER_ERRORS: &[ErrorSpec] = &[ doc: "A speculative lint run was stopped because its result is no longer needed.", class_attributes: &[], required: &[], + optional: &[], fixed: &[], }, ErrorSpec { @@ -299,6 +335,7 @@ pub(crate) const COMPILER_ERRORS: &[ErrorSpec] = &[ doc: "Raised when project specification configuration is invalid.", class_attributes: &[], required: &[], + optional: &[], fixed: &[], }, ErrorSpec { @@ -307,6 +344,7 @@ pub(crate) const COMPILER_ERRORS: &[ErrorSpec] = &[ doc: "Raised when an authored config value has the wrong runtime type.", class_attributes: &[], required: &["key", "expected", "actual_type"], + optional: &[], fixed: &[], }, ErrorSpec { @@ -315,6 +353,7 @@ pub(crate) const COMPILER_ERRORS: &[ErrorSpec] = &[ doc: "Base class for expected dbt interop errors.", class_attributes: &[("code", ClassValue::Text("C240"))], required: &[], + optional: &[], fixed: &[], }, ErrorSpec { @@ -323,6 +362,7 @@ pub(crate) const COMPILER_ERRORS: &[ErrorSpec] = &[ doc: "Raised when dbt project configuration is missing or invalid.", class_attributes: &[("code", ClassValue::Text("C240"))], required: &[], + optional: &[], fixed: &[], }, ErrorSpec { @@ -331,6 +371,7 @@ pub(crate) const COMPILER_ERRORS: &[ErrorSpec] = &[ doc: "Raised when dbt command execution fails.", class_attributes: &[("code", ClassValue::Text("C241"))], required: &[], + optional: &[], fixed: &[], }, ErrorSpec { @@ -339,6 +380,43 @@ pub(crate) const COMPILER_ERRORS: &[ErrorSpec] = &[ doc: "Raised when `sqb dbt` arguments cannot be routed safely.", class_attributes: &[("code", ClassValue::Text("C230"))], required: &[], + optional: &[], + fixed: &[], + }, + ErrorSpec { + name: "PlannerInputError", + parent: VALUE_ERROR_PARENT, + doc: "Raised when planner inputs cannot be resolved safely.", + class_attributes: &[("code", ClassValue::Text("S000"))], + required: &[], + optional: &[], + fixed: &[], + }, + ErrorSpec { + name: "NativeSqlTestPlanningError", + parent: "PlannerInputError", + doc: "Raised when the native SQL-test planning boundary returns invalid data.", + class_attributes: &[], + required: &[], + optional: &[], + fixed: &[], + }, + ErrorSpec { + name: "FutureCursorSafetyError", + parent: "PlannerInputError", + doc: "Raised when an effective cursor exceeds the configured future limit.", + class_attributes: &[], + required: &[], + optional: &[], + fixed: &[], + }, + ErrorSpec { + name: "MaximumAutomaticStartError", + parent: "PlannerInputError", + doc: "Raised when automatic cursor recovery would exceed its configured horizon.", + class_attributes: &[], + required: &[], + optional: &["evidence"], fixed: &[], }, ]; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compiler_records/shape_classes.rs b/crates/sqlbuild-python/src/bindings/_helpers/compiler_records/shape_classes.rs index fed51c5c23..a9e9ea9d42 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compiler_records/shape_classes.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compiler_records/shape_classes.rs @@ -3,35 +3,18 @@ use pyo3::exceptions::PyLookupError; use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult, Python}; -use sqlbuild_cache::stage_capture::constants::HOSTED_SHAPE_TYPES; +use sqlbuild_cache::hosted_shapes::main::hosted_native_name::hosted_native_name; use crate::bindings::_helpers::compile_driver::python_calls::imported; use crate::bindings::_helpers::records::constants::NATIVE_MODULE; -/// The native class name hosting `module:class`, failing when the qualified key is ambiguous. -pub(crate) fn hosted_name(module: &str, class: &str) -> PyResult> { - let mut found: Option<&'static str> = None; - for (native, python) in HOSTED_SHAPE_TYPES { - let Some((python_module, python_class)) = python.split_once(':') else { - continue; - }; - if python_module != module || python_class != class { - continue; - } - let Some(native_name) = native - .strip_prefix(NATIVE_MODULE) - .and_then(|rest| rest.strip_prefix(':')) - else { - continue; - }; - if found.is_some_and(|existing| existing != native_name) { - return Err(PyLookupError::new_err(format!( - "{module}:{class} names more than one natively hosted shape" - ))); - } - found = Some(native_name); - } - Ok(found) +/// The native class name hosting `module:class`, failing loudly when the name is ambiguous. +fn hosted_name(module: &str, class: &str) -> PyResult> { + hosted_native_name(module, class).map_err(|_| { + PyLookupError::new_err(format!( + "{module}:{class} names more than one natively hosted shape" + )) + }) } /// `class` of the Python module object `module`, or the native class hosting it. diff --git a/crates/sqlbuild-python/src/bindings/_helpers/cursor/conversion.rs b/crates/sqlbuild-python/src/bindings/_helpers/cursor/conversion.rs index 4d22920f54..f78954520c 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/cursor/conversion.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/cursor/conversion.rs @@ -21,7 +21,7 @@ use crate::bindings::_helpers::cursor::values::{ pub(crate) const MODELS_MODULE: &str = "sqlbuild._native"; const EXCEPTIONS_MODULE: &str = "sqlbuild.cursor_algebra.exceptions"; -const PLANNER_TYPES_MODULE: &str = "sqlbuild.compiler.planner.types"; +const PLANNER_TYPES_MODULE: &str = "sqlbuild._native"; const GRAIN_NAMES: [(&str, Grain); 6] = [ ("second", Grain::Second), ("minute", Grain::Minute), diff --git a/crates/sqlbuild-python/src/bindings/_helpers/dbt_interop/plan_output.rs b/crates/sqlbuild-python/src/bindings/_helpers/dbt_interop/plan_output.rs index 61ba953612..946cd50284 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/dbt_interop/plan_output.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/dbt_interop/plan_output.rs @@ -16,7 +16,7 @@ use crate::bindings::_helpers::compile_driver::python_calls::{ const NATIVE_RECORDS_MODULE: &str = "sqlbuild._native"; const NATIVE_RECORDS: &str = "sqlbuild._native"; -const PLANNER_MODELS: &str = "sqlbuild.compiler.planner.models"; +const PLANNER_MODELS: &str = "sqlbuild._native"; const HOOKS: &str = "sqlbuild.runtime.contracts.models"; const BUILD_OPTIONS: (&[&str], &[&str]) = (&["environment", "request", "hooks"], &[]); const ATTACH_OPTIONS: (&[&str], &[&str]) = ( @@ -204,14 +204,8 @@ pub(crate) fn build_sqlbuild_plan_output<'py>( ); match result { Err(error) - if error.is_instance( - py, - &imported( - py, - "sqlbuild.compiler.planner.exceptions", - "PlannerInputError", - )?, - ) => + if error + .is_instance(py, &imported(py, "sqlbuild._native", "PlannerInputError")?) => { call_with( &imported(py, "sqlbuild._native", "build_display_only_sqlbuild_plan")?, diff --git a/crates/sqlbuild-python/src/bindings/_helpers/format_command/selection.rs b/crates/sqlbuild-python/src/bindings/_helpers/format_command/selection.rs index e74a2b9a2c..e7418db0c8 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/format_command/selection.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/format_command/selection.rs @@ -18,7 +18,7 @@ use crate::bindings::_helpers::command_runtime::adapters::resolve_cli_adapter; use crate::bindings::_helpers::compile_driver::python_calls::{call_with, imported, within}; const NATIVE_MODULE: &str = "sqlbuild._native"; -const PLANNER_ERRORS_MODULE: &str = "sqlbuild.compiler.planner.exceptions"; +const PLANNER_ERRORS_MODULE: &str = "sqlbuild._native"; const DISCOVERY_MODELS_MODULE: &str = "sqlbuild.compiler.discovery.models"; const UNIT_TEST_SELECTORS_MODULE: &str = "sqlbuild._native"; const GRAPH_SELECTION_MODULE: &str = "sqlbuild._native"; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/frontier/encoder.rs b/crates/sqlbuild-python/src/bindings/_helpers/frontier/encoder.rs index 8f83ba02a0..2f12ff6312 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/frontier/encoder.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/frontier/encoder.rs @@ -9,10 +9,10 @@ use pyo3::types::{ PyStringMethods, PyTuple, }; +use sqlbuild_cache::hosted_shapes::constants::HOSTED_SHAPE_TYPES; use sqlbuild_cache::stage_capture::constants::{ - CANONICAL_TYPES, DECODED_SEQUENCES, HOSTED_SHAPE_TYPES, OMITTED_ATTRIBUTES, - PROJECTED_ATTRIBUTES, RESERVED_KEYS, SHARED_MARKER, SHARED_MIN_CHARS, SKIPPED_SLOTS, - UNORDERED_ATTRIBUTES, + CANONICAL_TYPES, DECODED_SEQUENCES, OMITTED_ATTRIBUTES, PROJECTED_ATTRIBUTES, RESERVED_KEYS, + SHARED_MARKER, SHARED_MIN_CHARS, SKIPPED_SLOTS, UNORDERED_ATTRIBUTES, }; use sqlbuild_cache::stage_capture::main::capture_sort_key::capture_sort_key; use sqlbuild_cache::stage_capture::main::compact_capture_json::compact_capture_json; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/graph/python_selection.rs b/crates/sqlbuild-python/src/bindings/_helpers/graph/python_selection.rs index 7cb087cb02..ef06bf1fa8 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/graph/python_selection.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/graph/python_selection.rs @@ -20,7 +20,7 @@ use crate::bindings::_helpers::boundary::panics::compiler_guard; use crate::bindings::_helpers::compile_driver::python_calls::imported; use crate::bindings::_helpers::graph::project_graph::NativeProjectGraph; -const PLANNER_EXCEPTIONS_MODULE: &str = "sqlbuild.compiler.planner.exceptions"; +const PLANNER_EXCEPTIONS_MODULE: &str = "sqlbuild._native"; type NameRows = Vec<(String, Vec)>; /// `(name, tags, POSIX relative path, upstream node names)` of one discovered node. diff --git a/crates/sqlbuild-python/src/bindings/_helpers/lineage/lineage_command.rs b/crates/sqlbuild-python/src/bindings/_helpers/lineage/lineage_command.rs index 698fbfa063..bb75f84d8b 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/lineage/lineage_command.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/lineage/lineage_command.rs @@ -50,7 +50,7 @@ const SELECTION_MODULE: &str = "sqlbuild._native"; const SELECTOR_TOKENS_MODULE: &str = "sqlbuild._native"; const SELECTOR_PARSE_MODULE: &str = "sqlbuild._native"; const SELECTOR_MATCH_MODULE: &str = "sqlbuild._native"; -const PLANNER_EXCEPTIONS: &str = "sqlbuild.compiler.planner.exceptions"; +const PLANNER_EXCEPTIONS: &str = "sqlbuild._native"; const UNKNOWN_SELECTOR_ERROR_CODE: &str = "S007"; const RICH_MODE: &str = "rich"; const ANALYSIS_HELP: &str = "enable SQL analysis or install SQLBuild with Polyglot support"; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/load/load_command.rs b/crates/sqlbuild-python/src/bindings/_helpers/load/load_command.rs index 8f35985120..5cffcea16c 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/load/load_command.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/load/load_command.rs @@ -36,7 +36,7 @@ const EFFECTIVE_SETTINGS_MODULE: &str = "sqlbuild._native"; const ADAPTER_NAME_MODULE: &str = "sqlbuild._native"; const RELATION_LOOKUP_MODULE: &str = "sqlbuild.adapter.relations.main.relation_lookup"; const CURSOR_OVERRIDES_MODULE: &str = "sqlbuild._native"; -const PLANNER_MODELS: &str = "sqlbuild.compiler.planner.models"; +const PLANNER_MODELS: &str = "sqlbuild._native"; const PROVIDER_SESSION_MODULE: &str = "sqlbuild.provider.main.session"; const PYTHON_NODE_MODELS: &str = "sqlbuild.python_nodes.models"; const PYTHON_NODE_TYPES: &str = "sqlbuild.python_nodes.types"; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/plan_output/display.rs b/crates/sqlbuild-python/src/bindings/_helpers/plan_output/display.rs index 51917f4b7d..f7eb47e474 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/plan_output/display.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/plan_output/display.rs @@ -24,8 +24,8 @@ use crate::bindings::_helpers::cursor::scalars::{ }; use crate::bindings::_helpers::plan_output::values::{optional_text, text}; -const PLANNER_TYPES: &str = "sqlbuild.compiler.planner.types"; -const PLANNER_MODELS: &str = "sqlbuild.compiler.planner.models"; +const PLANNER_TYPES: &str = "sqlbuild._native"; +const PLANNER_MODELS: &str = "sqlbuild._native"; const CURSOR_MODELS: &str = "sqlbuild._native"; const DATETIME_MODULE: &str = "datetime"; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_changes/results.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_changes/results.rs index 8fd29dcaf2..9151fe572d 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_changes/results.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_changes/results.rs @@ -11,9 +11,9 @@ use sqlbuild_analysis::planner_changes::models::{ use crate::bindings::_helpers::compile_driver::python_calls::{call_with, imported}; -pub(crate) const MODELS_MODULE: &str = "sqlbuild.compiler.planner.models"; -pub(crate) const TYPES_MODULE: &str = "sqlbuild.compiler.planner.types"; -const EXCEPTIONS_MODULE: &str = "sqlbuild.compiler.planner.exceptions"; +pub(crate) const MODELS_MODULE: &str = "sqlbuild._native"; +pub(crate) const TYPES_MODULE: &str = "sqlbuild._native"; +const EXCEPTIONS_MODULE: &str = "sqlbuild._native"; /// `PlannerInputError(message)` for an unknown replay policy. pub(crate) fn replay_policy_error(py: Python<'_>, error: UnknownReplayPolicy) -> PyErr { diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_entries/values.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_entries/values.rs index ca8c68b3eb..4f3dbdc918 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_entries/values.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_entries/values.rs @@ -7,9 +7,9 @@ use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyErr, PyResult, Python}; use pyo3::types::PyString; pub(super) const NATIVE_MODULE: &str = "sqlbuild._native"; -pub(super) const PLANNER_MODELS: &str = "sqlbuild.compiler.planner.models"; -pub(super) const PLANNER_TYPES: &str = "sqlbuild.compiler.planner.types"; -pub(super) const PLANNER_EXCEPTIONS: &str = "sqlbuild.compiler.planner.exceptions"; +pub(super) const PLANNER_MODELS: &str = "sqlbuild._native"; +pub(super) const PLANNER_TYPES: &str = "sqlbuild._native"; +pub(super) const PLANNER_EXCEPTIONS: &str = "sqlbuild._native"; pub(super) const MICROBATCH_CONCURRENCY_MESSAGE: &str = "batch_concurrency > 1 requires concurrent microbatches; selected model(s): "; pub(super) const COMPILE_EXCEPTIONS: &str = "sqlbuild.compiler.compile.exceptions"; @@ -26,7 +26,7 @@ pub(super) fn native<'py>( call_with(&imported(py, NATIVE_MODULE, name)?, pairs) } -/// `sqlbuild.compiler.planner.models.name(**pairs)`. +/// The planner record `name(**pairs)`. pub(super) fn planner_model<'py>( py: Python<'py>, name: &str, diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_identity/graph.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_identity/graph.rs index 7659f048da..bc34edda00 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_identity/graph.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_identity/graph.rs @@ -29,7 +29,7 @@ use crate::bindings::_helpers::graph::algorithms::{closure, inverted}; use crate::bindings::_helpers::graph::keys::KeyCache; use crate::bindings::_helpers::planner_identity::hashing::graph_key; -const EXCEPTIONS_MODULE: &str = "sqlbuild.compiler.planner.exceptions"; +const EXCEPTIONS_MODULE: &str = "sqlbuild._native"; const TABLE_FN_TEST_MODE: &str = "table_fn"; const INLINE_SQL_HOOK: &str = "inline SQL"; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_identity/hashing.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_identity/hashing.rs index af4645c209..65d24bc6d4 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_identity/hashing.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_identity/hashing.rs @@ -16,7 +16,7 @@ use crate::bindings::_helpers::boundary::panics::compiler_error; use crate::bindings::_helpers::compile_driver::python_calls::{imported, keywords}; use crate::bindings::types::CompilerDetach; -const EXCEPTIONS_MODULE: &str = "sqlbuild.compiler.planner.exceptions"; +const EXCEPTIONS_MODULE: &str = "sqlbuild._native"; /// The query fingerprint of a model, or the planner input error naming the model. pub(crate) fn model_query_hash( diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/compatibility.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/compatibility.rs index 743e4061b8..85904dedb9 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/compatibility.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/compatibility.rs @@ -10,7 +10,7 @@ use pyo3::{IntoPyObject, pyfunction, wrap_pyfunction}; use sqlbuild_analysis::planner_migrations::main::compatibility_findings::compatibility_findings; use sqlbuild_analysis::planner_migrations::models::CompatibilityFinding; -const TYPES_MODULE: &str = "sqlbuild.compiler.planner.types"; +const TYPES_MODULE: &str = "sqlbuild._native"; fn config(model: &Bound<'_, PyAny>, key: &str) -> PyResult> { model diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/constants.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/constants.rs index 110d7f3ace..89636a1534 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/constants.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/constants.rs @@ -1,5 +1,5 @@ -pub(super) const PLANNER_MODELS: &str = "sqlbuild.compiler.planner.models"; -pub(super) const MIGRATION_TYPES: &str = "sqlbuild.compiler.migrations.types"; +pub(super) const PLANNER_MODELS: &str = "sqlbuild._native"; +pub(super) const MIGRATION_TYPES: &str = "sqlbuild._native"; pub(super) const AUTOMATIC: &str = "automatic"; pub(super) const DONE: &str = "done"; pub(super) const PENDING_ARCHIVE: &str = "pending_archive"; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/model_entries.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/model_entries.rs index ac1fb2aa3d..86203c56d0 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/model_entries.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/model_entries.rs @@ -77,12 +77,7 @@ pub(super) fn decide<'py>(args: &KeywordArguments<'py>) -> PyResult( fn input_error(py: Python<'_>, message: String) -> PyResult { Ok(PyErr::from_value( - imported( - py, - "sqlbuild.compiler.planner.exceptions", - "PlannerInputError", - )? - .call1((message,))?, + imported(py, "sqlbuild._native", "PlannerInputError")?.call1((message,))?, )) } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_models/records.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_models/records.rs index 5885e93320..2dc22e5db5 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_models/records.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_models/records.rs @@ -143,8 +143,7 @@ fn forward_only_backfill(py: Python<'_>) -> PyResult> { let kwargs = PyDict::new(py); kwargs.set_item( "action", - imported(py, "sqlbuild.compiler.planner.types", "BackfillAction")? - .getattr("FORWARD_ONLY")?, + imported(py, "sqlbuild._native", "BackfillAction")?.getattr("FORWARD_ONLY")?, )?; imported(py, "sqlbuild._native", "BackfillResult")?.call((), Some(&kwargs)) } @@ -155,13 +154,9 @@ fn utc_now(py: Python<'_>) -> PyResult> { fn scenario_artifact_kind(py: Python<'_>, mut values: Vec>) -> PyResult>> { if let Some(kind) = values.first_mut() { - *kind = imported( - py, - "sqlbuild.compiler.planner.types", - "ScenarioArtifactKind", - )? - .call1((kind.bind(py),))? - .unbind(); + *kind = imported(py, "sqlbuild._native", "ScenarioArtifactKind")? + .call1((kind.bind(py),))? + .unbind(); } Ok(values) } @@ -432,7 +427,7 @@ planner_record! { /// Per-function output from change detection. FunctionChangeResult, false, [ (fingerprint_sql = "fingerprint_sql", FieldDefault::Required, true, true), - (reason = "reason", FieldDefault::Member(("sqlbuild.compiler.planner.types", "PlanReason"), "NO_CHANGE"), true, true), + (reason = "reason", FieldDefault::Member(("sqlbuild._native", "PlanReason"), "NO_CHANGE"), true, true), ] } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_phases/constants.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_phases/constants.rs index ef2827ee14..957bc533a0 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_phases/constants.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_phases/constants.rs @@ -1,9 +1,9 @@ //! Python modules and names the planner phases exchange values with. pub(crate) const NATIVE_MODULE: &str = "sqlbuild._native"; -pub(crate) const PLANNER_MODELS: &str = "sqlbuild.compiler.planner.models"; -pub(crate) const PLANNER_TYPES: &str = "sqlbuild.compiler.planner.types"; -pub(crate) const PLANNER_EXCEPTIONS: &str = "sqlbuild.compiler.planner.exceptions"; +pub(crate) const PLANNER_MODELS: &str = "sqlbuild._native"; +pub(crate) const PLANNER_TYPES: &str = "sqlbuild._native"; +pub(crate) const PLANNER_EXCEPTIONS: &str = "sqlbuild._native"; pub(crate) const ADAPTER_MODELS: &str = "sqlbuild.adapter.contract.models"; pub(crate) const ADAPTER_TYPES: &str = "sqlbuild.adapter.contract.types"; pub(crate) const SPEC_MODELS: &str = "sqlbuild._native"; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_records/constants.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_records/constants.rs index 76e2105e04..a9e4a328d3 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_records/constants.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_records/constants.rs @@ -1,5 +1,6 @@ //! Layouts of the planner, migration, cursor and fingerprint value records hosted natively. +use crate::bindings::_helpers::record_behaviours::planner::forward_only_backfill; use crate::bindings::_helpers::records::models::{FieldDefault, FieldSpec, RecordSpec}; const R_PARSED_SELECTOR: RecordSpec = RecordSpec { @@ -220,6 +221,707 @@ const R_DURATION: RecordSpec = RecordSpec { FieldSpec::new("seconds", FieldDefault::Int(0)), ], }; +const R_MIGRATION_RELATION: RecordSpec = RecordSpec { + name: "MigrationRelation", + doc: "One physical relation named by a migration event.", + fields: &[ + FieldSpec::required("database"), + FieldSpec::required("schema"), + FieldSpec::required("name"), + ], +}; +const R_MIGRATION_EVENT: RecordSpec = RecordSpec { + name: "MigrationEvent", + doc: "One immutable fact that a origin relation was cloned over a destination.", + fields: &[ + FieldSpec::required("event_id"), + FieldSpec::required("target_name"), + FieldSpec::required("origin_model"), + FieldSpec::required("origin"), + FieldSpec::required("destination_model"), + FieldSpec::required("destination"), + FieldSpec::required("origin_version_hash"), + FieldSpec::required("discovery"), + FieldSpec::required("decision"), + FieldSpec::required("run_id"), + FieldSpec::required("created_at"), + ], +}; +const R_COLUMN_MIGRATION_EVENT: RecordSpec = RecordSpec { + name: "ColumnMigrationEvent", + doc: "One immutable fact that a column of a model relation was renamed in place.", + fields: &[ + FieldSpec::required("event_id"), + FieldSpec::required("target_name"), + FieldSpec::required("model_name"), + FieldSpec::required("relation"), + FieldSpec::required("origin_column"), + FieldSpec::required("destination_column"), + FieldSpec::required("discovery"), + FieldSpec::required("decision"), + FieldSpec::required("run_id"), + FieldSpec::required("created_at"), + ], +}; +const R_OLD_NAME_VIEW_EVENT: RecordSpec = RecordSpec { + name: "OldNameViewEvent", + doc: "One immutable fact about the old name of one recorded move.", + fields: &[ + FieldSpec::required("event_id"), + FieldSpec::required("target_name"), + FieldSpec::required("event_type"), + FieldSpec::required("migration_event_id"), + FieldSpec::required("destination_model"), + FieldSpec::required("old"), + FieldSpec::required("new"), + FieldSpec::required("run_id"), + FieldSpec::required("created_at"), + FieldSpec::new("view_retention", FieldDefault::None), + FieldSpec::new("archive_name", FieldDefault::None), + FieldSpec::new("column_aliases", FieldDefault::EmptyTuple), + FieldSpec::new("expires_at", FieldDefault::None), + FieldSpec::new("drop_reason", FieldDefault::None), + FieldSpec::new("grants_copied", FieldDefault::None), + FieldSpec::new("view_sql", FieldDefault::None), + ], +}; +const R_OLD_NAME_VIEW_HISTORY: RecordSpec = RecordSpec { + name: "OldNameViewHistory", + doc: "The recorded move and every old-name fact that references it.", + fields: &[ + FieldSpec::required("move"), + FieldSpec::required("required"), + FieldSpec::new("archived", FieldDefault::None), + FieldSpec::new("created", FieldDefault::None), + FieldSpec::new("dropped", FieldDefault::None), + ], +}; +const R_CURSOR_OVERRIDES: RecordSpec = RecordSpec { + name: "CursorOverrides", + doc: "Cursor override values from CLI flags; untyped `start`/`end` await planner typing.", + fields: &[ + FieldSpec::new("start_ts", FieldDefault::None), + FieldSpec::new("end_ts", FieldDefault::None), + FieldSpec::new("start_int", FieldDefault::None), + FieldSpec::new("end_int", FieldDefault::None), + FieldSpec::new("start", FieldDefault::None), + FieldSpec::new("end", FieldDefault::None), + ], +}; +const R_EMPTY_CURSOR_INPUT_DECISION: RecordSpec = RecordSpec { + name: "EmptyCursorInputDecision", + doc: "Cursor inputs whose lack of rows leaves no window, and whether other inputs have rows.", + fields: &[ + FieldSpec::new("input_names", FieldDefault::EmptyTuple), + FieldSpec::new("waiting_on_empty_inputs", FieldDefault::False), + ], +}; +const R_MODEL_CURSOR_SNAPSHOT: RecordSpec = RecordSpec { + name: "ModelCursorSnapshot", + doc: "Cursor MIN/MAX values gathered from warehouse for one incremental model.", + fields: &[ + FieldSpec::required("target_max"), + FieldSpec::required("upstream_mins"), + FieldSpec::required("upstream_maxes"), + FieldSpec { + name: "physical_target_max", + default: FieldDefault::None, + compared: false, + }, + FieldSpec::new("target_eligible_max", FieldDefault::None), + FieldSpec { + name: "target_relation", + default: FieldDefault::None, + compared: false, + }, + FieldSpec { + name: "destination_cursor_column", + default: FieldDefault::None, + compared: false, + }, + FieldSpec { + name: "input_evidence", + default: FieldDefault::EmptyTuple, + compared: false, + }, + FieldSpec { + name: "expected_watermark_count", + default: FieldDefault::Int(0), + compared: false, + }, + FieldSpec::new("unavailable_watermark_tags", FieldDefault::EmptyTuple), + FieldSpec::new( + "cursor_watermark_mode", + FieldDefault::Member("sqlbuild._native", "CursorWatermarkMode", "ALL"), + ), + FieldSpec::new("upstream_terminal_starts", FieldDefault::EmptyTuple), + FieldSpec::new("upstream_terminal_ends", FieldDefault::EmptyTuple), + FieldSpec::new("upstream_end_inputs", FieldDefault::EmptyTuple), + FieldSpec::new("upstream_availability_ends", FieldDefault::EmptyTuple), + FieldSpec { + name: "empty_input_names", + default: FieldDefault::EmptyTuple, + compared: false, + }, + FieldSpec { + name: "unreadable_input_names", + default: FieldDefault::EmptyTuple, + compared: false, + }, + ], +}; +const R_CURSOR_BOUNDS: RecordSpec = RecordSpec { + name: "CursorBounds", + doc: "Effective cursor start and end values for one incremental model.", + fields: &[ + FieldSpec::required("start"), + FieldSpec::required("end"), + FieldSpec::new("future_safety", FieldDefault::None), + FieldSpec::new("maximum_start_safety", FieldDefault::None), + ], +}; +const R_CURSOR_INPUT_EVIDENCE: RecordSpec = RecordSpec { + name: "CursorInputEvidence", + doc: "Observed bounds for one physical cursor input.", + fields: &[ + FieldSpec::required("relation"), + FieldSpec::required("cursor_column"), + FieldSpec::required("minimum"), + FieldSpec::required("maximum"), + ], +}; +const R_FUTURE_CURSOR_SAFETY_EVIDENCE: RecordSpec = RecordSpec { + name: "FutureCursorSafetyEvidence", + doc: "Structured evidence for one applied future-cursor cap.", + fields: &[ + FieldSpec::required("action"), + FieldSpec::required("max_distance"), + FieldSpec::required("invocation_time"), + FieldSpec::required("discovered_start"), + FieldSpec::required("discovered_end"), + FieldSpec::required("applied_start"), + FieldSpec::required("applied_end"), + FieldSpec::required("maximum_allowed_start"), + FieldSpec::required("maximum_allowed_end"), + FieldSpec::required("future_start_detected"), + FieldSpec::required("future_end_detected"), + FieldSpec::required("determining_relation"), + FieldSpec::required("determining_cursor_column"), + FieldSpec::new("inputs", FieldDefault::EmptyTuple), + ], +}; +const R_MAXIMUM_START_SAFETY_EVIDENCE: RecordSpec = RecordSpec { + name: "MaximumStartSafetyEvidence", + doc: "Structured evidence for an automatic-start eligibility decision.", + fields: &[ + FieldSpec::required("action"), + FieldSpec::required("max_ahead"), + FieldSpec::required("invocation_time"), + FieldSpec::required("physical_target_max"), + FieldSpec::required("highest_eligible_target_max"), + FieldSpec::required("effective_start"), + FieldSpec::required("maximum_allowed_start"), + FieldSpec::required("target_relation"), + FieldSpec::required("cursor_column"), + ], +}; +const R_CURSOR_INPUT_RELATION: RecordSpec = RecordSpec { + name: "CursorInputRelation", + doc: "One cursor-bearing input relation for runtime range discovery.", + fields: &[ + FieldSpec::required("relation"), + FieldSpec::required("cursor_column"), + FieldSpec::new("cursor_grain", FieldDefault::None), + FieldSpec::new("is_model_backed", FieldDefault::False), + FieldSpec::new("is_runtime_produced", FieldDefault::False), + FieldSpec::new("terminal_cursor_start", FieldDefault::None), + FieldSpec::new("terminal_cursor_end", FieldDefault::None), + FieldSpec::new("input_name", FieldDefault::None), + FieldSpec::new("input_kind", FieldDefault::None), + ], +}; +const R_MAXIMUM_START_POLICY_INPUTS: RecordSpec = RecordSpec { + name: "MaximumStartPolicyInputs", + doc: "Effective policy and materialization safety inputs for automatic starts.", + fields: &[ + FieldSpec::new("config", FieldDefault::None), + FieldSpec::new("invocation_time", FieldDefault::None), + FieldSpec::new("incremental_strategy", FieldDefault::None), + FieldSpec::new("incremental_mode", FieldDefault::None), + ], +}; +const R_SCHEMA_FINDING: RecordSpec = RecordSpec { + name: "SchemaFinding", + doc: "One detected schema difference between expected and warehouse columns.", + fields: &[ + FieldSpec::required("kind"), + FieldSpec::required("column_name"), + FieldSpec::required("source"), + FieldSpec::new("expected_type", FieldDefault::None), + FieldSpec::new("actual_type", FieldDefault::None), + ], +}; +const R_RETENTION_PLAN_ENTRY: RecordSpec = RecordSpec { + name: "RetentionPlanEntry", + doc: "Warehouse retention work independent of model identity actions.", + fields: &[ + FieldSpec::required("request"), + FieldSpec::required("model_names"), + FieldSpec::required("actual_days"), + FieldSpec::required("effective_days"), + FieldSpec::required("source"), + FieldSpec::required("direction"), + FieldSpec::required("phase"), + FieldSpec::new("statements", FieldDefault::EmptyTuple), + FieldSpec::new("irreversible_warning", FieldDefault::None), + FieldSpec::new("decrease_policy", FieldDefault::Text("deny")), + ], +}; +const R_TABLE_TYPE_PLAN_ENTRY: RecordSpec = RecordSpec { + name: "TableTypePlanEntry", + doc: "Snowflake table-type work independent of model identity actions.", + fields: &[ + FieldSpec::required("model_name"), + FieldSpec::required("destination"), + FieldSpec::required("copy_name"), + FieldSpec::required("desired_type"), + FieldSpec::required("actual_type"), + FieldSpec::required("source"), + FieldSpec::required("downgrade"), + FieldSpec::required("downgrade_policy"), + FieldSpec::new("irreversible_warning", FieldDefault::None), + ], +}; +const R_MODEL_MIGRATION_PLAN_ENTRY: RecordSpec = RecordSpec { + name: "ModelMigrationPlanEntry", + doc: "One declared or discovered model migration and its per-run decision.", + fields: &[ + FieldSpec::required("model_name"), + FieldSpec::required("discovery"), + FieldSpec::required("decision"), + FieldSpec::required("compatibility"), + FieldSpec::required("origin_model"), + FieldSpec::required("origin"), + FieldSpec::required("destination"), + FieldSpec::required("target_name"), + FieldSpec::new("origin_version_hash", FieldDefault::Text("")), + FieldSpec::new("origin_is_transient", FieldDefault::False), + FieldSpec::new("stage_is_transient", FieldDefault::None), + FieldSpec::new("transfer", FieldDefault::None), + FieldSpec::new("transfer_fallback", FieldDefault::None), + FieldSpec::new("promotion", FieldDefault::None), + FieldSpec::new("completed_at", FieldDefault::None), + FieldSpec::new("compatibility_findings", FieldDefault::EmptyTuple), + FieldSpec::new("message", FieldDefault::None), + FieldSpec::new("origin_tracked", FieldDefault::False), + FieldSpec::new( + "missing_origin_policy", + FieldDefault::Member("sqlbuild._native", "MissingMigrationOriginPolicy", "ALLOW"), + ), + ], +}; +const R_OLD_NAME_VIEW_PLAN_ENTRY: RecordSpec = RecordSpec { + name: "OldNameViewPlanEntry", + doc: "What one build does at the old name of one migrated model.", + fields: &[ + FieldSpec::required("model_name"), + FieldSpec::required("origin"), + FieldSpec::required("destination"), + FieldSpec::required("action"), + FieldSpec::required("target_name"), + FieldSpec::new("retention", FieldDefault::None), + FieldSpec::new("expires_at", FieldDefault::None), + FieldSpec::new("column_aliases", FieldDefault::EmptyTuple), + FieldSpec::new("stores_history", FieldDefault::False), + FieldSpec::new("migration_event_id", FieldDefault::None), + FieldSpec::new("records_requirement", FieldDefault::False), + FieldSpec::new("reason", FieldDefault::None), + FieldSpec::new("grants_copied", FieldDefault::None), + FieldSpec::new("grants_supported", FieldDefault::False), + FieldSpec::new("archived", FieldDefault::False), + ], +}; +const R_OLD_NAME_VIEW: RecordSpec = RecordSpec { + name: "OldNameView", + doc: "One compatibility view recorded at an old name and not recorded as dropped.", + fields: &[ + FieldSpec::required("destination_model"), + FieldSpec::required("old"), + FieldSpec::required("new"), + FieldSpec::required("expires_at"), + FieldSpec::required("column_aliases"), + FieldSpec::required("migration_event_id"), + FieldSpec::required("target_name"), + FieldSpec::new("name_reused_by", FieldDefault::None), + ], +}; +const R_COLUMN_MIGRATION_PLAN_ENTRY: RecordSpec = RecordSpec { + name: "ColumnMigrationPlanEntry", + doc: "One declared or detected in-place column rename and its per-run decision.", + fields: &[ + FieldSpec::required("model_name"), + FieldSpec::required("destination"), + FieldSpec::required("origin_column"), + FieldSpec::required("destination_column"), + FieldSpec::required("discovery"), + FieldSpec::required("decision"), + FieldSpec::required("target_name"), + FieldSpec::new("completed_at", FieldDefault::None), + FieldSpec::new("message", FieldDefault::None), + FieldSpec::new( + "missing_origin_policy", + FieldDefault::Member("sqlbuild._native", "MissingMigrationOriginPolicy", "ALLOW"), + ), + ], +}; +const R_MODEL_PLAN_ENTRY: RecordSpec = RecordSpec { + name: "ModelPlanEntry", + doc: "Per-model execution plan entry with action, reason, and resolved artifacts.", + fields: &[ + FieldSpec::required("key"), + FieldSpec::required("name"), + FieldSpec::required("relative_path"), + FieldSpec::required("materialization_type"), + FieldSpec::required("action"), + FieldSpec::required("reason"), + FieldSpec::required("destination"), + FieldSpec::required("fingerprint_query_sql"), + FieldSpec::required("resolved_sql"), + FieldSpec::required("logical_ddl"), + FieldSpec::new("migration_fingerprint", FieldDefault::None), + FieldSpec::new("incremental_strategy", FieldDefault::None), + FieldSpec::new("incremental_mode", FieldDefault::None), + FieldSpec::new("microbatch_strategy", FieldDefault::None), + FieldSpec::new("cursor_watermark_mode", FieldDefault::None), + FieldSpec::new("cursor_column", FieldDefault::None), + FieldSpec::new("cursor_type", FieldDefault::None), + FieldSpec::new("cursor_grain", FieldDefault::None), + FieldSpec::new("cursor_start", FieldDefault::None), + FieldSpec::new("cursor_end", FieldDefault::None), + FieldSpec::new("lookback", FieldDefault::None), + FieldSpec::new("cursor_bounds", FieldDefault::None), + FieldSpec::new("empty_cursor_inputs", FieldDefault::EmptyTuple), + FieldSpec::new("waiting_on_empty_inputs", FieldDefault::False), + FieldSpec::new("cursor_input_relations", FieldDefault::EmptyTuple), + FieldSpec::new("batch_size", FieldDefault::None), + FieldSpec::new("batch_concurrency", FieldDefault::Int(1)), + FieldSpec::new("unaccounted_partition_policy", FieldDefault::None), + FieldSpec::new("microbatch_range", FieldDefault::None), + FieldSpec::new("microbatch_limit", FieldDefault::None), + FieldSpec::new("declared_microbatch_limit_action", FieldDefault::None), + FieldSpec::new("microbatch_safety_limit", FieldDefault::None), + FieldSpec::new("microbatch_safety_limit_action", FieldDefault::None), + FieldSpec::new("microbatch_limit_count", FieldDefault::None), + FieldSpec::new("microbatch_limit_action", FieldDefault::None), + FieldSpec::new("microbatch_limit_warning", FieldDefault::None), + FieldSpec::new("start_cursor_override", FieldDefault::None), + FieldSpec::new("end_cursor_override", FieldDefault::None), + FieldSpec::new("future_cursor_config", FieldDefault::None), + FieldSpec::new("start_cursor_config", FieldDefault::None), + FieldSpec::new("invocation_time", FieldDefault::None), + FieldSpec::new("unique_key", FieldDefault::EmptyTuple), + FieldSpec::new("merge_exclude_columns", FieldDefault::EmptyTuple), + FieldSpec::new("snapshot_strategy", FieldDefault::None), + FieldSpec::new("updated_at_column", FieldDefault::None), + FieldSpec::new("check_columns", FieldDefault::EmptyTuple), + FieldSpec::new("observed_at_column", FieldDefault::None), + FieldSpec::new("historical_input", FieldDefault::None), + FieldSpec::new("valid_from_column", FieldDefault::None), + FieldSpec::new("valid_to_column", FieldDefault::None), + FieldSpec::new("initial_valid_from", FieldDefault::None), + FieldSpec::new("invalidate_hard_deletes", FieldDefault::False), + FieldSpec::new("snapshot_full_refresh", FieldDefault::None), + FieldSpec::new("snapshot_schema_change", FieldDefault::None), + FieldSpec::new("table_type", FieldDefault::Text("transient")), + FieldSpec::new("on_schema_change", FieldDefault::None), + FieldSpec::new("type_enforcement", FieldDefault::False), + FieldSpec::new("declared_columns", FieldDefault::EmptyTuple), + FieldSpec::new("contract_enforced", FieldDefault::False), + FieldSpec::new("contract_columns", FieldDefault::EmptyTuple), + FieldSpec::new("contract_dynamic_columns", FieldDefault::EmptyTuple), + FieldSpec::new("pre_hooks", FieldDefault::None), + FieldSpec::new("post_hooks", FieldDefault::None), + FieldSpec::new("previous_query_sql", FieldDefault::None), + FieldSpec::new("fingerprint_metadata_json", FieldDefault::None), + FieldSpec::new("previous_metadata_json", FieldDefault::None), + FieldSpec::new("fingerprint_version_hash", FieldDefault::None), + FieldSpec::new("previous_version_hash", FieldDefault::None), + FieldSpec::new("query_changed", FieldDefault::False), + FieldSpec::new("config_changed", FieldDefault::False), + FieldSpec::new("schema_actions", FieldDefault::EmptyTuple), + FieldSpec::new("schema_findings", FieldDefault::EmptyTuple), + FieldSpec::new("backfill", FieldDefault::Factory(forward_only_backfill)), + FieldSpec::new("changed_functions", FieldDefault::EmptyTuple), + FieldSpec::new("custom_materialization_name", FieldDefault::None), + FieldSpec::new("custom_config", FieldDefault::EmptyDict), + FieldSpec::new("custom_placeholders", FieldDefault::EmptyDict), + FieldSpec::new("column_rename_hints", FieldDefault::EmptyTuple), + ], +}; +const R_CLONE_SOURCE_PLAN_ENTRY: RecordSpec = RecordSpec { + name: "CloneSourcePlanEntry", + doc: "One managed source relation selected for target cloning.", + fields: &[ + FieldSpec::required("key"), + FieldSpec::required("name"), + FieldSpec::required("destination"), + ], +}; +const R_SEED_PLAN_ENTRY: RecordSpec = RecordSpec { + name: "SeedPlanEntry", + doc: "Per-seed execution plan entry.", + fields: &[ + FieldSpec::required("key"), + FieldSpec::required("name"), + FieldSpec::required("destination"), + FieldSpec::required("file_path"), + FieldSpec::required("columns"), + FieldSpec::required("csv_settings"), + FieldSpec::new("fingerprint_definition", FieldDefault::Text("")), + FieldSpec::new("fingerprint_version_hash", FieldDefault::Text("")), + FieldSpec::new("fingerprint_metadata_json", FieldDefault::Text("{}")), + FieldSpec::new( + "action", + FieldDefault::Member("sqlbuild._native", "PlanAction", "LOAD_SEED"), + ), + FieldSpec::new( + "reason", + FieldDefault::Member("sqlbuild._native", "PlanReason", "FIRST_RUN"), + ), + ], +}; +const R_SOURCE_LOAD_PLAN_ENTRY: RecordSpec = RecordSpec { + name: "SourceLoadPlanEntry", + doc: "Per-managed-source loader execution plan entry.", + fields: &[ + FieldSpec::required("key"), + FieldSpec::required("name"), + FieldSpec::required("loader"), + FieldSpec::required("destination"), + FieldSpec::new( + "resource_kind", + FieldDefault::Member( + "sqlbuild.runtime.contracts.types", + "ExecutionResourceKind", + "SOURCE", + ), + ), + FieldSpec::new("write_strategy", FieldDefault::None), + FieldSpec::new("cursor_column", FieldDefault::None), + FieldSpec::new("unique_key", FieldDefault::EmptyTuple), + FieldSpec::new("is_reload", FieldDefault::False), + FieldSpec::new("integration_kind", FieldDefault::None), + ], +}; +const R_FUNCTION_PLAN_ENTRY: RecordSpec = RecordSpec { + name: "FunctionPlanEntry", + doc: "Per-SQL-function execution plan entry.", + fields: &[ + FieldSpec::required("key"), + FieldSpec::required("name"), + FieldSpec::required("relative_path"), + FieldSpec::required("destination"), + FieldSpec::required("arguments"), + FieldSpec::required("returns"), + FieldSpec::required("body_sql"), + FieldSpec::required("fingerprint_query_sql"), + FieldSpec::required("fingerprint_destination"), + FieldSpec::new("return_columns", FieldDefault::EmptyTuple), + FieldSpec::new( + "language", + FieldDefault::Member("sqlbuild.compiler.compile.types", "FunctionLanguage", "SQL"), + ), + FieldSpec::new("source_file_path", FieldDefault::None), + FieldSpec::new("runtime_version", FieldDefault::None), + FieldSpec::new("entry_point", FieldDefault::None), + FieldSpec::new("packages", FieldDefault::EmptyTuple), + FieldSpec::new("previous_query_sql", FieldDefault::None), + FieldSpec::new( + "reason", + FieldDefault::Member("sqlbuild._native", "PlanReason", "NO_CHANGE"), + ), + ], +}; +const R_AUDIT_PLAN_ENTRY: RecordSpec = RecordSpec { + name: "AuditPlanEntry", + doc: "Per-audit execution plan entry with resolved SQL and scheduling metadata.", + fields: &[ + FieldSpec::required("key"), + FieldSpec::required("name"), + FieldSpec::required("definition_name"), + FieldSpec::required("resolved_sql"), + FieldSpec::required("unresolved_sql"), + FieldSpec::required("attachment_kind"), + FieldSpec::required("severity"), + FieldSpec::required("requested_run_scope"), + FieldSpec::required("effective_run_scope"), + FieldSpec::new( + "evaluation_mode", + FieldDefault::Member( + "sqlbuild.compiler.auditing.types", + "AuditEvaluationMode", + "VIOLATIONS", + ), + ), + FieldSpec::new("value_column", FieldDefault::None), + FieldSpec::new("sample_count_column", FieldDefault::None), + FieldSpec::new("sample_unit", FieldDefault::None), + FieldSpec::new("thresholds", FieldDefault::None), + FieldSpec::new("minimum_samples", FieldDefault::None), + FieldSpec::new("evidence_resolved_sql", FieldDefault::None), + FieldSpec::new("evidence_unresolved_sql", FieldDefault::None), + FieldSpec::new("evidence_limit", FieldDefault::None), + FieldSpec::new("scope_deps", FieldDefault::EmptyTuple), + FieldSpec::new("attached_target_kind", FieldDefault::None), + FieldSpec::new("attached_target_name", FieldDefault::None), + FieldSpec::new("attached_column_name", FieldDefault::None), + FieldSpec::new("always_run", FieldDefault::False), + FieldSpec::new("description", FieldDefault::None), + FieldSpec::new("reads_outside_target_lineage", FieldDefault::False), + ], +}; +const R_CHAIN_STEP: RecordSpec = RecordSpec { + name: "ChainStep", + doc: "One step in a chained test execution sequence.", + fields: &[ + FieldSpec::required("model_name"), + FieldSpec::required("resolved_sql"), + FieldSpec::new("expected_cte_sql", FieldDefault::None), + FieldSpec::new("lifted_ctes", FieldDefault::EmptyTuple), + FieldSpec::new("comparison_body_sql", FieldDefault::None), + FieldSpec::new("expected_columns", FieldDefault::None), + FieldSpec::new("expected_lifted_ctes", FieldDefault::EmptyTuple), + ], +}; +const R_SQL_TEST_PLAN_ENTRY: RecordSpec = RecordSpec { + name: "SqlTestPlanEntry", + doc: "Per-test execution plan entry with chained resolution.", + fields: &[ + FieldSpec::required("key"), + FieldSpec::required("name"), + FieldSpec::new("source_path", FieldDefault::None), + FieldSpec::new("block_index", FieldDefault::None), + FieldSpec::new("parent_name", FieldDefault::None), + FieldSpec::new("case_name", FieldDefault::None), + FieldSpec::new("case_index", FieldDefault::None), + FieldSpec::new("case_fingerprint", FieldDefault::None), + FieldSpec::new("parameter_schema", FieldDefault::EmptyTuple), + FieldSpec::new("parameter_values", FieldDefault::EmptyTuple), + FieldSpec::new("mock_ref_names", FieldDefault::EmptyTuple), + FieldSpec::new("mock_source_names", FieldDefault::EmptyTuple), + FieldSpec::new("mock_seed_names", FieldDefault::EmptyTuple), + FieldSpec::new("mock_dbt_ref_names", FieldDefault::EmptyTuple), + FieldSpec::new("mock_table_function_names", FieldDefault::EmptyTuple), + FieldSpec::new("chain", FieldDefault::EmptyTuple), + FieldSpec::new("assertions", FieldDefault::EmptyTuple), + FieldSpec::new("scope_deps", FieldDefault::EmptyTuple), + FieldSpec::new("function_deps", FieldDefault::EmptyTuple), + FieldSpec::new("sql_analysis_enabled", FieldDefault::True), + ], +}; +const R_SCENARIO_RELATION_MAP: RecordSpec = RecordSpec { + name: "ScenarioRelationMap", + doc: "Resolved scenario hash prefix and scenario-owned relation names.", + fields: &[ + FieldSpec::required("scenario_name"), + FieldSpec::required("hash_prefix"), + FieldSpec::new("artifacts", FieldDefault::EmptyTuple), + ], +}; +const R_SCENARIO_RELATION_PLAN: RecordSpec = RecordSpec { + name: "ScenarioRelationPlan", + doc: "Resolved scenario relation locations for model/source/seed ref resolution.", + fields: &[ + FieldSpec::required("scenario_name"), + FieldSpec::required("relation_map"), + FieldSpec::new("model_locations", FieldDefault::EmptyDict), + FieldSpec::new("seed_locations", FieldDefault::EmptyDict), + FieldSpec::new("project_source_map", FieldDefault::EmptyDict), + FieldSpec::new("source_map", FieldDefault::EmptyDict), + FieldSpec::new("source_fixture_locations", FieldDefault::EmptyDict), + FieldSpec::new("ref_fixture_locations", FieldDefault::EmptyDict), + FieldSpec::new("seed_fixture_locations", FieldDefault::EmptyDict), + FieldSpec::new("dbt_ref_fixture_locations", FieldDefault::EmptyDict), + ], +}; +const R_SCENARIO_FIXTURE_PLAN: RecordSpec = RecordSpec { + name: "ScenarioFixturePlan", + doc: "Self-contained fixture SQL planned for scenario materialization.", + fields: &[ + FieldSpec::required("kind"), + FieldSpec::required("logical_name"), + FieldSpec::required("destination"), + FieldSpec::required("sql"), + ], +}; +const R_SCENARIO_EXPECTED_EXPECTATION_PLAN: RecordSpec = RecordSpec { + name: "ScenarioExpectedExpectationPlan", + doc: "Expected-output comparison inputs for one scenario target model.", + fields: &[ + FieldSpec::required("model_name"), + FieldSpec::required("actual_destination"), + FieldSpec::required("expected_sql"), + ], +}; +const R_SCENARIO_ASSERTION_EXPECTATION_PLAN: RecordSpec = RecordSpec { + name: "ScenarioAssertionExpectationPlan", + doc: "Zero-row assertion SQL for one scenario assertion CTE.", + fields: &[FieldSpec::required("name"), FieldSpec::required("sql")], +}; +const R_SCENARIO_EXECUTION_PLAN: RecordSpec = RecordSpec { + name: "ScenarioExecutionPlan", + doc: "Dry-run execution plan for a SQL scenario graph slice.", + fields: &[ + FieldSpec::required("key"), + FieldSpec::required("name"), + FieldSpec::required("graph_plan"), + FieldSpec::required("relation_plan"), + FieldSpec::new("fixture_plans", FieldDefault::EmptyTuple), + FieldSpec::new("seed_entries", FieldDefault::EmptyTuple), + FieldSpec::new("function_entries", FieldDefault::EmptyTuple), + FieldSpec::new("model_entries", FieldDefault::EmptyTuple), + FieldSpec::new("hook_functions", FieldDefault::EmptyTuple), + FieldSpec::new("expected_expectations", FieldDefault::EmptyTuple), + FieldSpec::new("assertion_expectations", FieldDefault::EmptyTuple), + ], +}; +const R_PLAN_OUTPUT: RecordSpec = RecordSpec { + name: "PlanOutput", + doc: "Complete execution plan produced by the planner.", + fields: &[ + FieldSpec::new("execution_order", FieldDefault::EmptyTuple), + FieldSpec::new("model_entries", FieldDefault::EmptyTuple), + FieldSpec::new("seed_entries", FieldDefault::EmptyTuple), + FieldSpec::new("source_load_entries", FieldDefault::EmptyTuple), + FieldSpec::new("function_entries", FieldDefault::EmptyTuple), + FieldSpec::new("audit_entries", FieldDefault::EmptyTuple), + FieldSpec::new("test_entries", FieldDefault::EmptyTuple), + FieldSpec::new("available_test_case_names", FieldDefault::EmptyTuple), + FieldSpec::new("selected_keys", FieldDefault::EmptyFrozenset), + FieldSpec::new("warnings", FieldDefault::EmptyTuple), + FieldSpec::new("retention_entries", FieldDefault::EmptyTuple), + FieldSpec::new("table_type_entries", FieldDefault::EmptyTuple), + FieldSpec::new("migration_entries", FieldDefault::EmptyTuple), + FieldSpec::new("column_migration_entries", FieldDefault::EmptyTuple), + FieldSpec::new("old_name_view_entries", FieldDefault::EmptyTuple), + FieldSpec::new("old_name_views", FieldDefault::EmptyTuple), + FieldSpec::new("upstream_deps", FieldDefault::EmptyDict), + FieldSpec::new("downstream_deps", FieldDefault::EmptyDict), + FieldSpec::new("model_locations", FieldDefault::EmptyDict), + FieldSpec::new("seed_locations", FieldDefault::EmptyDict), + FieldSpec::new("function_locations", FieldDefault::EmptyDict), + FieldSpec::new("source_map", FieldDefault::EmptyDict), + FieldSpec::new("source_read_map", FieldDefault::EmptyDict), + FieldSpec::new("python_source_read_map", FieldDefault::EmptyDict), + FieldSpec::new("hook_functions", FieldDefault::EmptyTuple), + FieldSpec::new("enforce_explicit_references", FieldDefault::True), + FieldSpec::new("provider_usages", FieldDefault::EmptyTuple), + FieldSpec::new("source_freshness", FieldDefault::None), + FieldSpec::new("python_identity_fingerprints", FieldDefault::EmptyDict), + FieldSpec::new("metadata", FieldDefault::EmptyDict), + FieldSpec::new("cursor_overrides", FieldDefault::None), + ], +}; /// The planner, migration, cursor and fingerprint value records. pub(crate) const PLANNER_RECORDS: &[RecordSpec] = &[ @@ -242,4 +944,40 @@ pub(crate) const PLANNER_RECORDS: &[RecordSpec] = &[ R_INTEGER_VALUE, R_ALIGNED_INTERVAL, R_DURATION, + R_MIGRATION_RELATION, + R_MIGRATION_EVENT, + R_COLUMN_MIGRATION_EVENT, + R_OLD_NAME_VIEW_EVENT, + R_OLD_NAME_VIEW_HISTORY, + R_CURSOR_OVERRIDES, + R_EMPTY_CURSOR_INPUT_DECISION, + R_MODEL_CURSOR_SNAPSHOT, + R_CURSOR_BOUNDS, + R_CURSOR_INPUT_EVIDENCE, + R_FUTURE_CURSOR_SAFETY_EVIDENCE, + R_MAXIMUM_START_SAFETY_EVIDENCE, + R_CURSOR_INPUT_RELATION, + R_MAXIMUM_START_POLICY_INPUTS, + R_SCHEMA_FINDING, + R_RETENTION_PLAN_ENTRY, + R_TABLE_TYPE_PLAN_ENTRY, + R_MODEL_MIGRATION_PLAN_ENTRY, + R_OLD_NAME_VIEW_PLAN_ENTRY, + R_OLD_NAME_VIEW, + R_COLUMN_MIGRATION_PLAN_ENTRY, + R_MODEL_PLAN_ENTRY, + R_CLONE_SOURCE_PLAN_ENTRY, + R_SEED_PLAN_ENTRY, + R_SOURCE_LOAD_PLAN_ENTRY, + R_FUNCTION_PLAN_ENTRY, + R_AUDIT_PLAN_ENTRY, + R_CHAIN_STEP, + R_SQL_TEST_PLAN_ENTRY, + R_SCENARIO_RELATION_MAP, + R_SCENARIO_RELATION_PLAN, + R_SCENARIO_FIXTURE_PLAN, + R_SCENARIO_EXPECTED_EXPECTATION_PLAN, + R_SCENARIO_ASSERTION_EXPECTATION_PLAN, + R_SCENARIO_EXECUTION_PLAN, + R_PLAN_OUTPUT, ]; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_resolution/cursor_bounds.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_resolution/cursor_bounds.rs index f968c85d17..1daebfe0ac 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_resolution/cursor_bounds.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_resolution/cursor_bounds.rs @@ -15,7 +15,7 @@ use sqlbuild_cursor::planner::main::raw_bounds::raw_bounds; use sqlbuild_cursor::planner::main::replay_start::replay_start; use sqlbuild_cursor::planner::models::{CursorKind, CursorSnapshot, ReplayPolicy}; -const MODELS: &str = "sqlbuild.compiler.planner.models"; +const MODELS: &str = "sqlbuild._native"; const MAXIMUM_START: &str = "sqlbuild._native"; pub(super) fn optional_scalar<'py>( diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_resolution/cursor_model.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_resolution/cursor_model.rs index cce4a7c1d2..c9d3b069c1 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_resolution/cursor_model.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_resolution/cursor_model.rs @@ -12,7 +12,7 @@ use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyModule, PyModuleMethods, PyRes use pyo3::types::PyDict; use pyo3::{IntoPyObject, PyErr, pyfunction, wrap_pyfunction}; -const MODELS: &str = "sqlbuild.compiler.planner.models"; +const MODELS: &str = "sqlbuild._native"; const CURSOR: &str = "sqlbuild._native"; fn config<'py>(model: &Bound<'py, PyAny>, key: &str) -> PyResult> { @@ -163,11 +163,7 @@ fn compute_model_cursor_bounds<'py>( ], )?; return Err(PyErr::from_value(call_with( - &imported( - py, - "sqlbuild.compiler.planner.exceptions", - "PlannerInputError", - )?, + &imported(py, "sqlbuild._native", "PlannerInputError")?, &[ ("message", refusal.get_item(0)?), ("code", string(py, "S302")), diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_resolution/cursor_overrides.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_resolution/cursor_overrides.rs index 5ef35e5179..cdc86a32b2 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_resolution/cursor_overrides.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_resolution/cursor_overrides.rs @@ -68,12 +68,12 @@ fn type_cursor_overrides<'py>( } } let kind = override_kind(&models, integer_literals).map_err(|message| { - match imported(py, "sqlbuild.compiler.planner.exceptions", "PlannerInputError").and_then(|class| call_with(&class, &[ + match imported(py, "sqlbuild._native", "PlannerInputError").and_then(|class| call_with(&class, &[ ("message", string(py, &message)), ("code", string(py, "S303")), ("help", string(py, "select models with one cursor type, or set each type explicitly with --start-cursor-ts/--end-cursor-ts and --start-cursor-int/--end-cursor-int")), ])) { Ok(error) => PyErr::from_value(error), Err(error) => error } })?; - let kind = imported(py, "sqlbuild.compiler.planner.types", "CursorType")?.call1((kind,))?; + let kind = imported(py, "sqlbuild._native", "CursorType")?.call1((kind,))?; method(&overrides, "typed_as", &[("cursor_type", kind)]) } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_resolution/cursor_ranges.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_resolution/cursor_ranges.rs index a62f123cf0..8b2bcb6e40 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_resolution/cursor_ranges.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_resolution/cursor_ranges.rs @@ -22,7 +22,7 @@ use sqlbuild_cursor::planner::main::rolling_window::rolling_window; use sqlbuild_cursor::planner::models::CursorEndClamp; use std::cmp::Ordering; -const MODELS: &str = "sqlbuild.compiler.planner.models"; +const MODELS: &str = "sqlbuild._native"; const CURSOR_TYPES: &str = "sqlbuild.cursor_algebra.types"; const CURSOR_MODELS: &str = "sqlbuild._native"; const COMPILE_EXCEPTIONS: &str = "sqlbuild.compiler.compile.exceptions"; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_resolution/cursor_safety.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_resolution/cursor_safety.rs index 3e72238c4b..104c8965de 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_resolution/cursor_safety.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_resolution/cursor_safety.rs @@ -7,8 +7,8 @@ use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyErr, PyResult, Python}; use sqlbuild_cursor::algebra::models::{DurationSpan, Grain, Scalar}; use sqlbuild_cursor::planner::main::safety_horizon::safety_horizon; -pub(super) const MODELS: &str = "sqlbuild.compiler.planner.models"; -pub(super) const EXCEPTIONS: &str = "sqlbuild.compiler.planner.exceptions"; +pub(super) const MODELS: &str = "sqlbuild._native"; +pub(super) const EXCEPTIONS: &str = "sqlbuild._native"; pub(super) fn safety_error(py: Python<'_>, class: &str, message: &str) -> PyErr { match imported(py, EXCEPTIONS, class).and_then(|class| class.call1((message,))) { diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_resolution/cursor_windows.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_resolution/cursor_windows.rs index c3342d244e..29be985ba0 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_resolution/cursor_windows.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_resolution/cursor_windows.rs @@ -10,7 +10,7 @@ use pyo3::{IntoPyObject, pyfunction, wrap_pyfunction}; use sqlbuild_cursor::algebra::main::floor_to_grain::floor_to_grain; use sqlbuild_cursor::planner::main::empty_inputs::empty_inputs; -const MODELS: &str = "sqlbuild.compiler.planner.models"; +const MODELS: &str = "sqlbuild._native"; #[pyfunction] #[pyo3(signature = (*, empty_input_names, waits_for_every_input, inputs_with_rows, has_window_start))] @@ -143,11 +143,7 @@ fn unreadable_cursor_inputs_error<'py>( input_names: Vec, ) -> PyResult> { call_with( - &imported( - py, - "sqlbuild.compiler.planner.exceptions", - "PlannerInputError", - )?, + &imported(py, "sqlbuild._native", "PlannerInputError")?, &[ ( "message", diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_resolution/orchestration.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_resolution/orchestration.rs index 600cfd0673..ee0a98d2fe 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_resolution/orchestration.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_resolution/orchestration.rs @@ -10,7 +10,7 @@ use pyo3::types::{PyBool, PyDict, PyDictMethods, PyList, PyString, PyTuple}; use pyo3::{IntoPyObject, PyErr, pyfunction, wrap_pyfunction}; const HOOK_MODULE: &str = "sqlbuild.compiler.discovery.models"; -const ERROR_MODULE: &str = "sqlbuild.compiler.planner.exceptions"; +const ERROR_MODULE: &str = "sqlbuild._native"; struct Resolution<'py> { adapter: Bound<'py, PyAny>, diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_scenario/cli.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_scenario/cli.rs index 04db2673d2..9a24f89aa1 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_scenario/cli.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_scenario/cli.rs @@ -161,11 +161,7 @@ fn raise_for_warnings(warnings: &Bound<'_, PyAny>) -> PyResult<()> { } if let Some(code) = code { return Err(PyErr::from_value(call_with( - &imported( - py, - "sqlbuild.compiler.planner.exceptions", - "PlannerInputError", - )?, + &imported(py, "sqlbuild._native", "PlannerInputError")?, &[ ("message", string(py, &messages.join("\n"))), ("code", code), diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_scenario/values.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_scenario/values.rs index a5e5f5805d..57b3d25c93 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_scenario/values.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_scenario/values.rs @@ -4,8 +4,8 @@ use crate::bindings::_helpers::compile_driver::python_calls::{call_with, importe use pyo3::prelude::{Bound, PyAny, PyErr, PyResult, Python}; use pyo3::types::PyString; -pub(super) const MODELS: &str = "sqlbuild.compiler.planner.models"; -pub(super) const TYPES: &str = "sqlbuild.compiler.planner.types"; +pub(super) const MODELS: &str = "sqlbuild._native"; +pub(super) const TYPES: &str = "sqlbuild._native"; pub(super) fn string<'py>(py: Python<'py>, text: &str) -> Bound<'py, PyAny> { PyString::new(py, text).into_any() @@ -22,11 +22,7 @@ pub(super) fn error( args.push(("help", string(py, help))); } Ok(PyErr::from_value(call_with( - &imported( - py, - "sqlbuild.compiler.planner.exceptions", - "PlannerInputError", - )?, + &imported(py, "sqlbuild._native", "PlannerInputError")?, &args, )?)) } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_scope/values.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_scope/values.rs index e675a9ce32..56545f3785 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_scope/values.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_scope/values.rs @@ -8,7 +8,7 @@ use sqlbuild_analysis::assembly::project::types::ObjectKey; use sqlbuild_analysis::planner_scope::models::LoaderGraph; pub(crate) const NATIVE: &str = "sqlbuild._native"; -pub(crate) const MODELS: &str = "sqlbuild.compiler.planner.models"; +pub(crate) const MODELS: &str = "sqlbuild._native"; pub(crate) const SOURCE_MODELS: &str = "sqlbuild._native"; pub(crate) fn keys(values: &Bound<'_, PyAny>) -> PyResult> { diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_selection/carrier.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_selection/carrier.rs index 0bf31dc9c1..a34b4384f6 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_selection/carrier.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_selection/carrier.rs @@ -8,7 +8,7 @@ use sqlbuild_analysis::graph::errors::SelectorError; use crate::bindings::_helpers::compile_driver::python_calls::imported; -const PLANNER_EXCEPTIONS_MODULE: &str = "sqlbuild.compiler.planner.exceptions"; +const PLANNER_EXCEPTIONS_MODULE: &str = "sqlbuild._native"; const PRIVATE_USE: std::ops::Range = 0xF0000..0xFFFFE; /// `str.translate` tables to and from carrier characters; empty when no text holds a surrogate. diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_strategy/constants.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_strategy/constants.rs index d46eb709b9..8c7249e73c 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_strategy/constants.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_strategy/constants.rs @@ -4,10 +4,10 @@ pub(crate) const NATIVE_MODULE: &str = "sqlbuild._native"; pub(crate) const AUDIT_TYPES_MODULE: &str = "sqlbuild.compiler.auditing.types"; pub(crate) const COMPILE_TYPES_MODULE: &str = "sqlbuild.compiler.compile.types"; pub(crate) const COMPILE_MODELS_MODULE: &str = "sqlbuild.compiler.compile.models"; -pub(crate) const PLAN_MODELS_MODULE: &str = "sqlbuild.compiler.planner.models"; +pub(crate) const PLAN_MODELS_MODULE: &str = "sqlbuild._native"; pub(crate) const MODEL_ATTACHMENT: &str = "model"; pub(crate) const SOURCE_ATTACHMENT: &str = "source"; -pub(crate) const PLANNER_TYPES_MODULE: &str = "sqlbuild.compiler.planner.types"; +pub(crate) const PLANNER_TYPES_MODULE: &str = "sqlbuild._native"; pub(crate) const ADAPTER_MODELS_MODULE: &str = "sqlbuild.adapter.contract.models"; pub(crate) const CURSOR_TYPES_MODULE: &str = "sqlbuild.cursor_algebra.types"; pub(crate) const INCREMENTAL_MATERIALIZATION: &str = "incremental"; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_strategy/values.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_strategy/values.rs index 61936868fe..01d145e27a 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_strategy/values.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_strategy/values.rs @@ -15,7 +15,7 @@ use sqlbuild_analysis::planner_strategy::models::{ use crate::bindings::_helpers::compile_driver::python_calls::{call_with, imported}; -pub(crate) const PLANNER_EXCEPTIONS_MODULE: &str = "sqlbuild.compiler.planner.exceptions"; +pub(crate) const PLANNER_EXCEPTIONS_MODULE: &str = "sqlbuild._native"; pub(crate) const COMPILE_EXCEPTIONS_MODULE: &str = "sqlbuild.compiler.compile.exceptions"; const YML_SOURCE: &str = "yml"; const FULL_BACKFILL: &str = "full"; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/record_behaviours/call_arguments.rs b/crates/sqlbuild-python/src/bindings/_helpers/record_behaviours/call_arguments.rs new file mode 100644 index 0000000000..700d176871 --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/record_behaviours/call_arguments.rs @@ -0,0 +1,18 @@ +//! Reading the arguments of a native method call. + +use pyo3::prelude::{Bound, PyAny, PyResult}; +use pyo3::types::{PyDict, PyDictMethods, PyTuple, PyTupleMethods}; + +/// The argument `name`, passed by keyword or at `position`. +pub(crate) fn call_argument<'py>( + (args, kwargs): (&Bound<'py, PyTuple>, Option<&Bound<'py, PyDict>>), + name: &str, + position: usize, +) -> PyResult> { + if let Some(kwargs) = kwargs + && let Some(value) = kwargs.get_item(name)? + { + return Ok(value); + } + args.get_item(position) +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/record_behaviours/constants.rs b/crates/sqlbuild-python/src/bindings/_helpers/record_behaviours/constants.rs index fcc457bb4f..46e169df58 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/record_behaviours/constants.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/record_behaviours/constants.rs @@ -4,6 +4,11 @@ use crate::bindings::_helpers::record_behaviours::auditing::{ finish_measurement_contract, finish_threshold_bound, finish_thresholds, }; use crate::bindings::_helpers::record_behaviours::config::settings_sql_validation; +use crate::bindings::_helpers::record_behaviours::cursors::{ + finish_cursor_bounds, finish_cursor_snapshot, finish_future_evidence, finish_input_evidence, + finish_input_relation, finish_maximum_evidence, input_relation_runtime_owned, + snapshot_watermarks_available, +}; use crate::bindings::_helpers::record_behaviours::durations::{ duration_add_to, duration_fixed_seconds, duration_has_calendar_component, duration_is_single_unit_in, duration_is_zero, duration_parse, duration_subtract_from, @@ -15,6 +20,19 @@ use crate::bindings::_helpers::record_behaviours::freshness::{ use crate::bindings::_helpers::record_behaviours::lexical::{ finish_lexical_syntax, lexical_cache_key, lexical_native_mapping, lexical_reads_differently, }; +use crate::bindings::_helpers::record_behaviours::migrations::{ + column_decision_blocks_build, column_decision_label, column_decision_records_event, + column_decision_renames, column_event_mentions, column_event_renamed, decision_blocks_build, + decision_checks_compatibility, decision_moves_data, event_mentions, history_new, history_old, + history_status, relation_identity_property, relation_matches, value_plan_label, +}; +use crate::bindings::_helpers::record_behaviours::planner::{ + column_migration_blocks_build, column_migration_origin_missing, finish_cursor_overrides, + materialization_is_table_backed, model_migration_blocks_build, model_migration_origin_hidden, + model_migration_origin_missing, model_migration_storage_transition, + model_migration_target_label, old_name_runs_steps, overrides_have_untyped, overrides_typed_as, + plan_python_source_entries, retention_decreases, +}; use crate::bindings::_helpers::record_behaviours::python_nodes::graph_selected_sql_refs; use crate::bindings::_helpers::record_behaviours::scopes::{ completeness_complete, target_query_unknown, @@ -33,6 +51,13 @@ pub(crate) const BEHAVIOUR_FINISHERS: &[(&str, RecordFinish)] = &[ ("SourceFreshnessRecord", finish_freshness_record), ("SqlLexicalSyntax", finish_lexical_syntax), ("AlignedInterval", finish_aligned_interval), + ("CursorOverrides", finish_cursor_overrides), + ("ModelCursorSnapshot", finish_cursor_snapshot), + ("CursorBounds", finish_cursor_bounds), + ("CursorInputEvidence", finish_input_evidence), + ("FutureCursorSafetyEvidence", finish_future_evidence), + ("MaximumStartSafetyEvidence", finish_maximum_evidence), + ("CursorInputRelation", finish_input_relation), ]; /// Computed attributes of compiler shapes. pub(crate) const BEHAVIOUR_PROPERTIES: &[(&str, &str, ClassProperty)] = &[ @@ -57,6 +82,87 @@ pub(crate) const BEHAVIOUR_PROPERTIES: &[(&str, &str, ClassProperty)] = &[ ), ("Duration", "fixed_seconds", duration_fixed_seconds), ("Duration", "total_months", duration_total_months), + ("MigrationDecision", "moves_data", decision_moves_data), + ("MigrationDecision", "blocks_build", decision_blocks_build), + ( + "MigrationDecision", + "checks_compatibility", + decision_checks_compatibility, + ), + ("MigrationDecision", "label", value_plan_label), + ("MigrationPromotion", "label", value_plan_label), + ( + "ColumnMigrationDecision", + "renames", + column_decision_renames, + ), + ( + "ColumnMigrationDecision", + "records_event", + column_decision_records_event, + ), + ( + "ColumnMigrationDecision", + "blocks_build", + column_decision_blocks_build, + ), + ("ColumnMigrationDecision", "label", column_decision_label), + ("MigrationRelation", "identity", relation_identity_property), + ("OldNameViewHistory", "old", history_old), + ("OldNameViewHistory", "new", history_new), + ("CursorOverrides", "has_untyped", overrides_have_untyped), + ( + "ModelCursorSnapshot", + "watermarks_available", + snapshot_watermarks_available, + ), + ( + "CursorInputRelation", + "is_runtime_owned", + input_relation_runtime_owned, + ), + ("RetentionPlanEntry", "decreases", retention_decreases), + ( + "ModelMigrationPlanEntry", + "origin_missing", + model_migration_origin_missing, + ), + ( + "ModelMigrationPlanEntry", + "origin_hidden", + model_migration_origin_hidden, + ), + ( + "ModelMigrationPlanEntry", + "storage_transition", + model_migration_storage_transition, + ), + ( + "ModelMigrationPlanEntry", + "target_label", + model_migration_target_label, + ), + ( + "ModelMigrationPlanEntry", + "blocks_build", + model_migration_blocks_build, + ), + ("OldNameViewPlanEntry", "runs_steps", old_name_runs_steps), + ( + "ColumnMigrationPlanEntry", + "origin_missing", + column_migration_origin_missing, + ), + ( + "ColumnMigrationPlanEntry", + "blocks_build", + column_migration_blocks_build, + ), + ( + "PlanOutput", + "python_source_entries", + plan_python_source_entries, + ), ]; /// Methods of compiler shapes. pub(crate) const BEHAVIOUR_METHODS: &[(&str, &str, ClassMethod)] = &[ @@ -73,6 +179,17 @@ pub(crate) const BEHAVIOUR_METHODS: &[(&str, &str, ClassMethod)] = &[ ), ("Duration", "is_single_unit_in", duration_is_single_unit_in), ("Duration", "add_to", duration_add_to), + ("MigrationRelation", "matches", relation_matches), + ("MigrationEvent", "mentions", event_mentions), + ("ColumnMigrationEvent", "mentions", column_event_mentions), + ("ColumnMigrationEvent", "renamed", column_event_renamed), + ("OldNameViewHistory", "status", history_status), + ( + "MaterializationType", + "is_table_backed", + materialization_is_table_backed, + ), + ("CursorOverrides", "typed_as", overrides_typed_as), ("Duration", "subtract_from", duration_subtract_from), ]; /// Compiler records ordered by their compared fields. diff --git a/crates/sqlbuild-python/src/bindings/_helpers/record_behaviours/cursors.rs b/crates/sqlbuild-python/src/bindings/_helpers/record_behaviours/cursors.rs new file mode 100644 index 0000000000..9fc69a029e --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/record_behaviours/cursors.rs @@ -0,0 +1,184 @@ +//! Cursor value normalization of the natively hosted planner cursor records. + +use pyo3::prelude::{Bound, Py, PyAny, PyAnyMethods, PyResult, Python}; +use pyo3::types::{PyString, PyTuple}; + +use crate::bindings::_helpers::compile_driver::python_calls::imported; +use crate::bindings::_helpers::cursor::scalars::parse_value; +use crate::bindings::_helpers::records::constants::NATIVE_MODULE; + +/// One cursor value conversion. +type Conversion = for<'py> fn(&Bound<'py, PyAny>) -> PyResult>; + +const SCALAR_CLASSES: [&str; 3] = ["TimestampValue", "DateValue", "IntegerValue"]; +const CURSOR_TYPES_MODULE: &str = "sqlbuild.cursor_algebra.types"; +const BOUND_SENTINEL: &str = "BoundSentinel"; +const CURSOR_ERRORS_MODULE: &str = "sqlbuild.cursor_algebra.exceptions"; +const CURSOR_ALGEBRA_ERROR: &str = "CursorAlgebraError"; + +/// A cursor scalar from `raw`: typed values unchanged, text read as an integer, else a timestamp. +pub(crate) fn inferred<'py>(raw: &Bound<'py, PyAny>) -> PyResult> { + let py: Python<'py> = raw.py(); + for class in SCALAR_CLASSES { + if raw.is_instance(&imported(py, NATIVE_MODULE, class)?)? { + return Ok(raw.clone()); + } + } + let text: Bound<'py, PyString> = raw.str()?; + match parse_value(text.as_any(), &PyString::new(py, "integer").into_any()) { + Ok(value) => Ok(value), + Err(error) + if error.is_instance( + py, + &imported(py, CURSOR_ERRORS_MODULE, CURSOR_ALGEBRA_ERROR)?, + ) => + { + parse_value(text.as_any(), &PyString::new(py, "timestamp").into_any()) + } + Err(error) => Err(error), + } +} + +/// `inferred(raw)`, keeping `None`. +pub(crate) fn optional_inferred<'py>(raw: &Bound<'py, PyAny>) -> PyResult> { + if raw.is_none() { + return Ok(raw.clone()); + } + inferred(raw) +} + +/// A cursor bound: a sentinel or its token text as the sentinel, else an inferred scalar. +fn bound<'py>(raw: &Bound<'py, PyAny>) -> PyResult> { + let sentinel: Bound<'py, PyAny> = imported(raw.py(), CURSOR_TYPES_MODULE, BOUND_SENTINEL)?; + if raw.is_instance(&sentinel)? { + return Ok(raw.clone()); + } + for member in sentinel.try_iter()? { + let member: Bound<'py, PyAny> = member?; + if raw.eq(member.getattr("value")?)? { + return Ok(member); + } + } + inferred(raw) +} + +/// `values` with `convert` applied at each of `indexes`. +fn converted( + py: Python<'_>, + mut values: Vec>, + (indexes, convert): (&[usize], Conversion), +) -> PyResult>> { + for index in indexes { + let value: Bound<'_, PyAny> = convert(values[*index].bind(py))?; + values[*index] = value.unbind(); + } + Ok(values) +} + +/// `values` with the items at each of `indexes` read as a tuple of cursor scalars. +fn converted_tuples( + py: Python<'_>, + mut values: Vec>, + indexes: &[usize], +) -> PyResult>> { + for index in indexes { + let items: Vec> = values[*index] + .bind(py) + .try_iter()? + .map(|item| inferred(&item?)) + .collect::>()?; + values[*index] = PyTuple::new(py, items)?.into_any().unbind(); + } + Ok(values) +} + +/// `ModelCursorSnapshot`: watermark values parsed as cursor scalars. +pub(crate) fn finish_cursor_snapshot( + py: Python<'_>, + values: Vec>, +) -> PyResult>> { + let mut values: Vec> = converted(py, values, (&[0, 3, 4], optional_inferred))?; + values = converted_tuples(py, values, &[1, 2, 11, 12, 14])?; + let pairs: Vec> = values[13] + .bind(py) + .try_iter()? + .map(|pair| { + let pair: Bound<'_, PyAny> = pair?; + PyTuple::new( + py, + [ + optional_inferred(&pair.get_item(0)?)?, + optional_inferred(&pair.get_item(1)?)?, + ], + ) + }) + .collect::>()?; + values[13] = PyTuple::new(py, pairs)?.into_any().unbind(); + let mode: Bound<'_, PyAny> = + imported(py, NATIVE_MODULE, "CursorWatermarkMode")?.call1((values[10].bind(py),))?; + values[10] = mode.unbind(); + Ok(values) +} + +/// `ModelCursorSnapshot.watermarks_available`: whether every physical watermark was read. +pub(crate) fn snapshot_watermarks_available(instance: &Bound<'_, PyAny>) -> PyResult> { + let available: bool = !instance + .getattr("unavailable_watermark_tags")? + .is_truthy()?; + Ok(pyo3::types::PyBool::new(instance.py(), available) + .to_owned() + .into_any() + .unbind()) +} + +/// `CursorBounds`: start and end read as bounds. +pub(crate) fn finish_cursor_bounds( + py: Python<'_>, + values: Vec>, +) -> PyResult>> { + converted(py, values, (&[0, 1], bound)) +} + +/// `CursorInputEvidence`: an optional minimum and a maximum scalar. +pub(crate) fn finish_input_evidence( + py: Python<'_>, + values: Vec>, +) -> PyResult>> { + let values: Vec> = converted(py, values, (&[2], optional_inferred))?; + converted(py, values, (&[3], inferred)) +} + +/// `FutureCursorSafetyEvidence`: its seven cursor values read as scalars. +pub(crate) fn finish_future_evidence( + py: Python<'_>, + values: Vec>, +) -> PyResult>> { + converted(py, values, (&[2, 3, 4, 5, 6, 7, 8], inferred)) +} + +/// `MaximumStartSafetyEvidence`: its cursor values read as scalars. +pub(crate) fn finish_maximum_evidence( + py: Python<'_>, + values: Vec>, +) -> PyResult>> { + let values: Vec> = converted(py, values, (&[2, 3, 5, 6], inferred))?; + converted(py, values, (&[4], optional_inferred)) +} + +/// `CursorInputRelation`: its grain as a `CursorGrain` and its terminal bounds as scalars. +pub(crate) fn finish_input_relation( + py: Python<'_>, + mut values: Vec>, +) -> PyResult>> { + if !values[2].bind(py).is_none() { + let grain: Bound<'_, PyAny> = + imported(py, NATIVE_MODULE, "CursorGrain")?.call1((values[2].bind(py),))?; + values[2] = grain.unbind(); + } + converted(py, values, (&[5, 6], optional_inferred)) +} + +/// `CursorInputRelation.is_runtime_owned`: whether bounds are read after upstream execution. +pub(crate) fn input_relation_runtime_owned(instance: &Bound<'_, PyAny>) -> PyResult> { + Ok(instance.getattr("is_runtime_produced")?.unbind()) +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/record_behaviours/migrations.rs b/crates/sqlbuild-python/src/bindings/_helpers/record_behaviours/migrations.rs new file mode 100644 index 0000000000..86cba9622e --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/record_behaviours/migrations.rs @@ -0,0 +1,210 @@ +//! Behaviour of the natively hosted migration decisions and migration state records. + +use pyo3::IntoPyObjectExt; +use pyo3::prelude::{Bound, Py, PyAny, PyAnyMethods, PyResult, Python}; +use pyo3::types::{PyDict, PyTuple}; + +use sqlbuild_core::migration_values::main::column_decision_traits::column_decision_traits; +use sqlbuild_core::migration_values::main::migration_decision_traits::migration_decision_traits; +use sqlbuild_core::migration_values::main::plan_label::plan_label; +use sqlbuild_core::migration_values::models::{ColumnDecisionTraits, DecisionTraits}; + +use crate::bindings::_helpers::compile_driver::python_calls::imported; +use crate::bindings::_helpers::record_behaviours::call_arguments::call_argument; +use crate::bindings::_helpers::records::constants::NATIVE_MODULE; + +const VIEW_STATUS: &str = "OldNameViewStatus"; + +fn value(member: &Bound<'_, PyAny>) -> PyResult { + member.str()?.extract() +} + +fn model_traits(member: &Bound<'_, PyAny>) -> PyResult { + Ok(migration_decision_traits(&value(member)?)) +} + +fn column_traits(member: &Bound<'_, PyAny>) -> PyResult { + Ok(column_decision_traits(&value(member)?)) +} + +/// `MigrationDecision.moves_data`. +pub(crate) fn decision_moves_data(member: &Bound<'_, PyAny>) -> PyResult> { + model_traits(member)?.moves_data.into_py_any(member.py()) +} + +/// `MigrationDecision.blocks_build`. +pub(crate) fn decision_blocks_build(member: &Bound<'_, PyAny>) -> PyResult> { + model_traits(member)?.blocks_build.into_py_any(member.py()) +} + +/// `MigrationDecision.checks_compatibility`. +pub(crate) fn decision_checks_compatibility(member: &Bound<'_, PyAny>) -> PyResult> { + model_traits(member)? + .checks_compatibility + .into_py_any(member.py()) +} + +/// `MigrationDecision.label` and `MigrationPromotion.label`. +pub(crate) fn value_plan_label(member: &Bound<'_, PyAny>) -> PyResult> { + plan_label(&value(member)?).into_py_any(member.py()) +} + +/// `ColumnMigrationDecision.renames`. +pub(crate) fn column_decision_renames(member: &Bound<'_, PyAny>) -> PyResult> { + column_traits(member)?.renames.into_py_any(member.py()) +} + +/// `ColumnMigrationDecision.records_event`. +pub(crate) fn column_decision_records_event(member: &Bound<'_, PyAny>) -> PyResult> { + column_traits(member)? + .records_event + .into_py_any(member.py()) +} + +/// `ColumnMigrationDecision.blocks_build`. +pub(crate) fn column_decision_blocks_build(member: &Bound<'_, PyAny>) -> PyResult> { + column_traits(member)?.blocks_build.into_py_any(member.py()) +} + +/// `ColumnMigrationDecision.label`. +pub(crate) fn column_decision_label(member: &Bound<'_, PyAny>) -> PyResult> { + match column_traits(member)?.label { + Some(label) => label.into_py_any(member.py()), + None => Err(pyo3::exceptions::PyKeyError::new_err(value(member)?)), + } +} + +/// `text.lower()`, keeping `None`. +fn lowered<'py>(text: &Bound<'py, PyAny>) -> PyResult> { + if text.is_none() { + return Ok(text.clone()); + } + text.call_method0("lower") +} + +/// `(database.lower() or None, (schema or "").lower(), name.lower())`. +fn relation_identity<'py>(relation: &Bound<'py, PyAny>) -> PyResult> { + let py: Python<'py> = relation.py(); + let schema: Bound<'py, PyAny> = relation.getattr("schema")?; + let schema: Bound<'py, PyAny> = if schema.is_none() { + "".into_bound_py_any(py)? + } else { + schema.call_method0("lower")? + }; + PyTuple::new( + py, + [ + lowered(&relation.getattr("database")?)?, + schema, + relation.getattr("name")?.call_method0("lower")?, + ], + ) +} + +/// Whether two relations match, an unknown database matching any database. +fn relations_match(left: &Bound<'_, PyAny>, right: &Bound<'_, PyAny>) -> PyResult { + let left: Bound<'_, PyTuple> = relation_identity(left)?; + let right: Bound<'_, PyTuple> = relation_identity(right)?; + if !left.get_item(1)?.eq(right.get_item(1)?)? || !left.get_item(2)?.eq(right.get_item(2)?)? { + return Ok(false); + } + let (left, right) = (left.get_item(0)?, right.get_item(0)?); + Ok(left.is_none() || right.is_none() || left.eq(right)?) +} + +/// `MigrationRelation.identity`: the case-insensitive relation identity. +pub(crate) fn relation_identity_property(relation: &Bound<'_, PyAny>) -> PyResult> { + Ok(relation_identity(relation)?.into_any().unbind()) +} + +/// `MigrationRelation.matches(other)`. +pub(crate) fn relation_matches( + relation: &Bound<'_, PyAny>, + args: &Bound<'_, PyTuple>, + kwargs: Option<&Bound<'_, PyDict>>, +) -> PyResult> { + let other: Bound<'_, PyAny> = call_argument((args, kwargs), "other", 0)?; + relations_match(relation, &other)?.into_py_any(relation.py()) +} + +/// `MigrationEvent.mentions(relation)`: the relation is the event's origin or destination. +pub(crate) fn event_mentions( + event: &Bound<'_, PyAny>, + args: &Bound<'_, PyTuple>, + kwargs: Option<&Bound<'_, PyDict>>, +) -> PyResult> { + let relation: Bound<'_, PyAny> = call_argument((args, kwargs), "relation", 0)?; + let mentions: bool = relations_match(&event.getattr("origin")?, &relation)? + || relations_match(&event.getattr("destination")?, &relation)?; + mentions.into_py_any(event.py()) +} + +/// `ColumnMigrationEvent.mentions(relation=..., columns=...)`. +pub(crate) fn column_event_mentions( + event: &Bound<'_, PyAny>, + args: &Bound<'_, PyTuple>, + kwargs: Option<&Bound<'_, PyDict>>, +) -> PyResult> { + let relation: Bound<'_, PyAny> = call_argument((args, kwargs), "relation", 0)?; + let columns: Bound<'_, PyAny> = call_argument((args, kwargs), "columns", 1)?; + let mentions: bool = relations_match(&event.getattr("relation")?, &relation)? + && (columns.contains(event.getattr("origin_column")?.call_method0("lower")?)? + || columns.contains(event.getattr("destination_column")?.call_method0("lower")?)?); + mentions.into_py_any(event.py()) +} + +/// `ColumnMigrationEvent.renamed(origin_column=..., destination_column=...)`. +pub(crate) fn column_event_renamed( + event: &Bound<'_, PyAny>, + args: &Bound<'_, PyTuple>, + kwargs: Option<&Bound<'_, PyDict>>, +) -> PyResult> { + let same = |field: &str, name: &str, position: usize| -> PyResult { + let given: Bound<'_, PyAny> = call_argument((args, kwargs), name, position)?; + event + .getattr(field)? + .call_method0("lower")? + .eq(given.call_method0("lower")?) + }; + let renamed: bool = same("origin_column", "origin_column", 0)? + && same("destination_column", "destination_column", 1)?; + renamed.into_py_any(event.py()) +} + +/// `OldNameViewHistory.status(now=...)`: the current state of the move's old-name steps. +pub(crate) fn history_status( + history: &Bound<'_, PyAny>, + args: &Bound<'_, PyTuple>, + kwargs: Option<&Bound<'_, PyDict>>, +) -> PyResult> { + let py: Python<'_> = history.py(); + let now: Bound<'_, PyAny> = call_argument((args, kwargs), "now", 0)?; + let created: Bound<'_, PyAny> = history.getattr("created")?; + let status: &str = if !history.getattr("dropped")?.is_none() { + "DROPPED" + } else if !created.is_none() { + let expires_at: Bound<'_, PyAny> = created.getattr("expires_at")?; + if !expires_at.is_none() && expires_at.le(&now)? { + "EXPIRED" + } else { + "LIVE" + } + } else if !history.getattr("archived")?.is_none() { + "PENDING_VIEW" + } else { + "PENDING_ARCHIVE" + }; + Ok(imported(py, NATIVE_MODULE, VIEW_STATUS)? + .getattr(status)? + .unbind()) +} + +/// `OldNameViewHistory.old`: the relation name the compatibility view occupies. +pub(crate) fn history_old(history: &Bound<'_, PyAny>) -> PyResult> { + Ok(history.getattr("required")?.getattr("old")?.unbind()) +} + +/// `OldNameViewHistory.new`: the relation the compatibility view reads. +pub(crate) fn history_new(history: &Bound<'_, PyAny>) -> PyResult> { + Ok(history.getattr("required")?.getattr("new")?.unbind()) +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/record_behaviours/mod.rs b/crates/sqlbuild-python/src/bindings/_helpers/record_behaviours/mod.rs index bd1cc759be..8c4dadf5a8 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/record_behaviours/mod.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/record_behaviours/mod.rs @@ -1,9 +1,13 @@ pub(crate) mod auditing; +pub(crate) mod call_arguments; pub(crate) mod config; pub(crate) mod constants; +pub(crate) mod cursors; pub(crate) mod durations; pub(crate) mod freshness; pub(crate) mod lexical; +pub(crate) mod migrations; +pub(crate) mod planner; pub(crate) mod python_nodes; pub(crate) mod scopes; pub(crate) mod values; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/record_behaviours/planner.rs b/crates/sqlbuild-python/src/bindings/_helpers/record_behaviours/planner.rs new file mode 100644 index 0000000000..5ccafc6aea --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/record_behaviours/planner.rs @@ -0,0 +1,311 @@ +//! Computed attributes of the natively hosted plan entry records and cursor overrides. + +use pyo3::IntoPyObjectExt; +use pyo3::prelude::{Bound, Py, PyAny, PyAnyMethods, PyErr, PyResult, Python}; +use pyo3::types::{PyDict, PyDictMethods, PyString, PyTuple}; + +use sqlbuild_core::migration_values::main::migration_decision_traits::migration_decision_traits; +use sqlbuild_core::migration_values::main::model_migration_blocks_build::model_migration_blocks_build as blocks_build; +use sqlbuild_core::migration_values::main::storage_transition::storage_transition; +use sqlbuild_core::migration_values::models::{DecisionTraits, ModelMigrationFacts}; + +use crate::bindings::_helpers::compile_driver::python_calls::imported; +use crate::bindings::_helpers::cursor::scalars::parse_value; +use crate::bindings::_helpers::record_behaviours::call_arguments::call_argument; +use crate::bindings::_helpers::records::constants::NATIVE_MODULE; + +const DECREASING_DIRECTIONS: [&str; 2] = ["decrease", "mixed"]; +const STEP_ACTIONS: [&str; 2] = ["archive_and_view", "view_only"]; +const TABLE_BACKED: [&str; 3] = ["table", "incremental", "snapshot"]; +const ORIGIN_MISSING: &str = "origin_missing"; +const SOURCE_MISSING: &str = "source_missing"; +const DENY_POLICY: &str = "deny"; +const INCOMPATIBLE: &str = "incompatible"; +const DEFAULT_TARGET_LABEL: &str = "default"; +const PLANNER_INPUT_ERROR: &str = "PlannerInputError"; +const INTEGER_CURSOR: &str = "integer"; +/// `CursorOverrides` field positions: `start_ts`, `end_ts`, `start_int`, `end_int`, `start`, `end`. +const OVERRIDE_FIELDS: [&str; 6] = ["start_ts", "end_ts", "start_int", "end_int", "start", "end"]; +const TIMESTAMP_FLAGS: [&str; 2] = ["--start-cursor-ts", "--end-cursor-ts"]; +const INTEGER_FLAGS: [&str; 2] = ["--start-cursor-int", "--end-cursor-int"]; +const UNTYPED_FLAGS: [&str; 2] = ["--start-cursor", "--end-cursor"]; + +fn text(value: &Bound<'_, PyAny>) -> PyResult { + value.str()?.extract() +} + +fn flag(py: Python<'_>, value: bool) -> PyResult> { + value.into_py_any(py) +} + +/// `RetentionPlanEntry.decreases`: whether the entry lowers any live retention value. +pub(crate) fn retention_decreases(instance: &Bound<'_, PyAny>) -> PyResult> { + let direction: String = text(&instance.getattr("direction")?)?; + flag( + instance.py(), + DECREASING_DIRECTIONS.contains(&direction.as_str()), + ) +} + +/// `OldNameViewPlanEntry.runs_steps`: whether the build archives the old relation or creates the view. +pub(crate) fn old_name_runs_steps(instance: &Bound<'_, PyAny>) -> PyResult> { + let action: String = text(&instance.getattr("action")?)?; + flag(instance.py(), STEP_ACTIONS.contains(&action.as_str())) +} + +/// `PlanOutput.python_source_entries`: the source entries Python reads resolve. +pub(crate) fn plan_python_source_entries(instance: &Bound<'_, PyAny>) -> PyResult> { + for name in ["python_source_read_map", "source_read_map"] { + let entries: Bound<'_, PyAny> = instance.getattr(name)?; + if entries.is_truthy()? { + return Ok(entries.unbind()); + } + } + Ok(instance.getattr("source_map")?.unbind()) +} + +/// `MaterializationType.is_table_backed(materialized=...)`. +pub(crate) fn materialization_is_table_backed( + _class: &Bound<'_, PyAny>, + args: &Bound<'_, PyTuple>, + kwargs: Option<&Bound<'_, PyDict>>, +) -> PyResult> { + let materialized: Bound<'_, PyAny> = call_argument((args, kwargs), "materialized", 0)?; + let backed: bool = match materialized.downcast::() { + Ok(value) => TABLE_BACKED.contains(&value.extract::()?.as_str()), + Err(_) => false, + }; + flag(args.py(), backed) +} + +fn decision_traits(entry: &Bound<'_, PyAny>) -> PyResult<(String, DecisionTraits)> { + let decision: String = text(&entry.getattr("decision")?)?; + let traits: DecisionTraits = migration_decision_traits(&decision); + Ok((decision, traits)) +} + +fn model_facts(entry: &Bound<'_, PyAny>) -> PyResult { + let (decision, traits) = decision_traits(entry)?; + Ok(ModelMigrationFacts { + origin_missing: decision == ORIGIN_MISSING, + origin_tracked: entry.getattr("origin_tracked")?.is_truthy()?, + deny_missing_origin: text(&entry.getattr("missing_origin_policy")?)? == DENY_POLICY, + decision: traits, + incompatible: text(&entry.getattr("compatibility")?)? == INCOMPATIBLE, + }) +} + +/// `ModelMigrationPlanEntry.origin_missing`. +pub(crate) fn model_migration_origin_missing(entry: &Bound<'_, PyAny>) -> PyResult> { + flag(entry.py(), model_facts(entry)?.origin_missing) +} + +/// `ModelMigrationPlanEntry.origin_hidden`: a missing origin with build history in this target. +pub(crate) fn model_migration_origin_hidden(entry: &Bound<'_, PyAny>) -> PyResult> { + let facts: ModelMigrationFacts = model_facts(entry)?; + flag(entry.py(), facts.origin_missing && facts.origin_tracked) +} + +/// `ModelMigrationPlanEntry.storage_transition`: the table-type transition label, when known. +pub(crate) fn model_migration_storage_transition(entry: &Bound<'_, PyAny>) -> PyResult> { + let origin: bool = entry.getattr("origin_is_transient")?.is_truthy()?; + let stage: Bound<'_, PyAny> = entry.getattr("stage_is_transient")?; + let stage: Option = if stage.is_none() { + None + } else { + Some(stage.is_truthy()?) + }; + storage_transition(origin, stage).into_py_any(entry.py()) +} + +/// `ModelMigrationPlanEntry.target_label`: the target name, naming the unnamed target `default`. +pub(crate) fn model_migration_target_label(entry: &Bound<'_, PyAny>) -> PyResult> { + let target: Bound<'_, PyAny> = entry.getattr("target_name")?; + if target.is_truthy()? { + return Ok(target.unbind()); + } + DEFAULT_TARGET_LABEL.into_py_any(entry.py()) +} + +/// `ModelMigrationPlanEntry.blocks_build`: whether the migration stops a build before execution. +pub(crate) fn model_migration_blocks_build(entry: &Bound<'_, PyAny>) -> PyResult> { + flag(entry.py(), blocks_build(&model_facts(entry)?)) +} + +/// `ColumnMigrationPlanEntry.origin_missing`. +pub(crate) fn column_migration_origin_missing(entry: &Bound<'_, PyAny>) -> PyResult> { + flag( + entry.py(), + text(&entry.getattr("decision")?)? == SOURCE_MISSING, + ) +} + +/// `ColumnMigrationPlanEntry.blocks_build`. +pub(crate) fn column_migration_blocks_build(entry: &Bound<'_, PyAny>) -> PyResult> { + let decision: Bound<'_, PyAny> = entry.getattr("decision")?; + let blocks: bool = if text(&decision)? == SOURCE_MISSING { + text(&entry.getattr("missing_origin_policy")?)? == DENY_POLICY + } else { + decision.getattr("blocks_build")?.is_truthy()? + }; + flag(entry.py(), blocks) +} + +/// The `PlannerInputError` carrying `message`. +fn planner_error(py: Python<'_>, message: String) -> PyErr { + match imported(py, NATIVE_MODULE, PLANNER_INPUT_ERROR).and_then(|class| class.call1((message,))) + { + Ok(error) => PyErr::from_value(error), + Err(error) => error, + } +} + +/// A `--*-cursor-ts` value: text checked as an ISO timestamp and parsed, others unchanged. +fn override_timestamp<'py>( + flag_name: &str, + value: &Bound<'py, PyAny>, +) -> PyResult> { + let py: Python<'py> = value.py(); + if value.is_none() || !value.is_instance_of::() { + return Ok(value.clone()); + } + let datetime: Bound<'py, PyAny> = imported(py, "datetime", "datetime")?; + if let Err(error) = datetime.call_method1("fromisoformat", (value,)) { + let shown: String = error.value(py).str()?.extract()?; + return Err(planner_error( + py, + format!( + "{flag_name} value '{}' is not a valid ISO timestamp: {shown}", + text(value)? + ), + )); + } + parse_value(value, &PyString::new(py, "timestamp").into_any()) +} + +/// A `--*-cursor-int` value: text checked as a whole number and parsed, others unchanged. +fn override_integer<'py>( + flag_name: &str, + value: &Bound<'py, PyAny>, +) -> PyResult> { + let py: Python<'py> = value.py(); + if value.is_none() || !value.is_instance_of::() { + return Ok(value.clone()); + } + let shown: String = text(value)?; + let invalid = || { + planner_error( + py, + format!("{flag_name} value '{shown}' is not a valid integer"), + ) + }; + let Ok(decimal) = imported(py, "decimal", "Decimal")?.call1((value,)) else { + return Err(invalid()); + }; + let whole: Bound<'py, PyAny> = py.get_type::().call1((&decimal,))?; + if !decimal.eq(&whole)? { + return Err(planner_error( + py, + format!("{flag_name} value '{shown}' is not a whole number"), + )); + } + let parsed: Bound<'py, PyAny> = + parse_value(value, &PyString::new(py, INTEGER_CURSOR).into_any())?; + if parsed.is_instance(&imported(py, NATIVE_MODULE, "IntegerValue")?)? { + return Ok(parsed); + } + Err(invalid()) +} + +/// `CursorOverrides`: typed flag values parsed, untyped `start`/`end` kept as text. +pub(crate) fn finish_cursor_overrides( + py: Python<'_>, + mut values: Vec>, +) -> PyResult>> { + for (index, flag_name) in TIMESTAMP_FLAGS.iter().enumerate() { + let parsed: Bound<'_, PyAny> = override_timestamp(flag_name, values[index].bind(py))?; + values[index] = parsed.unbind(); + } + for (offset, flag_name) in INTEGER_FLAGS.iter().enumerate() { + let parsed: Bound<'_, PyAny> = override_integer(flag_name, values[2 + offset].bind(py))?; + values[2 + offset] = parsed.unbind(); + } + Ok(values) +} + +/// `CursorOverrides.has_untyped`: whether `--start-cursor` or `--end-cursor` awaits a type. +pub(crate) fn overrides_have_untyped(instance: &Bound<'_, PyAny>) -> PyResult> { + let untyped: bool = + !instance.getattr("start")?.is_none() || !instance.getattr("end")?.is_none(); + flag(instance.py(), untyped) +} + +/// `parsed or fallback`. +fn either<'py>( + parsed: Bound<'py, PyAny>, + fallback: Bound<'py, PyAny>, +) -> PyResult> { + if parsed.is_truthy()? { + return Ok(parsed); + } + Ok(fallback) +} + +/// `CursorOverrides.typed_as(cursor_type=...)`: `--start-cursor`/`--end-cursor` read as that type. +pub(crate) fn overrides_typed_as( + instance: &Bound<'_, PyAny>, + args: &Bound<'_, PyTuple>, + kwargs: Option<&Bound<'_, PyDict>>, +) -> PyResult> { + let py: Python<'_> = instance.py(); + if !overrides_have_untyped(instance)?.bind(py).is_truthy()? { + return Ok(instance.clone().unbind()); + } + let cursor_type: String = text(&call_argument((args, kwargs), "cursor_type", 0)?)?; + let field = |name: &str| instance.getattr(name); + let (start, end) = (field(OVERRIDE_FIELDS[4])?, field(OVERRIDE_FIELDS[5])?); + let keywords: Bound<'_, PyDict> = PyDict::new(py); + if cursor_type == INTEGER_CURSOR { + keywords.set_item("start_ts", field("start_ts")?)?; + keywords.set_item("end_ts", field("end_ts")?)?; + keywords.set_item( + "start_int", + either( + override_integer(UNTYPED_FLAGS[0], &start)?, + field("start_int")?, + )?, + )?; + keywords.set_item( + "end_int", + either(override_integer(UNTYPED_FLAGS[1], &end)?, field("end_int")?)?, + )?; + } else { + keywords.set_item( + "start_ts", + either( + override_timestamp(UNTYPED_FLAGS[0], &start)?, + field("start_ts")?, + )?, + )?; + keywords.set_item( + "end_ts", + either( + override_timestamp(UNTYPED_FLAGS[1], &end)?, + field("end_ts")?, + )?, + )?; + keywords.set_item("start_int", field("start_int")?)?; + keywords.set_item("end_int", field("end_int")?)?; + } + Ok(instance.get_type().call((), Some(&keywords))?.unbind()) +} + +/// `ModelPlanEntry.backfill`'s default: a forward-only backfill. +pub(crate) fn forward_only_backfill(py: Python<'_>) -> PyResult> { + let keywords: Bound<'_, PyDict> = PyDict::new(py); + keywords.set_item( + "action", + imported(py, NATIVE_MODULE, "BackfillAction")?.getattr("FORWARD_ONLY")?, + )?; + imported(py, NATIVE_MODULE, "BackfillResult")?.call((), Some(&keywords)) +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/records/constants.rs b/crates/sqlbuild-python/src/bindings/_helpers/records/constants.rs index 2fe3621e30..a85bee6a21 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/records/constants.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/records/constants.rs @@ -50,6 +50,7 @@ const CLI_ERRORS: &[ErrorSpec] = &[ doc: "Expected CLI-facing error that should be rendered without a traceback.", class_attributes: &[("code", ClassValue::Text("C000"))], required: &[], + optional: &[], fixed: &[], }, ErrorSpec { @@ -58,6 +59,7 @@ const CLI_ERRORS: &[ErrorSpec] = &[ doc: "Staged compile artifacts changed before publication, so nothing stale is deleted.", class_attributes: &[], required: &[], + optional: &[], fixed: &[], }, ErrorSpec { @@ -69,6 +71,7 @@ const CLI_ERRORS: &[ErrorSpec] = &[ ("exit_code", ClassValue::Int(2)), ], required: &[], + optional: &[], fixed: &[], }, ErrorSpec { @@ -77,6 +80,7 @@ const CLI_ERRORS: &[ErrorSpec] = &[ doc: "Expected query-diff failure with an explicit outcome and exit code.", class_attributes: &[], required: &["status", "exit_code"], + optional: &[], fixed: &[], }, ErrorSpec { @@ -85,6 +89,7 @@ const CLI_ERRORS: &[ErrorSpec] = &[ doc: "Safety or preflight prevented complete value evidence.", class_attributes: &[], required: &[], + optional: &[], fixed: &[ ("status", ClassValue::Text("incomplete")), ("exit_code", ClassValue::Int(2)), @@ -96,6 +101,7 @@ const CLI_ERRORS: &[ErrorSpec] = &[ doc: "Setup, execution, ownership publication, or cleanup failed.", class_attributes: &[], required: &[], + optional: &[], fixed: &[ ("status", ClassValue::Text("execution_failed")), ("exit_code", ClassValue::Int(3)), @@ -110,6 +116,7 @@ const CLI_ERRORS: &[ErrorSpec] = &[ ("exit_code", ClassValue::Int(2)), ], required: &["details"], + optional: &[], fixed: &[], }, ErrorSpec { @@ -118,6 +125,7 @@ const CLI_ERRORS: &[ErrorSpec] = &[ doc: "Raised when model migration state inputs or rows are invalid.", class_attributes: &[("code", ClassValue::Text("M000"))], required: &[], + optional: &[], fixed: &[], }, ]; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/records/error_classes.rs b/crates/sqlbuild-python/src/bindings/_helpers/records/error_classes.rs index c5d83e70b9..67c2079c95 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/records/error_classes.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/records/error_classes.rs @@ -115,7 +115,7 @@ fn initialize( if let Some(fields) = fields { for key in fields.keys() { let key: String = key.extract()?; - if !spec.required.contains(&key.as_str()) { + if !spec.required.contains(&key.as_str()) && !spec.optional.contains(&key.as_str()) { return Err(PyTypeError::new_err(format!( "{name}.__init__() got an unexpected keyword argument '{key}'" ))); @@ -148,6 +148,13 @@ fn initialize( }; error.setattr(*required, value)?; } + for optional in spec.optional { + let value = match fields { + Some(fields) => fields.get_item(optional)?, + None => None, + }; + error.setattr(*optional, value.unwrap_or_else(|| py.None().into_bound(py)))?; + } for (attribute, value) in spec.fixed { error.setattr(*attribute, class_value(py, *value)?)?; } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/records/fields.rs b/crates/sqlbuild-python/src/bindings/_helpers/records/fields.rs index bfb0b4f31c..72e86e794e 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/records/fields.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/records/fields.rs @@ -114,6 +114,7 @@ fn default_value<'py>( FieldDefault::EmptyFrozenset => Ok(PyFrozenSet::empty(py)?.into_any()), FieldDefault::Member(module, class, member) => imported(py, module, class)?.getattr(member), FieldDefault::Call(module, name) => imported(py, module, name)?.call0(), + FieldDefault::Factory(factory) => factory(py), FieldDefault::CallText(module, name, text) => imported(py, module, name)?.call1((text,)), } } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/records/models.rs b/crates/sqlbuild-python/src/bindings/_helpers/records/models.rs index b8fb97993b..309b81f8bd 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/records/models.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/records/models.rs @@ -9,7 +9,7 @@ use pyo3::{IntoPyObjectExt, pyclass, pymethods}; use crate::bindings::_helpers::records::fields::{ build_values, compared_values, frozen_error, ordered_comparison, record_spec, repr_text, }; -use crate::bindings::_helpers::records::types::{ClassMethod, ClassProperty}; +use crate::bindings::_helpers::records::types::{ClassMethod, ClassProperty, RecordFactory}; /// How a record field is filled when the caller does not pass it. #[derive(Debug, Clone, Copy)] @@ -31,6 +31,8 @@ pub(crate) enum FieldDefault { Member(&'static str, &'static str, &'static str), /// `getattr(import(module), name)()`, evaluated for every instance. Call(&'static str, &'static str), + /// A native factory, evaluated for every instance. + Factory(RecordFactory), /// `getattr(import(module), name)(text)`, evaluated for every instance. CallText(&'static str, &'static str, &'static str), } @@ -76,6 +78,8 @@ pub(crate) struct ErrorSpec { pub(crate) class_attributes: &'static [(&'static str, ClassValue)], /// Keyword-only constructor fields stored as attributes. pub(crate) required: &'static [&'static str], + /// Keyword-only constructor fields stored as attributes, `None` when not passed. + pub(crate) optional: &'static [&'static str], /// Attributes every instance is given. pub(crate) fixed: &'static [(&'static str, ClassValue)], } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/records/types.rs b/crates/sqlbuild-python/src/bindings/_helpers/records/types.rs index afd480ce0d..6bd40b1bda 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/records/types.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/records/types.rs @@ -12,3 +12,6 @@ pub(crate) type ClassProperty = fn(&Bound<'_, PyAny>) -> PyResult>; /// Runs one method of a native class with its receiver (the instance, or the class when unbound). pub(crate) type ClassMethod = fn(&Bound<'_, PyAny>, &Bound<'_, PyTuple>, Option<&Bound<'_, PyDict>>) -> PyResult>; + +/// Builds the default value of one record field. +pub(crate) type RecordFactory = fn(Python<'_>) -> PyResult>; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/refactor/offline_compile.rs b/crates/sqlbuild-python/src/bindings/_helpers/refactor/offline_compile.rs index 1aaca63c39..9a940ea320 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/refactor/offline_compile.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/refactor/offline_compile.rs @@ -15,7 +15,7 @@ const NATIVE_MODULE: &str = "sqlbuild._native"; const COMPILE_MODELS: &str = "sqlbuild.compiler.compile.models"; const COMPILE_TYPES: &str = "sqlbuild.compiler.compile.types"; const COMPILE_ERRORS: &str = "sqlbuild.compiler.compile.exceptions"; -const PLANNER_ERRORS: &str = "sqlbuild.compiler.planner.exceptions"; +const PLANNER_ERRORS: &str = "sqlbuild._native"; const RULES_ERRORS: &str = "sqlbuild.rule_engine.exceptions"; const SPEC_ERRORS: &str = "sqlbuild._native"; const ENTRY_MODELS: &str = "sqlbuild._native"; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/scenario_command/conversions.rs b/crates/sqlbuild-python/src/bindings/_helpers/scenario_command/conversions.rs index 486212aaf4..67c07cfe32 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/scenario_command/conversions.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/scenario_command/conversions.rs @@ -25,7 +25,7 @@ const FAIL_STATUS: &str = "FAIL"; const RUN_DIR: &str = "run"; const SCENARIOS_DIR: &str = "scenarios"; const LOCAL_DIR: &str = "local"; -const PLANNER_TYPES: &str = "sqlbuild.compiler.planner.types"; +const PLANNER_TYPES: &str = "sqlbuild._native"; const QUALIFIED_NAME_MODULE: &str = "sqlbuild.adapter.relations.main.resolve_relation_location_qualified_name"; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/semantic_shapes/shapes.rs b/crates/sqlbuild-python/src/bindings/_helpers/semantic_shapes/shapes.rs index 0b3d10ea2e..2f493570e6 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/semantic_shapes/shapes.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/semantic_shapes/shapes.rs @@ -10,7 +10,7 @@ use crate::bindings::_helpers::analysis::binding_catalog::{ use crate::bindings::_helpers::boundary::keyword_request::Request; const ADAPTER_MODELS_MODULE: &str = "sqlbuild.adapter.contract.models"; -const PLANNER_TYPES_MODULE: &str = "sqlbuild.compiler.planner.types"; +const PLANNER_TYPES_MODULE: &str = "sqlbuild._native"; const NATIVE_MODULE: &str = "sqlbuild._native"; const DATACLASSES_MODULE: &str = "dataclasses"; const UNKNOWN: &str = "UNKNOWN"; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/sql_test_command/test_command.rs b/crates/sqlbuild-python/src/bindings/_helpers/sql_test_command/test_command.rs index 2649457aa3..f9711d0da0 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/sql_test_command/test_command.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/sql_test_command/test_command.rs @@ -38,7 +38,7 @@ const NATIVE_MODULE: &str = "sqlbuild._native"; const EXTERNAL_REFS_MODULE: &str = "sqlbuild._native"; const STATIC_COMMAND_MODULE: &str = "sqlbuild._native"; const TEST_PLAN_MODULE: &str = "sqlbuild._native"; -const PLANNER_ERRORS: &str = "sqlbuild.compiler.planner.exceptions"; +const PLANNER_ERRORS: &str = "sqlbuild._native"; const EXECUTOR_PIPELINE_MODELS: &str = "sqlbuild.executor.pipeline.models"; const EXECUTOR_RUN_MODULE: &str = "sqlbuild.executor.pipeline.main.run"; const DECIMAL_KIND: &str = "decimal"; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/sql_test_glue/constants.rs b/crates/sqlbuild-python/src/bindings/_helpers/sql_test_glue/constants.rs index 5b41fbb0ec..463bd380c9 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/sql_test_glue/constants.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/sql_test_glue/constants.rs @@ -4,7 +4,7 @@ pub(super) const UDF: &str = "udf"; pub(super) const TABLE_FN: &str = "table_fn"; pub(super) const ERROR_SEVERITY: &str = "error"; pub(super) const COMPILE_TYPES: &str = "sqlbuild.compiler.compile.types"; -pub(super) const PLAN_MODELS: &str = "sqlbuild.compiler.planner.models"; +pub(super) const PLAN_MODELS: &str = "sqlbuild._native"; pub(super) const NATIVE_PLANNING: &str = "sqlbuild._native"; pub(super) const FIXTURE_PREFIXES: [(&str, &str); 4] = [ ("model", "__ref__"), @@ -13,7 +13,7 @@ pub(super) const FIXTURE_PREFIXES: [(&str, &str); 4] = [ ("sql_test", "__expected__"), ]; pub(super) const CURSOR_ALGEBRA_EXCEPTIONS: &str = "sqlbuild.cursor_algebra.exceptions"; -pub(super) const PLANNER_TYPES: &str = "sqlbuild.compiler.planner.types"; +pub(super) const PLANNER_TYPES: &str = "sqlbuild._native"; pub(super) const INTEGER_CURSOR_TYPE: &str = "integer"; pub(super) const TIMESTAMP_CURSOR_TYPE: &str = "timestamp"; pub(super) const DEFAULT_TIMESTAMP_START: &str = "1900-01-01 00:00:00"; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/sql_test_glue/plan_driver.rs b/crates/sqlbuild-python/src/bindings/_helpers/sql_test_glue/plan_driver.rs index 92028ee3d5..4652f20271 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/sql_test_glue/plan_driver.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/sql_test_glue/plan_driver.rs @@ -151,12 +151,8 @@ fn planned<'py>(args: &KeywordArguments<'py>, artifact: bool) -> PyResult( .filter(|(_, suffix)| !suffix.contains(SENTINEL)) else { return Err(PyErr::from_value( - imported( - py, - "sqlbuild.compiler.planner.exceptions", - "NativeSqlTestPlanningError", - )? - .call1(("adapter SQL function call template is not deterministic",))?, + imported(py, "sqlbuild._native", "NativeSqlTestPlanningError")? + .call1(("adapter SQL function call template is not deterministic",))?, )); }; fields.push(string(py, prefix)); diff --git a/crates/sqlbuild-python/src/bindings/_helpers/state_rows/constants.rs b/crates/sqlbuild-python/src/bindings/_helpers/state_rows/constants.rs index ba226c37d6..9535e6cc60 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/state_rows/constants.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/state_rows/constants.rs @@ -1,7 +1,7 @@ //! Python modules, names and messages of migration state rows. -pub(crate) const MIGRATION_MODELS: &str = "sqlbuild.compiler.migrations.models"; -pub(crate) const MIGRATION_TYPES: &str = "sqlbuild.compiler.migrations.types"; +pub(crate) const MIGRATION_MODELS: &str = "sqlbuild._native"; +pub(crate) const MIGRATION_TYPES: &str = "sqlbuild._native"; pub(crate) const MIGRATION_EXCEPTIONS: &str = "sqlbuild._native"; pub(crate) const STATE_TABLE_DDL: &str = "sqlbuild.adapter.state_sql.main.render_state_table_create_sql"; diff --git a/python-boundary.toml b/python-boundary.toml index 71d8cf95cf..a92193fd2d 100644 --- a/python-boundary.toml +++ b/python-boundary.toml @@ -2,7 +2,7 @@ # migration work that moves to Rust. `remaining_lines` must equal the current count of that work: # a PR that moves code lowers it, and adding movable Python fails the check. # Native failures must surface as errors; `handled_native_errors` lists the reviewed exceptions. -remaining_lines = 3851 +remaining_lines = 2319 [[stays]] path = "executor" @@ -803,7 +803,3 @@ note = "Shape rule: data constants only (INTEGRATION_RESULT_PATH_ENV), no logic; path = "compiler/compile/types.py" reason = "executor" note = "Shape rule: typing protocols, enums and aliases only (CompiledProjectView, CompiledModelView, CompiledResourceType, AuditInput, ...), no logic; consumed by executor/janitor/_helpers/plan.py, executor/pipeline/_helpers/testing.py, executor/scenario/_helpers/local/execution.py and project audit factories that annotate AuditInput." - -[[handled_native_errors]] -location = "compiler/planner/models.py::CursorBounds._parse_inferred" -note = "A bound that is not integer text is parsed as a timestamp instead; CursorAlgebraError is an input error, and native compiler failures are not caught." diff --git a/src/sqlbuild/_native.pyi b/src/sqlbuild/_native.pyi index 01f4c8b415..9b97418121 100644 --- a/src/sqlbuild/_native.pyi +++ b/src/sqlbuild/_native.pyi @@ -26,12 +26,14 @@ from sqlbuild.adapter.contract.models import ( ExpressionInferenceProfile, QueryResult, RelationInfo, + RetentionRequest, RowDiffSampling, RowDiffTolerances, ) from sqlbuild.adapter.contract.types import ( AdapterExecute, FrameworkType, + MigrationTransfer, TablePromotionMode, TypeDialect, ) @@ -104,64 +106,10 @@ from sqlbuild.compiler.discovery.types import LoaderConnectionMode from sqlbuild.compiler.lineage.types import ( InferredNullability, ) -from sqlbuild.compiler.migrations.models import ( - ColumnMigrationEvent, - MigrationEvent, - MigrationRelation, - OldNameViewEvent, - OldNameViewHistory, -) -from sqlbuild.compiler.migrations.types import ( - MigrationDecision, - MigrationPromotion, - OldNameViewEventType, -) from sqlbuild.compiler.pipeline.models import ( CompilePipelineResult, ProjectGraph, ) -from sqlbuild.compiler.planner.exceptions import PlannerInputError -from sqlbuild.compiler.planner.models import ( - AuditPlanEntry, - ChainStep, - CloneSourcePlanEntry, - ColumnMigrationPlanEntry, - CursorBounds, - CursorInputEvidence, - CursorInputRelation, - CursorOverrides, - EmptyCursorInputDecision, - FunctionPlanEntry, - FutureCursorSafetyEvidence, - MaximumStartPolicyInputs, - MaximumStartSafetyEvidence, - ModelCursorSnapshot, - ModelMigrationPlanEntry, - ModelPlanEntry, - OldNameView, - OldNameViewPlanEntry, - PlanOutput, - RetentionPlanEntry, - ScenarioExecutionPlan, - ScenarioFixturePlan, - ScenarioRelationMap, - ScenarioRelationPlan, - SchemaFinding, - SeedPlanEntry, - SourceLoadPlanEntry, - SqlTestPlanEntry, - TableTypePlanEntry, -) -from sqlbuild.compiler.planner.types import ( - BackfillAction, - CursorGrain, - CursorType, - MaterializationType, - OnSchemaChange, - PlanAction, - PlanReason, - ScenarioArtifactKind, -) from sqlbuild.compiler.python_nodes.types import PythonNodeKind from sqlbuild.compiler.references.models import ( ProjectRelation, @@ -15517,3 +15465,1321 @@ class Duration(NativeRecord): def total_months(self) -> int: ... def add_to(self, moment: datetime) -> datetime: ... def subtract_from(self, moment: datetime) -> datetime: ... + +class MigrationDecision(StrEnum): + """Per-run decision for one declared or discovered model migration.""" + + DONE = "done" + MIGRATE = "migrate" + REDO = "redo" + SUPERSEDED_REPLACE = "superseded_replace" + FORCED_REPLACE = "forced_replace" + CONFLICT = "conflict" + ORIGIN_MISSING = "origin_missing" + RENAMED = "renamed" + @property + def moves_data(self) -> bool: ... + @property + def blocks_build(self) -> bool: ... + @property + def checks_compatibility(self) -> bool: ... + @property + def label(self) -> str: ... + +class MigrationPromotion(StrEnum): + """How a completed migration stage replaces or creates the destination.""" + + SWAP = "swap" + RENAME = "rename" + TRANSACTIONAL_RENAME = "transactional_rename" + @property + def label(self) -> str: ... + +class ColumnMigrationDecision(StrEnum): + """Per-run decision for one declared or detected column rename.""" + + RENAME = "rename" + RECORD = "record" + DONE = "done" + CONFLICT = "conflict" + SOURCE_MISSING = "source_missing" + STILL_PRODUCED = "still_produced" + UNSUPPORTED = "unsupported" + @property + def renames(self) -> bool: ... + @property + def records_event(self) -> bool: ... + @property + def blocks_build(self) -> bool: ... + @property + def label(self) -> str: ... + +class OldNameViewEventType(StrEnum): + """Immutable facts about what happened at a migrated relation's old name.""" + + REQUIRED = "required" + ORIGIN_ARCHIVED = "origin_archived" + VIEW_CREATED = "view_created" + VIEW_DROPPED = "view_dropped" + +class OldNameViewDropReason(StrEnum): + """Why a compatibility view at an old name was dropped.""" + + EXPIRED = "expired" + EARLY = "early" + MISSING = "missing" + +class OldNameViewStatus(StrEnum): + """Projected state of the old-name steps of one recorded move.""" + + PENDING_ARCHIVE = "pending_archive" + PENDING_VIEW = "pending_view" + LIVE = "live" + EXPIRED = "expired" + DROPPED = "dropped" + +class MigrationRelation(NativeRecord): + """One physical relation named by a migration event.""" + + database: str | None + schema: str | None + name: str + def __init__( + self, + database: str | None, + schema: str | None, + name: str, + ) -> None: ... + @property + def identity(self) -> tuple[str | None, str, str]: ... + def matches(self, other: MigrationRelation) -> bool: ... + +class MigrationEvent(NativeRecord): + """One immutable fact that a origin relation was cloned over a destination.""" + + event_id: str + target_name: str | None + origin_model: str | None + origin: MigrationRelation + destination_model: str + destination: MigrationRelation + origin_version_hash: str + discovery: MigrationDiscovery + decision: MigrationDecision + run_id: str + created_at: datetime + def __init__( + self, + event_id: str, + target_name: str | None, + origin_model: str | None, + origin: MigrationRelation, + destination_model: str, + destination: MigrationRelation, + origin_version_hash: str, + discovery: MigrationDiscovery, + decision: MigrationDecision, + run_id: str, + created_at: datetime, + ) -> None: ... + def mentions(self, relation: MigrationRelation) -> bool: ... + +class ColumnMigrationEvent(NativeRecord): + """One immutable fact that a column of a model relation was renamed in place.""" + + event_id: str + target_name: str | None + model_name: str + relation: MigrationRelation + origin_column: str + destination_column: str + discovery: MigrationDiscovery + decision: ColumnMigrationDecision + run_id: str + created_at: datetime + def __init__( + self, + event_id: str, + target_name: str | None, + model_name: str, + relation: MigrationRelation, + origin_column: str, + destination_column: str, + discovery: MigrationDiscovery, + decision: ColumnMigrationDecision, + run_id: str, + created_at: datetime, + ) -> None: ... + def mentions(self, *, relation: MigrationRelation, columns: frozenset[str]) -> bool: ... + def renamed(self, *, origin_column: str, destination_column: str) -> bool: ... + +class OldNameViewEvent(NativeRecord): + """One immutable fact about the old name of one recorded move.""" + + event_id: str + target_name: str | None + event_type: OldNameViewEventType + migration_event_id: str + destination_model: str + old: MigrationRelation + new: MigrationRelation + run_id: str + created_at: datetime + view_retention: str | None + archive_name: str | None + column_aliases: tuple[tuple[str, str], ...] + expires_at: datetime | None + drop_reason: OldNameViewDropReason | None + grants_copied: tuple[str, ...] | None + view_sql: str | None + def __init__( + self, + event_id: str, + target_name: str | None, + event_type: OldNameViewEventType, + migration_event_id: str, + destination_model: str, + old: MigrationRelation, + new: MigrationRelation, + run_id: str, + created_at: datetime, + view_retention: str | None = ..., + archive_name: str | None = ..., + column_aliases: tuple[tuple[str, str], ...] = ..., + expires_at: datetime | None = ..., + drop_reason: OldNameViewDropReason | None = ..., + grants_copied: tuple[str, ...] | None = ..., + view_sql: str | None = ..., + ) -> None: ... + +class OldNameViewHistory(NativeRecord): + """The recorded move and every old-name fact that references it.""" + + move: MigrationEvent + required: OldNameViewEvent + archived: OldNameViewEvent | None + created: OldNameViewEvent | None + dropped: OldNameViewEvent | None + def __init__( + self, + move: MigrationEvent, + required: OldNameViewEvent, + archived: OldNameViewEvent | None = ..., + created: OldNameViewEvent | None = ..., + dropped: OldNameViewEvent | None = ..., + ) -> None: ... + def status(self, *, now: datetime) -> OldNameViewStatus: ... + @property + def old(self) -> MigrationRelation: ... + @property + def new(self) -> MigrationRelation: ... + +class BackfillAction(StrEnum): + FULL = "full" + BOUNDED = "bounded" + FORWARD_ONLY = "forward" + +class OnSchemaChange(StrEnum): + IGNORE = "ignore" + FAIL = "fail" + APPEND_NEW_COLUMNS = "append_new_columns" + SYNC_ALL_COLUMNS = "sync_all_columns" + +class ContractPolicy(StrEnum): + NONE = "none" + ENFORCED = "enforced" + +class RetentionDirection(StrEnum): + INCREASE = "increase" + DECREASE = "decrease" + MIXED = "mixed" + MATCH = "match" + APPLY_AFTER_CREATE = "apply_after_create" + +class RetentionPlanPhase(StrEnum): + PRE = "pre" + POST = "post" + AFTER_CREATE = "after_create" + NONE = "none" + +class PlanAction(StrEnum): + CREATE_VIEW = "create_view" + CREATE_TABLE = "create_table" + INCREMENTAL_APPEND = "incremental_append" + INCREMENTAL_DELETE_INSERT = "incremental_delete_insert" + INCREMENTAL_MERGE = "incremental_merge" + SNAPSHOT = "snapshot" + LOAD_SEED = "load_seed" + SKIP = "skip" + CUSTOM = "custom" + +class PlanReason(StrEnum): + FIRST_RUN = "first_run" + RENAMED = "renamed" + FULL_REFRESH = "full_refresh" + QUERY_CHANGED = "query_changed" + FUNCTION_CHANGED = "function_changed" + CONFIG_CHANGED = "config_changed" + SCHEMA_CHANGED = "schema_changed" + NORMAL_INCREMENTAL = "normal_incremental" + NO_CHANGE = "no_change" + DISABLED = "disabled" + SOURCE_FRESHNESS_ERROR = "source_freshness_error" + EXTERNAL_UPSTREAM_FAILED = "external_upstream_failed" + +class IncrementalStrategy(StrEnum): + APPEND = "append" + DELETE_INSERT = "delete_insert" + MERGE = "merge" + +class MicrobatchStrategy(StrEnum): + """How an ordinary microbatch range is selected.""" + + ROLLING_WINDOW = "rolling_window" + WATERMARK = "watermark" + +class IncrementalMode(StrEnum): + FULL = "full" + MICROBATCH = "microbatch" + +class SnapshotStrategy(StrEnum): + TIMESTAMP = "timestamp" + CHECK = "check" + +class HistoricalInput(StrEnum): + SNAPSHOT = "snapshot" + CHANGES = "changes" + +class InitialValidFrom(StrEnum): + UPDATED_AT = "updated_at" + OBSERVED_AT = "observed_at" + EXECUTION_TIME = "execution_time" + +class SnapshotSchemaChangePolicy(StrEnum): + DENY = "deny" + REQUIRE_CONFIRMATION = "require_confirmation" + APPEND_NEW_COLUMNS = "append_new_columns" + +class CursorType(StrEnum): + TIMESTAMP = "timestamp" + INTEGER = "integer" + +class CursorGrain(StrEnum): + SECOND = "second" + MINUTE = "minute" + HOUR = "hour" + DAY = "day" + MONTH = "month" + YEAR = "year" + +class MaterializationType(StrEnum): + VIEW = "view" + TABLE = "table" + INCREMENTAL = "incremental" + SNAPSHOT = "snapshot" + SEED = "seed" + CUSTOM = "custom" + @classmethod + def is_table_backed(cls, *, materialized: object | None) -> bool: ... + +class ScenarioArtifactKind(StrEnum): + SOURCE = "source" + REF = "ref" + SEED = "seed" + DBT_REF = "dbt_ref" + MODEL = "model" + +class PlannerInputError(ValueError): + """Raised when planner inputs cannot be resolved safely.""" + + code: str + message: str + help: str | None + def __init__( + self, message: str, *, code: str | None = None, help: str | None = None + ) -> None: ... + +class NativeSqlTestPlanningError(PlannerInputError): + """Raised when the native SQL-test planning boundary returns invalid data.""" + +class FutureCursorSafetyError(PlannerInputError): + """Raised when an effective cursor exceeds the configured future limit.""" + +class MaximumAutomaticStartError(PlannerInputError): + """Raised when automatic cursor recovery would exceed its configured horizon.""" + + evidence: Any + +class CursorOverrides(NativeRecord): + """Cursor override values from CLI flags; untyped `start`/`end` await planner typing.""" + + start_ts: CursorScalar | None + end_ts: CursorScalar | None + start_int: CursorScalar | None + end_int: CursorScalar | None + start: str | None + end: str | None + def __init__( + self, + *, + start_ts: str | CursorScalar | None = None, + end_ts: str | CursorScalar | None = None, + start_int: str | CursorScalar | None = None, + end_int: str | CursorScalar | None = None, + start: str | None = None, + end: str | None = None, + ) -> None: ... + @property + def has_untyped(self) -> bool: ... + def typed_as(self, *, cursor_type: CursorType) -> CursorOverrides: ... + +class EmptyCursorInputDecision(NativeRecord): + """Cursor inputs whose lack of rows leaves no window, and whether other inputs have rows.""" + + input_names: tuple[str, ...] + waiting_on_empty_inputs: bool + def __init__( + self, + input_names: tuple[str, ...] = ..., + waiting_on_empty_inputs: bool = ..., + ) -> None: ... + +class ModelCursorSnapshot(NativeRecord): + """Cursor MIN/MAX values gathered from warehouse for one incremental model.""" + + target_max: CursorScalar | None + upstream_mins: tuple[CursorScalar, ...] + upstream_maxes: tuple[CursorScalar, ...] + physical_target_max: CursorScalar | None + target_eligible_max: CursorScalar | None + target_relation: str | None + destination_cursor_column: str | None + input_evidence: tuple[CursorInputEvidence, ...] + expected_watermark_count: int + unavailable_watermark_tags: tuple[str, ...] + cursor_watermark_mode: CursorWatermarkMode + upstream_terminal_starts: tuple[CursorScalar, ...] + upstream_terminal_ends: tuple[CursorScalar, ...] + upstream_end_inputs: tuple[tuple[CursorScalar | None, CursorScalar | None], ...] + upstream_availability_ends: tuple[CursorScalar, ...] + empty_input_names: tuple[str, ...] + unreadable_input_names: tuple[str, ...] + def __init__(self, **values: Any) -> None: ... + @property + def watermarks_available(self) -> bool: ... + +class CursorBounds(NativeRecord): + """Effective cursor start and end values for one incremental model.""" + + start: Bound + end: Bound + future_safety: FutureCursorSafetyEvidence | None + maximum_start_safety: MaximumStartSafetyEvidence | None + def __init__( + self, + *, + start: Bound | str, + end: Bound | str, + future_safety: FutureCursorSafetyEvidence | None = None, + maximum_start_safety: MaximumStartSafetyEvidence | None = None, + ) -> None: ... + +class CursorInputEvidence(NativeRecord): + """Observed bounds for one physical cursor input.""" + + relation: str + cursor_column: str + minimum: CursorScalar | None + maximum: CursorScalar + def __init__( + self, + *, + relation: str, + cursor_column: str, + minimum: CursorScalar | str | None, + maximum: CursorScalar | str, + ) -> None: ... + +class FutureCursorSafetyEvidence(NativeRecord): + """Structured evidence for one applied future-cursor cap.""" + + action: FutureCursorAction + max_distance: str + invocation_time: CursorScalar + discovered_start: CursorScalar + discovered_end: CursorScalar + applied_start: CursorScalar + applied_end: CursorScalar + maximum_allowed_start: CursorScalar + maximum_allowed_end: CursorScalar + future_start_detected: bool + future_end_detected: bool + determining_relation: str | None + determining_cursor_column: str | None + inputs: tuple[CursorInputEvidence, ...] + def __init__(self, **values: Any) -> None: ... + +class MaximumStartSafetyEvidence(NativeRecord): + """Structured evidence for an automatic-start eligibility decision.""" + + action: FutureCursorAction + max_ahead: str + invocation_time: CursorScalar + physical_target_max: CursorScalar + highest_eligible_target_max: CursorScalar | None + effective_start: CursorScalar + maximum_allowed_start: CursorScalar + target_relation: str + cursor_column: str + def __init__( + self, + *, + action: FutureCursorAction, + max_ahead: str, + invocation_time: CursorScalar | str, + physical_target_max: CursorScalar | str, + highest_eligible_target_max: CursorScalar | str | None, + effective_start: CursorScalar | str, + maximum_allowed_start: CursorScalar | str, + target_relation: str, + cursor_column: str, + ) -> None: ... + +class CursorInputRelation(NativeRecord): + """One cursor-bearing input relation for runtime range discovery.""" + + relation: str + cursor_column: str + cursor_grain: CursorGrain | None + is_model_backed: bool + is_runtime_produced: bool + terminal_cursor_start: CursorScalar | None + terminal_cursor_end: CursorScalar | None + input_name: str | None + input_kind: GraphResourceKind | None + def __init__( + self, + *, + relation: str, + cursor_column: str, + cursor_grain: CursorGrain | str | None = None, + is_model_backed: bool = False, + is_runtime_produced: bool = False, + terminal_cursor_start: CursorScalar | str | None = None, + terminal_cursor_end: CursorScalar | str | None = None, + input_name: str | None = None, + input_kind: GraphResourceKind | None = None, + ) -> None: ... + @property + def is_runtime_owned(self) -> bool: ... + +class MaximumStartPolicyInputs(NativeRecord): + """Effective policy and materialization safety inputs for automatic starts.""" + + config: StartCursorsConfig | None + invocation_time: datetime | None + incremental_strategy: str | None + incremental_mode: str | None + def __init__( + self, + config: StartCursorsConfig | None = ..., + invocation_time: datetime | None = ..., + incremental_strategy: str | None = ..., + incremental_mode: str | None = ..., + ) -> None: ... + +class SchemaFinding(NativeRecord): + """One detected schema difference between expected and warehouse columns.""" + + kind: SchemaChangeKind + column_name: str + source: SchemaColumnSource + expected_type: str | None + actual_type: str | None + def __init__( + self, + kind: SchemaChangeKind, + column_name: str, + source: SchemaColumnSource, + expected_type: str | None = ..., + actual_type: str | None = ..., + ) -> None: ... + +class RetentionPlanEntry(NativeRecord): + """Warehouse retention work independent of model identity actions.""" + + request: RetentionRequest + model_names: tuple[str, ...] + actual_days: int | None + effective_days: int | None + source: str + direction: RetentionDirection + phase: RetentionPlanPhase + statements: tuple[str, ...] + irreversible_warning: str | None + decrease_policy: str + def __init__( + self, + request: RetentionRequest, + model_names: tuple[str, ...], + actual_days: int | None, + effective_days: int | None, + source: str, + direction: RetentionDirection, + phase: RetentionPlanPhase, + statements: tuple[str, ...] = ..., + irreversible_warning: str | None = ..., + decrease_policy: str = ..., + ) -> None: ... + @property + def decreases(self) -> bool: ... + +class TableTypePlanEntry(NativeRecord): + """Snowflake table-type work independent of model identity actions.""" + + model_name: str + destination: CompiledRelationLocation + copy_name: str + desired_type: str + actual_type: str | None + source: str + downgrade: bool + downgrade_policy: str + irreversible_warning: str | None + def __init__( + self, + model_name: str, + destination: CompiledRelationLocation, + copy_name: str, + desired_type: str, + actual_type: str | None, + source: str, + downgrade: bool, + downgrade_policy: str, + irreversible_warning: str | None = ..., + ) -> None: ... + +class ModelMigrationPlanEntry(NativeRecord): + """One declared or discovered model migration and its per-run decision.""" + + model_name: str + discovery: MigrationDiscovery + decision: MigrationDecision + compatibility: MigrationCompatibility + origin_model: str | None + origin: CompiledRelationLocation + destination: CompiledRelationLocation + target_name: str | None + origin_version_hash: str + origin_is_transient: bool + stage_is_transient: bool | None + transfer: MigrationTransfer | None + transfer_fallback: MigrationTransfer | None + promotion: MigrationPromotion | None + completed_at: datetime | None + compatibility_findings: tuple[str, ...] + message: str | None + origin_tracked: bool + missing_origin_policy: MissingMigrationOriginPolicy + def __init__( + self, + model_name: str, + discovery: MigrationDiscovery, + decision: MigrationDecision, + compatibility: MigrationCompatibility, + origin_model: str | None, + origin: CompiledRelationLocation, + destination: CompiledRelationLocation, + target_name: str | None, + origin_version_hash: str = ..., + origin_is_transient: bool = ..., + stage_is_transient: bool | None = ..., + transfer: MigrationTransfer | None = ..., + transfer_fallback: MigrationTransfer | None = ..., + promotion: MigrationPromotion | None = ..., + completed_at: datetime | None = ..., + compatibility_findings: tuple[str, ...] = ..., + message: str | None = ..., + origin_tracked: bool = ..., + missing_origin_policy: MissingMigrationOriginPolicy = ..., + ) -> None: ... + @property + def origin_missing(self) -> bool: ... + @property + def origin_hidden(self) -> bool: ... + @property + def storage_transition(self) -> str | None: ... + @property + def target_label(self) -> str: ... + @property + def blocks_build(self) -> bool: ... + +class OldNameViewPlanEntry(NativeRecord): + """What one build does at the old name of one migrated model.""" + + model_name: str + origin: CompiledRelationLocation + destination: CompiledRelationLocation + action: OldNameViewAction + target_name: str | None + retention: str | None + expires_at: datetime | None + column_aliases: tuple[tuple[str, str], ...] + stores_history: bool + migration_event_id: str | None + records_requirement: bool + reason: str | None + grants_copied: int | None + grants_supported: bool + archived: bool + def __init__( + self, + model_name: str, + origin: CompiledRelationLocation, + destination: CompiledRelationLocation, + action: OldNameViewAction, + target_name: str | None, + retention: str | None = ..., + expires_at: datetime | None = ..., + column_aliases: tuple[tuple[str, str], ...] = ..., + stores_history: bool = ..., + migration_event_id: str | None = ..., + records_requirement: bool = ..., + reason: str | None = ..., + grants_copied: int | None = ..., + grants_supported: bool = ..., + archived: bool = ..., + ) -> None: ... + @property + def runs_steps(self) -> bool: ... + +class OldNameView(NativeRecord): + """One compatibility view recorded at an old name and not recorded as dropped.""" + + destination_model: str + old: CompiledRelationLocation + new: CompiledRelationLocation + expires_at: datetime | None + column_aliases: tuple[tuple[str, str], ...] + migration_event_id: str + target_name: str | None + name_reused_by: str | None + def __init__( + self, + destination_model: str, + old: CompiledRelationLocation, + new: CompiledRelationLocation, + expires_at: datetime | None, + column_aliases: tuple[tuple[str, str], ...], + migration_event_id: str, + target_name: str | None, + name_reused_by: str | None = ..., + ) -> None: ... + +class ColumnMigrationPlanEntry(NativeRecord): + """One declared or detected in-place column rename and its per-run decision.""" + + model_name: str + destination: CompiledRelationLocation + origin_column: str + destination_column: str + discovery: MigrationDiscovery + decision: ColumnMigrationDecision + target_name: str | None + completed_at: datetime | None + message: str | None + missing_origin_policy: MissingMigrationOriginPolicy + def __init__( + self, + model_name: str, + destination: CompiledRelationLocation, + origin_column: str, + destination_column: str, + discovery: MigrationDiscovery, + decision: ColumnMigrationDecision, + target_name: str | None, + completed_at: datetime | None = ..., + message: str | None = ..., + missing_origin_policy: MissingMigrationOriginPolicy = ..., + ) -> None: ... + @property + def origin_missing(self) -> bool: ... + @property + def blocks_build(self) -> bool: ... + +class ModelPlanEntry(NativeRecord): + """Per-model execution plan entry with action, reason, and resolved artifacts.""" + + key: CompiledObjectKey + name: str + relative_path: Path + materialization_type: MaterializationType + action: PlanAction + reason: PlanReason + destination: CompiledRelationLocation + fingerprint_query_sql: str + resolved_sql: str + logical_ddl: str + migration_fingerprint: str | None + incremental_strategy: str | None + incremental_mode: str | None + microbatch_strategy: str | None + cursor_watermark_mode: str | None + cursor_column: str | None + cursor_type: str | None + cursor_grain: str | None + cursor_start: str | None + cursor_end: str | None + lookback: str | None + cursor_bounds: CursorBounds | None + empty_cursor_inputs: tuple[str, ...] + waiting_on_empty_inputs: bool + cursor_input_relations: tuple[CursorInputRelation, ...] + batch_size: str | None + batch_concurrency: int + unaccounted_partition_policy: str | None + microbatch_range: CursorBounds | None + microbatch_limit: int | None + declared_microbatch_limit_action: MicrobatchLimitAction | None + microbatch_safety_limit: int | None + microbatch_safety_limit_action: MicrobatchLimitAction | None + microbatch_limit_count: int | None + microbatch_limit_action: MicrobatchLimitAction | None + microbatch_limit_warning: str | None + start_cursor_override: str | None + end_cursor_override: str | None + future_cursor_config: FutureCursorsConfig | None + start_cursor_config: StartCursorsConfig | None + invocation_time: datetime | None + unique_key: tuple[str, ...] + merge_exclude_columns: tuple[str, ...] + snapshot_strategy: str | None + updated_at_column: str | None + check_columns: tuple[str, ...] + observed_at_column: str | None + historical_input: str | None + valid_from_column: str | None + valid_to_column: str | None + initial_valid_from: str | None + invalidate_hard_deletes: bool + snapshot_full_refresh: str | None + snapshot_schema_change: str | None + table_type: str + on_schema_change: OnSchemaChange | None + type_enforcement: bool + declared_columns: tuple[ColumnInfo, ...] + contract_enforced: bool + contract_columns: tuple[ColumnInfo, ...] + contract_dynamic_columns: tuple[SchemaDynamicColumnFamily, ...] + pre_hooks: object + post_hooks: object + previous_query_sql: str | None + fingerprint_metadata_json: str | None + previous_metadata_json: str | None + fingerprint_version_hash: str | None + previous_version_hash: str | None + query_changed: bool + config_changed: bool + schema_actions: tuple[SchemaAction, ...] + schema_findings: tuple[SchemaFinding, ...] + backfill: BackfillResult + changed_functions: tuple[str, ...] + custom_materialization_name: str | None + custom_config: dict[str, object] + custom_placeholders: dict[str, str] + column_rename_hints: tuple[ColumnRenameHint, ...] + def __init__( + self, + key: CompiledObjectKey, + name: str, + relative_path: Path, + materialization_type: MaterializationType, + action: PlanAction, + reason: PlanReason, + destination: CompiledRelationLocation, + fingerprint_query_sql: str, + resolved_sql: str, + logical_ddl: str, + migration_fingerprint: str | None = ..., + incremental_strategy: str | None = ..., + incremental_mode: str | None = ..., + microbatch_strategy: str | None = ..., + cursor_watermark_mode: str | None = ..., + cursor_column: str | None = ..., + cursor_type: str | None = ..., + cursor_grain: str | None = ..., + cursor_start: str | None = ..., + cursor_end: str | None = ..., + lookback: str | None = ..., + cursor_bounds: CursorBounds | None = ..., + empty_cursor_inputs: tuple[str, ...] = ..., + waiting_on_empty_inputs: bool = ..., + cursor_input_relations: tuple[CursorInputRelation, ...] = ..., + batch_size: str | None = ..., + batch_concurrency: int = ..., + unaccounted_partition_policy: str | None = ..., + microbatch_range: CursorBounds | None = ..., + microbatch_limit: int | None = ..., + declared_microbatch_limit_action: MicrobatchLimitAction | None = ..., + microbatch_safety_limit: int | None = ..., + microbatch_safety_limit_action: MicrobatchLimitAction | None = ..., + microbatch_limit_count: int | None = ..., + microbatch_limit_action: MicrobatchLimitAction | None = ..., + microbatch_limit_warning: str | None = ..., + start_cursor_override: str | None = ..., + end_cursor_override: str | None = ..., + future_cursor_config: FutureCursorsConfig | None = ..., + start_cursor_config: StartCursorsConfig | None = ..., + invocation_time: datetime | None = ..., + unique_key: tuple[str, ...] = ..., + merge_exclude_columns: tuple[str, ...] = ..., + snapshot_strategy: str | None = ..., + updated_at_column: str | None = ..., + check_columns: tuple[str, ...] = ..., + observed_at_column: str | None = ..., + historical_input: str | None = ..., + valid_from_column: str | None = ..., + valid_to_column: str | None = ..., + initial_valid_from: str | None = ..., + invalidate_hard_deletes: bool = ..., + snapshot_full_refresh: str | None = ..., + snapshot_schema_change: str | None = ..., + table_type: str = ..., + on_schema_change: OnSchemaChange | None = ..., + type_enforcement: bool = ..., + declared_columns: tuple[ColumnInfo, ...] = ..., + contract_enforced: bool = ..., + contract_columns: tuple[ColumnInfo, ...] = ..., + contract_dynamic_columns: tuple[SchemaDynamicColumnFamily, ...] = ..., + pre_hooks: object = ..., + post_hooks: object = ..., + previous_query_sql: str | None = ..., + fingerprint_metadata_json: str | None = ..., + previous_metadata_json: str | None = ..., + fingerprint_version_hash: str | None = ..., + previous_version_hash: str | None = ..., + query_changed: bool = ..., + config_changed: bool = ..., + schema_actions: tuple[SchemaAction, ...] = ..., + schema_findings: tuple[SchemaFinding, ...] = ..., + backfill: BackfillResult = ..., + changed_functions: tuple[str, ...] = ..., + custom_materialization_name: str | None = ..., + custom_config: dict[str, object] = ..., + custom_placeholders: dict[str, str] = ..., + column_rename_hints: tuple[ColumnRenameHint, ...] = ..., + ) -> None: ... + +class CloneSourcePlanEntry(NativeRecord): + """One managed source relation selected for target cloning.""" + + key: CompiledObjectKey + name: str + destination: CompiledRelationLocation + def __init__( + self, + key: CompiledObjectKey, + name: str, + destination: CompiledRelationLocation, + ) -> None: ... + +class SeedPlanEntry(NativeRecord): + """Per-seed execution plan entry.""" + + key: CompiledObjectKey + name: str + destination: CompiledRelationLocation + file_path: Path + columns: tuple[ColumnInfo, ...] + csv_settings: SeedCsvSettings + fingerprint_definition: str + fingerprint_version_hash: str + fingerprint_metadata_json: str + action: PlanAction + reason: PlanReason + def __init__( + self, + key: CompiledObjectKey, + name: str, + destination: CompiledRelationLocation, + file_path: Path, + columns: tuple[ColumnInfo, ...], + csv_settings: SeedCsvSettings, + fingerprint_definition: str = ..., + fingerprint_version_hash: str = ..., + fingerprint_metadata_json: str = ..., + action: PlanAction = ..., + reason: PlanReason = ..., + ) -> None: ... + +class SourceLoadPlanEntry(NativeRecord): + """Per-managed-source loader execution plan entry.""" + + key: CompiledObjectKey + name: str + loader: str + destination: str + resource_kind: ExecutionResourceKind + write_strategy: SourceWriteStrategy | None + cursor_column: str | None + unique_key: tuple[str, ...] + is_reload: bool + integration_kind: str | None + def __init__( + self, + key: CompiledObjectKey, + name: str, + loader: str, + destination: str, + resource_kind: ExecutionResourceKind = ..., + write_strategy: SourceWriteStrategy | None = ..., + cursor_column: str | None = ..., + unique_key: tuple[str, ...] = ..., + is_reload: bool = ..., + integration_kind: str | None = ..., + ) -> None: ... + +class FunctionPlanEntry(NativeRecord): + """Per-SQL-function execution plan entry.""" + + key: CompiledObjectKey + name: str + relative_path: Path + destination: CompiledRelationLocation + arguments: tuple[object, ...] + returns: str + body_sql: str + fingerprint_query_sql: str + fingerprint_destination: CompiledRelationLocation + return_columns: tuple[FunctionReturnColumn, ...] + language: FunctionLanguage + source_file_path: Path | None + runtime_version: str | None + entry_point: str | None + packages: tuple[str, ...] + previous_query_sql: str | None + reason: PlanReason + def __init__( + self, + key: CompiledObjectKey, + name: str, + relative_path: Path, + destination: CompiledRelationLocation, + arguments: tuple[object, ...], + returns: str, + body_sql: str, + fingerprint_query_sql: str, + fingerprint_destination: CompiledRelationLocation, + return_columns: tuple[FunctionReturnColumn, ...] = ..., + language: FunctionLanguage = ..., + source_file_path: Path | None = ..., + runtime_version: str | None = ..., + entry_point: str | None = ..., + packages: tuple[str, ...] = ..., + previous_query_sql: str | None = ..., + reason: PlanReason = ..., + ) -> None: ... + +class AuditPlanEntry(NativeRecord): + """Per-audit execution plan entry with resolved SQL and scheduling metadata.""" + + key: CompiledObjectKey + name: str + definition_name: str + resolved_sql: str + unresolved_sql: str + attachment_kind: AuditAttachmentKind + severity: AuditSeverity + requested_run_scope: AuditRunScope + effective_run_scope: AuditRunScope + evaluation_mode: AuditEvaluationMode + value_column: str | None + sample_count_column: str | None + sample_unit: str | None + thresholds: MeasurementThresholds | None + minimum_samples: int | None + evidence_resolved_sql: str | None + evidence_unresolved_sql: str | None + evidence_limit: int | None + scope_deps: tuple[CompiledObjectKey, ...] + attached_target_kind: AttachedAuditTargetKind | None + attached_target_name: str | None + attached_column_name: str | None + always_run: bool + description: str | None + reads_outside_target_lineage: bool + def __init__( + self, + key: CompiledObjectKey, + name: str, + definition_name: str, + resolved_sql: str, + unresolved_sql: str, + attachment_kind: AuditAttachmentKind, + severity: AuditSeverity, + requested_run_scope: AuditRunScope, + effective_run_scope: AuditRunScope, + evaluation_mode: AuditEvaluationMode = ..., + value_column: str | None = ..., + sample_count_column: str | None = ..., + sample_unit: str | None = ..., + thresholds: MeasurementThresholds | None = ..., + minimum_samples: int | None = ..., + evidence_resolved_sql: str | None = ..., + evidence_unresolved_sql: str | None = ..., + evidence_limit: int | None = ..., + scope_deps: tuple[CompiledObjectKey, ...] = ..., + attached_target_kind: AttachedAuditTargetKind | None = ..., + attached_target_name: str | None = ..., + attached_column_name: str | None = ..., + always_run: bool = ..., + description: str | None = ..., + reads_outside_target_lineage: bool = ..., + ) -> None: ... + +class ChainStep(NativeRecord): + """One step in a chained test execution sequence.""" + + model_name: str + resolved_sql: str + expected_cte_sql: str | None + lifted_ctes: tuple[tuple[str, str], ...] + comparison_body_sql: str | None + expected_columns: tuple[str, ...] | None + expected_lifted_ctes: tuple[tuple[str, str], ...] + def __init__( + self, + model_name: str, + resolved_sql: str, + expected_cte_sql: str | None = ..., + lifted_ctes: tuple[tuple[str, str], ...] = ..., + comparison_body_sql: str | None = ..., + expected_columns: tuple[str, ...] | None = ..., + expected_lifted_ctes: tuple[tuple[str, str], ...] = ..., + ) -> None: ... + +class SqlTestPlanEntry(NativeRecord): + """Per-test execution plan entry with chained resolution.""" + + key: CompiledObjectKey + name: str + source_path: Path | None + block_index: int | None + parent_name: str | None + case_name: str | None + case_index: int | None + case_fingerprint: str | None + parameter_schema: tuple[SqlTestParameterDeclaration, ...] + parameter_values: tuple[tuple[str, SqlValue], ...] + mock_ref_names: tuple[str, ...] + mock_source_names: tuple[str, ...] + mock_seed_names: tuple[str, ...] + mock_dbt_ref_names: tuple[str, ...] + mock_table_function_names: tuple[str, ...] + chain: tuple[ChainStep, ...] + assertions: tuple[SqlTestAssertionStep, ...] + scope_deps: tuple[CompiledObjectKey, ...] + function_deps: tuple[CompiledObjectKey, ...] + sql_analysis_enabled: bool + def __init__( + self, + key: CompiledObjectKey, + name: str, + source_path: Path | None = ..., + block_index: int | None = ..., + parent_name: str | None = ..., + case_name: str | None = ..., + case_index: int | None = ..., + case_fingerprint: str | None = ..., + parameter_schema: tuple[SqlTestParameterDeclaration, ...] = ..., + parameter_values: tuple[tuple[str, SqlValue], ...] = ..., + mock_ref_names: tuple[str, ...] = ..., + mock_source_names: tuple[str, ...] = ..., + mock_seed_names: tuple[str, ...] = ..., + mock_dbt_ref_names: tuple[str, ...] = ..., + mock_table_function_names: tuple[str, ...] = ..., + chain: tuple[ChainStep, ...] = ..., + assertions: tuple[SqlTestAssertionStep, ...] = ..., + scope_deps: tuple[CompiledObjectKey, ...] = ..., + function_deps: tuple[CompiledObjectKey, ...] = ..., + sql_analysis_enabled: bool = ..., + ) -> None: ... + +class ScenarioRelationMap(NativeRecord): + """Resolved scenario hash prefix and scenario-owned relation names.""" + + scenario_name: str + hash_prefix: str + artifacts: tuple[ScenarioArtifactName, ...] + def __init__( + self, + scenario_name: str, + hash_prefix: str, + artifacts: tuple[ScenarioArtifactName, ...] = ..., + ) -> None: ... + +class ScenarioRelationPlan(NativeRecord): + """Resolved scenario relation locations for model/source/seed ref resolution.""" + + scenario_name: str + relation_map: ScenarioRelationMap + model_locations: dict[str, CompiledRelationLocation] + seed_locations: dict[str, CompiledRelationLocation] + project_source_map: dict[str, SourceEntry] + source_map: dict[str, SourceEntry] + source_fixture_locations: dict[str, CompiledRelationLocation] + ref_fixture_locations: dict[str, CompiledRelationLocation] + seed_fixture_locations: dict[str, CompiledRelationLocation] + dbt_ref_fixture_locations: dict[str, CompiledRelationLocation] + def __init__( + self, + scenario_name: str, + relation_map: ScenarioRelationMap, + model_locations: dict[str, CompiledRelationLocation] = ..., + seed_locations: dict[str, CompiledRelationLocation] = ..., + project_source_map: dict[str, SourceEntry] = ..., + source_map: dict[str, SourceEntry] = ..., + source_fixture_locations: dict[str, CompiledRelationLocation] = ..., + ref_fixture_locations: dict[str, CompiledRelationLocation] = ..., + seed_fixture_locations: dict[str, CompiledRelationLocation] = ..., + dbt_ref_fixture_locations: dict[str, CompiledRelationLocation] = ..., + ) -> None: ... + +class ScenarioFixturePlan(NativeRecord): + """Self-contained fixture SQL planned for scenario materialization.""" + + kind: ScenarioArtifactKind + logical_name: str + destination: CompiledRelationLocation + sql: str + def __init__( + self, + kind: ScenarioArtifactKind, + logical_name: str, + destination: CompiledRelationLocation, + sql: str, + ) -> None: ... + +class ScenarioExpectedExpectationPlan(NativeRecord): + """Expected-output comparison inputs for one scenario target model.""" + + model_name: str + actual_destination: CompiledRelationLocation + expected_sql: str + def __init__( + self, + model_name: str, + actual_destination: CompiledRelationLocation, + expected_sql: str, + ) -> None: ... + +class ScenarioAssertionExpectationPlan(NativeRecord): + """Zero-row assertion SQL for one scenario assertion CTE.""" + + name: str + sql: str + def __init__( + self, + name: str, + sql: str, + ) -> None: ... + +class ScenarioExecutionPlan(NativeRecord): + """Dry-run execution plan for a SQL scenario graph slice.""" + + key: CompiledObjectKey + name: str + graph_plan: ScenarioGraphPlan + relation_plan: ScenarioRelationPlan + fixture_plans: tuple[ScenarioFixturePlan, ...] + seed_entries: tuple[SeedPlanEntry, ...] + function_entries: tuple[FunctionPlanEntry, ...] + model_entries: tuple[ModelPlanEntry, ...] + hook_functions: tuple[DiscoveredHookFunction, ...] + expected_expectations: tuple[ScenarioExpectedExpectationPlan, ...] + assertion_expectations: tuple[ScenarioAssertionExpectationPlan, ...] + def __init__( + self, + key: CompiledObjectKey, + name: str, + graph_plan: ScenarioGraphPlan, + relation_plan: ScenarioRelationPlan, + fixture_plans: tuple[ScenarioFixturePlan, ...] = ..., + seed_entries: tuple[SeedPlanEntry, ...] = ..., + function_entries: tuple[FunctionPlanEntry, ...] = ..., + model_entries: tuple[ModelPlanEntry, ...] = ..., + hook_functions: tuple[DiscoveredHookFunction, ...] = ..., + expected_expectations: tuple[ScenarioExpectedExpectationPlan, ...] = ..., + assertion_expectations: tuple[ScenarioAssertionExpectationPlan, ...] = ..., + ) -> None: ... + +class PlanOutput(NativeRecord): + """Complete execution plan produced by the planner.""" + + execution_order: tuple[CompiledObjectKey, ...] + model_entries: tuple[ModelPlanEntry, ...] + seed_entries: tuple[SeedPlanEntry, ...] + source_load_entries: tuple[SourceLoadPlanEntry, ...] + function_entries: tuple[FunctionPlanEntry, ...] + audit_entries: tuple[AuditPlanEntry, ...] + test_entries: tuple[SqlTestPlanEntry, ...] + available_test_case_names: tuple[str, ...] + selected_keys: frozenset[CompiledObjectKey] + warnings: tuple[PlanWarning, ...] + retention_entries: tuple[RetentionPlanEntry, ...] + table_type_entries: tuple[TableTypePlanEntry, ...] + migration_entries: tuple[ModelMigrationPlanEntry, ...] + column_migration_entries: tuple[ColumnMigrationPlanEntry, ...] + old_name_view_entries: tuple[OldNameViewPlanEntry, ...] + old_name_views: tuple[OldNameView, ...] + upstream_deps: dict[CompiledObjectKey, tuple[CompiledObjectKey, ...]] + downstream_deps: dict[CompiledObjectKey, tuple[CompiledObjectKey, ...]] + model_locations: dict[str, CompiledRelationLocation] + seed_locations: dict[str, CompiledRelationLocation] + function_locations: dict[str, CompiledRelationLocation] + source_map: dict[str, SourceEntry] + source_read_map: dict[str, SourceEntry] + python_source_read_map: dict[str, SourceEntry] + hook_functions: tuple[DiscoveredHookFunction, ...] + enforce_explicit_references: bool + provider_usages: tuple[PlanProviderUsage, ...] + source_freshness: DirectSourceFreshnessPlanningResult | None + python_identity_fingerprints: dict[tuple[str, str], Fingerprint] + metadata: dict[str, object] + cursor_overrides: CursorOverrides | None + def __init__( + self, + execution_order: tuple[CompiledObjectKey, ...] = ..., + model_entries: tuple[ModelPlanEntry, ...] = ..., + seed_entries: tuple[SeedPlanEntry, ...] = ..., + source_load_entries: tuple[SourceLoadPlanEntry, ...] = ..., + function_entries: tuple[FunctionPlanEntry, ...] = ..., + audit_entries: tuple[AuditPlanEntry, ...] = ..., + test_entries: tuple[SqlTestPlanEntry, ...] = ..., + available_test_case_names: tuple[str, ...] = ..., + selected_keys: frozenset[CompiledObjectKey] = ..., + warnings: tuple[PlanWarning, ...] = ..., + retention_entries: tuple[RetentionPlanEntry, ...] = ..., + table_type_entries: tuple[TableTypePlanEntry, ...] = ..., + migration_entries: tuple[ModelMigrationPlanEntry, ...] = ..., + column_migration_entries: tuple[ColumnMigrationPlanEntry, ...] = ..., + old_name_view_entries: tuple[OldNameViewPlanEntry, ...] = ..., + old_name_views: tuple[OldNameView, ...] = ..., + upstream_deps: dict[CompiledObjectKey, tuple[CompiledObjectKey, ...]] = ..., + downstream_deps: dict[CompiledObjectKey, tuple[CompiledObjectKey, ...]] = ..., + model_locations: dict[str, CompiledRelationLocation] = ..., + seed_locations: dict[str, CompiledRelationLocation] = ..., + function_locations: dict[str, CompiledRelationLocation] = ..., + source_map: dict[str, SourceEntry] = ..., + source_read_map: dict[str, SourceEntry] = ..., + python_source_read_map: dict[str, SourceEntry] = ..., + hook_functions: tuple[DiscoveredHookFunction, ...] = ..., + enforce_explicit_references: bool = ..., + provider_usages: tuple[PlanProviderUsage, ...] = ..., + source_freshness: DirectSourceFreshnessPlanningResult | None = ..., + python_identity_fingerprints: dict[tuple[str, str], Fingerprint] = ..., + metadata: dict[str, object] = ..., + cursor_overrides: CursorOverrides | None = ..., + ) -> None: ... + @property + def python_source_entries(self) -> dict[str, SourceEntry]: ... diff --git a/src/sqlbuild/adapter/contract/classes/snapshot_sql.py b/src/sqlbuild/adapter/contract/classes/snapshot_sql.py index 7187fe778e..a36b24e9c3 100644 --- a/src/sqlbuild/adapter/contract/classes/snapshot_sql.py +++ b/src/sqlbuild/adapter/contract/classes/snapshot_sql.py @@ -2,9 +2,9 @@ from __future__ import annotations +from sqlbuild._native import InitialValidFrom, SnapshotStrategy from sqlbuild.adapter.contract.models import SnapshotChangeTarget, SnapshotSqlDialect from sqlbuild.adapter.contract.types import SnapshotUpdateStyle -from sqlbuild.compiler.planner.types import InitialValidFrom, SnapshotStrategy class SnapshotSql: diff --git a/src/sqlbuild/cli/commands/_helpers/freshness/source_freshness.py b/src/sqlbuild/cli/commands/_helpers/freshness/source_freshness.py index 0db7e631c1..2cba6c4c93 100644 --- a/src/sqlbuild/cli/commands/_helpers/freshness/source_freshness.py +++ b/src/sqlbuild/cli/commands/_helpers/freshness/source_freshness.py @@ -8,13 +8,14 @@ from sqlbuild._native import ( CostContext, DirectSourceFreshnessPlanningResult, + ModelPlanEntry, + PlanOutput, SourceFreshnessIdentity, SourceFreshnessRecord, SourceFreshnessRenderers, write_source_freshness_records, ) from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter -from sqlbuild.compiler.planner.models import ModelPlanEntry, PlanOutput from sqlbuild.executor.build.models import BuildExecutionResult from sqlbuild.executor.scheduling.types import ExecutionStatus from sqlbuild.runtime.observability.classes.operation_lifecycle import OperationLifecycle diff --git a/src/sqlbuild/compiler/migrations/models.py b/src/sqlbuild/compiler/migrations/models.py deleted file mode 100644 index 115abbdb10..0000000000 --- a/src/sqlbuild/compiler/migrations/models.py +++ /dev/null @@ -1,155 +0,0 @@ -"""Model migration state domain models.""" - -from __future__ import annotations - -from dataclasses import dataclass -from datetime import datetime - -from sqlbuild._native import MigrationDiscovery -from sqlbuild.compiler.migrations.types import ( - ColumnMigrationDecision, - MigrationDecision, - OldNameViewDropReason, - OldNameViewEventType, - OldNameViewStatus, -) - - -@dataclass(frozen=True) -class MigrationRelation: - """One physical relation named by a migration event.""" - - database: str | None - schema: str | None - name: str - - @property - def identity(self) -> tuple[str | None, str, str]: - """Return the case-insensitive relation identity.""" - - return ( - None if self.database is None else self.database.lower(), - "" if self.schema is None else self.schema.lower(), - self.name.lower(), - ) - - def matches(self, other: MigrationRelation) -> bool: - """Compare relations, treating an unknown database as a wildcard.""" - - left: tuple[str | None, str, str] = self.identity - right: tuple[str | None, str, str] = other.identity - if left[1:] != right[1:]: - return False - return left[0] is None or right[0] is None or left[0] == right[0] - - -@dataclass(frozen=True) -class MigrationEvent: - """One immutable fact that a origin relation was cloned over a destination.""" - - event_id: str - target_name: str | None - origin_model: str | None - origin: MigrationRelation - destination_model: str - destination: MigrationRelation - origin_version_hash: str - discovery: MigrationDiscovery - decision: MigrationDecision - run_id: str - created_at: datetime - - def mentions(self, relation: MigrationRelation) -> bool: - """Return whether this event names the relation as origin or destination.""" - - return self.origin.matches(relation) or self.destination.matches(relation) - - -@dataclass(frozen=True) -class ColumnMigrationEvent: - """One immutable fact that a column of a model relation was renamed in place.""" - - event_id: str - target_name: str | None - model_name: str - relation: MigrationRelation - origin_column: str - destination_column: str - discovery: MigrationDiscovery - decision: ColumnMigrationDecision - run_id: str - created_at: datetime - - def mentions(self, *, relation: MigrationRelation, columns: frozenset[str]) -> bool: - """Return whether this event renamed one of the columns of the relation.""" - - return self.relation.matches(relation) and bool( - {self.origin_column.lower(), self.destination_column.lower()} & columns - ) - - def renamed(self, *, origin_column: str, destination_column: str) -> bool: - """Return whether this event renamed exactly this origin column to this destination.""" - - return ( - self.origin_column.lower() == origin_column.lower() - and self.destination_column.lower() == destination_column.lower() - ) - - -@dataclass(frozen=True) -class OldNameViewEvent: - """One immutable fact about the old name of one recorded move.""" - - event_id: str - target_name: str | None - event_type: OldNameViewEventType - migration_event_id: str - destination_model: str - old: MigrationRelation - new: MigrationRelation - run_id: str - created_at: datetime - view_retention: str | None = None - archive_name: str | None = None - column_aliases: tuple[tuple[str, str], ...] = () - expires_at: datetime | None = None - drop_reason: OldNameViewDropReason | None = None - grants_copied: tuple[str, ...] | None = None - view_sql: str | None = None - - -@dataclass(frozen=True) -class OldNameViewHistory: - """The recorded move and every old-name fact that references it.""" - - move: MigrationEvent - required: OldNameViewEvent - archived: OldNameViewEvent | None = None - created: OldNameViewEvent | None = None - dropped: OldNameViewEvent | None = None - - def status(self, *, now: datetime) -> OldNameViewStatus: - """Project the current state of this move's old-name steps.""" - - if self.dropped is not None: - return OldNameViewStatus.DROPPED - if self.created is not None: - expires_at: datetime | None = self.created.expires_at - if expires_at is not None and expires_at <= now: - return OldNameViewStatus.EXPIRED - return OldNameViewStatus.LIVE - if self.archived is not None: - return OldNameViewStatus.PENDING_VIEW - return OldNameViewStatus.PENDING_ARCHIVE - - @property - def old(self) -> MigrationRelation: - """Return the relation name the compatibility view occupies.""" - - return self.required.old - - @property - def new(self) -> MigrationRelation: - """Return the relation the compatibility view reads.""" - - return self.required.new diff --git a/src/sqlbuild/compiler/migrations/types.py b/src/sqlbuild/compiler/migrations/types.py deleted file mode 100644 index ff24918059..0000000000 --- a/src/sqlbuild/compiler/migrations/types.py +++ /dev/null @@ -1,154 +0,0 @@ -"""Stable model migration value types.""" - -from __future__ import annotations - -from enum import StrEnum - - -class MigrationDecision(StrEnum): - """Per-run decision for one declared or discovered model migration.""" - - DONE = "done" - MIGRATE = "migrate" - REDO = "redo" - SUPERSEDED_REPLACE = "superseded_replace" - FORCED_REPLACE = "forced_replace" - CONFLICT = "conflict" - ORIGIN_MISSING = "origin_missing" - RENAMED = "renamed" - - @property - def moves_data(self) -> bool: - """Return whether this decision clones the origin over the destination.""" - - return self in _MOVING_DECISIONS - - @property - def blocks_build(self) -> bool: - """Return whether this decision stops a build regardless of compatibility.""" - - return self in _BLOCKING_DECISIONS - - @property - def checks_compatibility(self) -> bool: - """Return whether this decision evaluated the origin against the destination.""" - - return self.moves_data or self == MigrationDecision.CONFLICT - - @property - def label(self) -> str: - """Return the human-readable plan label for this decision.""" - - return self.value.replace("_", " ") - - -_MOVING_DECISIONS: frozenset[MigrationDecision] = frozenset( - { - MigrationDecision.MIGRATE, - MigrationDecision.REDO, - MigrationDecision.SUPERSEDED_REPLACE, - MigrationDecision.FORCED_REPLACE, - } -) -_BLOCKING_DECISIONS: frozenset[MigrationDecision] = frozenset( - {MigrationDecision.CONFLICT, MigrationDecision.ORIGIN_MISSING} -) - - -class MigrationPromotion(StrEnum): - """How a completed migration stage replaces or creates the destination.""" - - SWAP = "swap" - RENAME = "rename" - TRANSACTIONAL_RENAME = "transactional_rename" - - @property - def label(self) -> str: - """Return the human-readable plan label for this promotion.""" - - return self.value.replace("_", " ") - - -class ColumnMigrationDecision(StrEnum): - """Per-run decision for one declared or detected column rename.""" - - RENAME = "rename" - RECORD = "record" - DONE = "done" - CONFLICT = "conflict" - SOURCE_MISSING = "source_missing" - STILL_PRODUCED = "still_produced" - UNSUPPORTED = "unsupported" - - @property - def renames(self) -> bool: - """Return whether this decision renames the warehouse column.""" - - return self == ColumnMigrationDecision.RENAME - - @property - def records_event(self) -> bool: - """Return whether this decision appends a column migration event.""" - - return self in _RECORDING_COLUMN_DECISIONS - - @property - def blocks_build(self) -> bool: - """Return whether this decision stops a build before any execution.""" - - return self in _BLOCKING_COLUMN_DECISIONS - - @property - def label(self) -> str: - """Return the human-readable plan label for this decision.""" - - return _COLUMN_DECISION_LABELS[self] - - -class OldNameViewEventType(StrEnum): - """Immutable facts about what happened at a migrated relation's old name.""" - - REQUIRED = "required" - ORIGIN_ARCHIVED = "origin_archived" - VIEW_CREATED = "view_created" - VIEW_DROPPED = "view_dropped" - - -class OldNameViewDropReason(StrEnum): - """Why a compatibility view at an old name was dropped.""" - - EXPIRED = "expired" - EARLY = "early" - MISSING = "missing" - - -class OldNameViewStatus(StrEnum): - """Projected state of the old-name steps of one recorded move.""" - - PENDING_ARCHIVE = "pending_archive" - PENDING_VIEW = "pending_view" - LIVE = "live" - EXPIRED = "expired" - DROPPED = "dropped" - - -_RECORDING_COLUMN_DECISIONS: frozenset[ColumnMigrationDecision] = frozenset( - {ColumnMigrationDecision.RENAME, ColumnMigrationDecision.RECORD} -) -_BLOCKING_COLUMN_DECISIONS: frozenset[ColumnMigrationDecision] = frozenset( - { - ColumnMigrationDecision.CONFLICT, - ColumnMigrationDecision.SOURCE_MISSING, - ColumnMigrationDecision.STILL_PRODUCED, - ColumnMigrationDecision.UNSUPPORTED, - } -) -_COLUMN_DECISION_LABELS: dict[ColumnMigrationDecision, str] = { - ColumnMigrationDecision.RENAME: "rename in place", - ColumnMigrationDecision.RECORD: "already renamed, record", - ColumnMigrationDecision.DONE: "done", - ColumnMigrationDecision.CONFLICT: "conflict", - ColumnMigrationDecision.SOURCE_MISSING: "source missing", - ColumnMigrationDecision.STILL_PRODUCED: "source still produced", - ColumnMigrationDecision.UNSUPPORTED: "unsupported", -} diff --git a/src/sqlbuild/compiler/pipeline/_helpers/materializations.py b/src/sqlbuild/compiler/pipeline/_helpers/materializations.py index a03e62ffdc..e40fee03a5 100644 --- a/src/sqlbuild/compiler/pipeline/_helpers/materializations.py +++ b/src/sqlbuild/compiler/pipeline/_helpers/materializations.py @@ -7,8 +7,8 @@ from types import ModuleType from typing import Any +from sqlbuild._native import PlannerInputError from sqlbuild.compiler.discovery.models import DiscoveredMaterializationFile -from sqlbuild.compiler.planner.exceptions import PlannerInputError from sqlbuild.executor.custom.models import MaterializationContext, MaterializationResult diff --git a/src/sqlbuild/compiler/pipeline/models.py b/src/sqlbuild/compiler/pipeline/models.py index db16309d72..1d198eee9e 100644 --- a/src/sqlbuild/compiler/pipeline/models.py +++ b/src/sqlbuild/compiler/pipeline/models.py @@ -10,14 +10,12 @@ import sqlbuild._native as _native from sqlbuild._native import ( LineageGraphViews, + PlanOutput, PythonPlanEntry, lineage_graph_views, ) from sqlbuild.compiler.compile.models import CompiledObjectKey from sqlbuild.compiler.compile.types import CompiledProjectView -from sqlbuild.compiler.planner.models import ( - PlanOutput, -) @dataclass(frozen=True) diff --git a/src/sqlbuild/compiler/planner/constants.py b/src/sqlbuild/compiler/planner/constants.py index cffbab89c0..fe462093f5 100644 --- a/src/sqlbuild/compiler/planner/constants.py +++ b/src/sqlbuild/compiler/planner/constants.py @@ -2,12 +2,8 @@ from __future__ import annotations -from sqlbuild._native import cursor_sentinel_to_token as sentinel_to_token -from sqlbuild.cursor_algebra.types import BoundSentinel - -SELECTOR_KIND_SEPARATOR: str = ":" -MICROBATCH_START_SENTINEL: str = sentinel_to_token(sentinel=BoundSentinel.START) -MICROBATCH_END_SENTINEL: str = sentinel_to_token(sentinel=BoundSentinel.END) +MICROBATCH_START_SENTINEL: str = "__SQB_CURSOR_START__" +MICROBATCH_END_SENTINEL: str = "__SQB_CURSOR_END__" MIGRATION_FINGERPRINT_METADATA_KEY: str = "migration_fingerprint" SNAPSHOT_DEFAULT_VALID_FROM_COLUMN: str = "valid_from" SNAPSHOT_DEFAULT_VALID_TO_COLUMN: str = "valid_to" diff --git a/src/sqlbuild/compiler/planner/exceptions.py b/src/sqlbuild/compiler/planner/exceptions.py deleted file mode 100644 index 18882d134f..0000000000 --- a/src/sqlbuild/compiler/planner/exceptions.py +++ /dev/null @@ -1,33 +0,0 @@ -"""Expected planner-stage exception types.""" - -from __future__ import annotations - -from typing import Any - - -class PlannerInputError(ValueError): - """Raised when planner inputs cannot be resolved safely.""" - - code: str = "S000" - - def __init__(self, message: str, *, code: str | None = None, help: str | None = None) -> None: - super().__init__(message) - self.message = message - self.code = code if code is not None else self.code - self.help = help - - -class NativeSqlTestPlanningError(PlannerInputError): - """Raised when the native SQL-test planning boundary returns invalid data.""" - - -class FutureCursorSafetyError(PlannerInputError): - """Raised when an effective cursor exceeds the configured future limit.""" - - -class MaximumAutomaticStartError(PlannerInputError): - """Raised when automatic cursor recovery would exceed its configured horizon.""" - - def __init__(self, message: str, *, evidence: Any | None = None) -> None: - super().__init__(message) - self.evidence = evidence diff --git a/src/sqlbuild/compiler/planner/models.py b/src/sqlbuild/compiler/planner/models.py deleted file mode 100644 index 2b466b4075..0000000000 --- a/src/sqlbuild/compiler/planner/models.py +++ /dev/null @@ -1,1043 +0,0 @@ -"""Planner domain models.""" - -from __future__ import annotations - -from dataclasses import dataclass, field -from datetime import datetime -from decimal import Decimal, InvalidOperation -from pathlib import Path -from typing import Any - -from sqlbuild._native import ( - BackfillResult, - ColumnRenameHint, - CursorWatermarkMode, - DateValue, - DirectSourceFreshnessPlanningResult, - Fingerprint, - FunctionReturnColumn, - FutureCursorAction, - FutureCursorsConfig, - GraphResourceKind, - IntegerValue, - MeasurementThresholds, - MicrobatchLimitAction, - MigrationCompatibility, - MigrationDiscovery, - MissingMigrationOriginPolicy, - OldNameViewAction, - PlanProviderUsage, - PlanWarning, - ScenarioArtifactName, - ScenarioGraphPlan, - SchemaAction, - SchemaChangeKind, - SchemaColumnSource, - SchemaDynamicColumnFamily, - SeedCsvSettings, - SourceEntry, - SourceWriteStrategy, - SqlTestAssertionStep, - SqlValue, - StartCursorsConfig, - TimestampValue, -) -from sqlbuild._native import Duration as Duration -from sqlbuild._native import cursor_parse as parse -from sqlbuild.adapter.contract.models import ColumnInfo, RetentionRequest -from sqlbuild.adapter.contract.types import MigrationTransfer -from sqlbuild.compiler.auditing.types import ( - AuditAttachmentKind, - AuditEvaluationMode, - AuditRunScope, - AuditSeverity, -) -from sqlbuild.compiler.compile.models import CompiledObjectKey, CompiledRelationLocation -from sqlbuild.compiler.compile.types import ( - AttachedAuditTargetKind, - FunctionLanguage, -) -from sqlbuild.compiler.discovery.models import DiscoveredHookFunction, SqlTestParameterDeclaration -from sqlbuild.compiler.migrations.types import ( - ColumnMigrationDecision, - MigrationDecision, - MigrationPromotion, -) -from sqlbuild.compiler.planner.exceptions import PlannerInputError -from sqlbuild.compiler.planner.types import ( - BackfillAction, - CursorGrain, - CursorType, - MaterializationType, - OnSchemaChange, - PlanAction, - PlanReason, - RetentionDirection, - RetentionPlanPhase, - ScenarioArtifactKind, -) -from sqlbuild.cursor_algebra.exceptions import CursorAlgebraError -from sqlbuild.cursor_algebra.types import Bound, BoundSentinel, CursorScalar -from sqlbuild.runtime.contracts.types import ExecutionResourceKind - - -@dataclass(frozen=True, kw_only=True, init=False) -class CursorOverrides: - """Cursor override values from CLI flags; untyped `start`/`end` await planner typing.""" - - start_ts: CursorScalar | None = None - end_ts: CursorScalar | None = None - start_int: CursorScalar | None = None - end_int: CursorScalar | None = None - start: str | None = None - end: str | None = None - - def __init__( - self, - *, - start_ts: str | CursorScalar | None = None, - end_ts: str | CursorScalar | None = None, - start_int: str | CursorScalar | None = None, - end_int: str | CursorScalar | None = None, - start: str | None = None, - end: str | None = None, - ) -> None: - object.__setattr__(self, "start", start) - object.__setattr__(self, "end", end) - object.__setattr__( - self, - "start_ts", - self._parse_timestamp(field_name="--start-cursor-ts", value=start_ts), - ) - object.__setattr__( - self, "end_ts", self._parse_timestamp(field_name="--end-cursor-ts", value=end_ts) - ) - object.__setattr__( - self, - "start_int", - self._parse_integer(field_name="--start-cursor-int", value=start_int), - ) - object.__setattr__( - self, "end_int", self._parse_integer(field_name="--end-cursor-int", value=end_int) - ) - - @property - def has_untyped(self) -> bool: - """Return whether `--start-cursor` or `--end-cursor` still needs a cursor type.""" - - return self.start is not None or self.end is not None - - def typed_as(self, *, cursor_type: CursorType) -> CursorOverrides: - """Return these overrides with `--start-cursor`/`--end-cursor` read as `cursor_type`.""" - - if not self.has_untyped: - return self - if cursor_type == CursorType.INTEGER: - return CursorOverrides( - start_ts=self.start_ts, - end_ts=self.end_ts, - start_int=self._parse_integer(field_name="--start-cursor", value=self.start) - or self.start_int, - end_int=self._parse_integer(field_name="--end-cursor", value=self.end) - or self.end_int, - ) - return CursorOverrides( - start_ts=self._parse_timestamp(field_name="--start-cursor", value=self.start) - or self.start_ts, - end_ts=self._parse_timestamp(field_name="--end-cursor", value=self.end) or self.end_ts, - start_int=self.start_int, - end_int=self.end_int, - ) - - @staticmethod - def _parse_timestamp( - *, field_name: str, value: str | CursorScalar | None - ) -> CursorScalar | None: - if value is None or not isinstance(value, str): - return value - try: - datetime.fromisoformat(value) - except (ValueError, TypeError) as error: - raise PlannerInputError( - f"{field_name} value '{value}' is not a valid ISO timestamp: {error}" - ) from None - return parse(raw=value, cursor_type="timestamp") - - @staticmethod - def _parse_integer(*, field_name: str, value: str | CursorScalar | None) -> CursorScalar | None: - if value is None or not isinstance(value, str): - return value - try: - decimal_value: Decimal = Decimal(value) - if decimal_value != int(decimal_value): - raise PlannerInputError(f"{field_name} value '{value}' is not a whole number") - parsed: CursorScalar = parse(raw=value, cursor_type="integer") - if isinstance(parsed, IntegerValue): - return parsed - except InvalidOperation: - raise PlannerInputError( - f"{field_name} value '{value}' is not a valid integer" - ) from None - raise PlannerInputError(f"{field_name} value '{value}' is not a valid integer") - - -@dataclass(frozen=True) -class EmptyCursorInputDecision: - """Cursor inputs whose lack of rows leaves no window, and whether other inputs have rows.""" - - input_names: tuple[str, ...] = () - waiting_on_empty_inputs: bool = False - - -@dataclass(frozen=True, kw_only=True, init=False) -class ModelCursorSnapshot: - """Cursor MIN/MAX values gathered from warehouse for one incremental model.""" - - target_max: CursorScalar | None - upstream_mins: tuple[CursorScalar, ...] - upstream_maxes: tuple[CursorScalar, ...] - physical_target_max: CursorScalar | None = field(default=None, compare=False) - target_eligible_max: CursorScalar | None = None - target_relation: str | None = field(default=None, compare=False) - destination_cursor_column: str | None = field(default=None, compare=False) - input_evidence: tuple[CursorInputEvidence, ...] = field(default=(), compare=False) - expected_watermark_count: int = field(default=0, compare=False) - unavailable_watermark_tags: tuple[str, ...] = () - cursor_watermark_mode: CursorWatermarkMode = CursorWatermarkMode.ALL - upstream_terminal_starts: tuple[CursorScalar, ...] = () - upstream_terminal_ends: tuple[CursorScalar, ...] = () - upstream_end_inputs: tuple[tuple[CursorScalar | None, CursorScalar | None], ...] = () - upstream_availability_ends: tuple[CursorScalar, ...] = () - empty_input_names: tuple[str, ...] = field(default=(), compare=False) - unreadable_input_names: tuple[str, ...] = field(default=(), compare=False) - - def __init__(self, **values: Any) -> None: - defaults: dict[str, object] = { - "physical_target_max": None, - "target_eligible_max": None, - "target_relation": None, - "destination_cursor_column": None, - "input_evidence": (), - "expected_watermark_count": 0, - "unavailable_watermark_tags": (), - "cursor_watermark_mode": CursorWatermarkMode.ALL, - "upstream_terminal_starts": (), - "upstream_terminal_ends": (), - "upstream_end_inputs": (), - "upstream_availability_ends": (), - "empty_input_names": (), - "unreadable_input_names": (), - } - defaults.update(values) - field_name: str - for field_name in self.__dataclass_fields__: - object.__setattr__(self, field_name, defaults[field_name]) - self.__post_init__() - - def __post_init__(self) -> None: - for field_name in ("target_max", "physical_target_max", "target_eligible_max"): - object.__setattr__( - self, field_name, CursorBounds._parse_optional(raw=getattr(self, field_name)) - ) - for field_name in ( - "upstream_mins", - "upstream_maxes", - "upstream_terminal_starts", - "upstream_terminal_ends", - "upstream_availability_ends", - ): - object.__setattr__( - self, - field_name, - tuple( - CursorBounds._parse_inferred(raw=value) for value in getattr(self, field_name) - ), - ) - object.__setattr__( - self, - "upstream_end_inputs", - tuple( - ( - CursorBounds._parse_optional(raw=physical), - CursorBounds._parse_optional(raw=terminal), - ) - for physical, terminal in self.upstream_end_inputs - ), - ) - object.__setattr__( - self, "cursor_watermark_mode", CursorWatermarkMode(self.cursor_watermark_mode) - ) - - @property - def watermarks_available(self) -> bool: - """Return whether every required physical watermark value was available.""" - - return not self.unavailable_watermark_tags - - -@dataclass(frozen=True, kw_only=True, init=False) -class CursorBounds: - """Effective cursor start and end values for one incremental model.""" - - start: Bound - end: Bound - future_safety: FutureCursorSafetyEvidence | None = None - maximum_start_safety: MaximumStartSafetyEvidence | None = None - - def __init__( - self, - *, - start: Bound | str, - end: Bound | str, - future_safety: FutureCursorSafetyEvidence | None = None, - maximum_start_safety: MaximumStartSafetyEvidence | None = None, - ) -> None: - object.__setattr__(self, "start", CursorBounds._parse_bound(raw=start)) - object.__setattr__(self, "end", CursorBounds._parse_bound(raw=end)) - object.__setattr__(self, "future_safety", future_safety) - object.__setattr__(self, "maximum_start_safety", maximum_start_safety) - - @staticmethod - def _parse_inferred(*, raw: object) -> CursorScalar: - if isinstance(raw, TimestampValue | DateValue | IntegerValue): - return raw - try: - return parse(raw=str(raw), cursor_type="integer") - except CursorAlgebraError: - return parse(raw=str(raw), cursor_type="timestamp") - - @staticmethod - def _parse_optional(*, raw: object | None) -> CursorScalar | None: - return None if raw is None else CursorBounds._parse_inferred(raw=raw) - - @staticmethod - def _parse_bound(*, raw: object) -> Bound: - if isinstance(raw, BoundSentinel): - return raw - if raw == BoundSentinel.START.value: - return BoundSentinel.START - if raw == BoundSentinel.END.value: - return BoundSentinel.END - return CursorBounds._parse_inferred(raw=raw) - - -@dataclass(frozen=True, kw_only=True, init=False) -class CursorInputEvidence: - """Observed bounds for one physical cursor input.""" - - relation: str - cursor_column: str - minimum: CursorScalar | None - maximum: CursorScalar - - def __init__( - self, - *, - relation: str, - cursor_column: str, - minimum: CursorScalar | str | None, - maximum: CursorScalar | str, - ) -> None: - object.__setattr__(self, "relation", relation) - object.__setattr__(self, "cursor_column", cursor_column) - object.__setattr__(self, "minimum", minimum) - object.__setattr__(self, "maximum", maximum) - self.__post_init__() - - def __post_init__(self) -> None: - object.__setattr__(self, "minimum", CursorBounds._parse_optional(raw=self.minimum)) - object.__setattr__(self, "maximum", CursorBounds._parse_inferred(raw=self.maximum)) - - -@dataclass(frozen=True, kw_only=True, init=False) -class FutureCursorSafetyEvidence: - """Structured evidence for one applied future-cursor cap.""" - - action: FutureCursorAction - max_distance: str - invocation_time: CursorScalar - discovered_start: CursorScalar - discovered_end: CursorScalar - applied_start: CursorScalar - applied_end: CursorScalar - maximum_allowed_start: CursorScalar - maximum_allowed_end: CursorScalar - future_start_detected: bool - future_end_detected: bool - determining_relation: str | None - determining_cursor_column: str | None - inputs: tuple[CursorInputEvidence, ...] = () - - def __init__(self, **values: Any) -> None: - values.setdefault("inputs", ()) - field_name: str - for field_name in self.__dataclass_fields__: - object.__setattr__(self, field_name, values[field_name]) - self.__post_init__() - - def __post_init__(self) -> None: - for field_name in ( - "invocation_time", - "discovered_start", - "discovered_end", - "applied_start", - "applied_end", - "maximum_allowed_start", - "maximum_allowed_end", - ): - object.__setattr__( - self, field_name, CursorBounds._parse_inferred(raw=getattr(self, field_name)) - ) - - -@dataclass(frozen=True, kw_only=True, init=False) -class MaximumStartSafetyEvidence: - """Structured evidence for an automatic-start eligibility decision.""" - - action: FutureCursorAction - max_ahead: str - invocation_time: CursorScalar - physical_target_max: CursorScalar - highest_eligible_target_max: CursorScalar | None - effective_start: CursorScalar - maximum_allowed_start: CursorScalar - target_relation: str - cursor_column: str - - def __init__( - self, - *, - action: FutureCursorAction, - max_ahead: str, - invocation_time: CursorScalar | str, - physical_target_max: CursorScalar | str, - highest_eligible_target_max: CursorScalar | str | None, - effective_start: CursorScalar | str, - maximum_allowed_start: CursorScalar | str, - target_relation: str, - cursor_column: str, - ) -> None: - for field_name, value in ( - ("action", action), - ("max_ahead", max_ahead), - ("invocation_time", invocation_time), - ("physical_target_max", physical_target_max), - ("highest_eligible_target_max", highest_eligible_target_max), - ("effective_start", effective_start), - ("maximum_allowed_start", maximum_allowed_start), - ("target_relation", target_relation), - ("cursor_column", cursor_column), - ): - object.__setattr__(self, field_name, value) - self.__post_init__() - - def __post_init__(self) -> None: - for field_name in ( - "invocation_time", - "physical_target_max", - "effective_start", - "maximum_allowed_start", - ): - object.__setattr__( - self, field_name, CursorBounds._parse_inferred(raw=getattr(self, field_name)) - ) - object.__setattr__( - self, - "highest_eligible_target_max", - CursorBounds._parse_optional(raw=self.highest_eligible_target_max), - ) - - -@dataclass(frozen=True, kw_only=True, init=False) -class CursorInputRelation: - """One cursor-bearing input relation for runtime range discovery.""" - - relation: str - cursor_column: str - cursor_grain: CursorGrain | None = None - is_model_backed: bool = False - is_runtime_produced: bool = False - terminal_cursor_start: CursorScalar | None = None - terminal_cursor_end: CursorScalar | None = None - input_name: str | None = None - input_kind: GraphResourceKind | None = None - - def __init__( - self, - *, - relation: str, - cursor_column: str, - cursor_grain: CursorGrain | str | None = None, - is_model_backed: bool = False, - is_runtime_produced: bool = False, - terminal_cursor_start: CursorScalar | str | None = None, - terminal_cursor_end: CursorScalar | str | None = None, - input_name: str | None = None, - input_kind: GraphResourceKind | None = None, - ) -> None: - object.__setattr__(self, "relation", relation) - object.__setattr__(self, "cursor_column", cursor_column) - object.__setattr__(self, "cursor_grain", cursor_grain) - object.__setattr__(self, "is_model_backed", is_model_backed) - object.__setattr__(self, "is_runtime_produced", is_runtime_produced) - object.__setattr__(self, "terminal_cursor_start", terminal_cursor_start) - object.__setattr__(self, "terminal_cursor_end", terminal_cursor_end) - object.__setattr__(self, "input_name", input_name) - object.__setattr__(self, "input_kind", input_kind) - self.__post_init__() - - def __post_init__(self) -> None: - object.__setattr__( - self, - "cursor_grain", - CursorGrain(self.cursor_grain) if self.cursor_grain is not None else None, - ) - object.__setattr__( - self, - "terminal_cursor_start", - CursorBounds._parse_optional(raw=self.terminal_cursor_start), - ) - object.__setattr__( - self, "terminal_cursor_end", CursorBounds._parse_optional(raw=self.terminal_cursor_end) - ) - - @property - def is_runtime_owned(self) -> bool: - """Return whether bounds must be discovered after scheduled upstream execution.""" - - return self.is_runtime_produced - - -@dataclass(frozen=True) -class MaximumStartPolicyInputs: - """Effective policy and materialization safety inputs for automatic starts.""" - - config: StartCursorsConfig | None = None - invocation_time: datetime | None = None - incremental_strategy: str | None = None - incremental_mode: str | None = None - - -@dataclass(frozen=True) -class SchemaFinding: - """One detected schema difference between expected and warehouse columns.""" - - kind: SchemaChangeKind - column_name: str - source: SchemaColumnSource - expected_type: str | None = None - actual_type: str | None = None - - -@dataclass(frozen=True) -class RetentionPlanEntry: - """Warehouse retention work independent of model identity actions.""" - - request: RetentionRequest - model_names: tuple[str, ...] - actual_days: int | None - effective_days: int | None - source: str - direction: RetentionDirection - phase: RetentionPlanPhase - statements: tuple[str, ...] = field(default_factory=tuple) - irreversible_warning: str | None = None - decrease_policy: str = "deny" - - @property - def decreases(self) -> bool: - """Whether this entry lowers any live retention value.""" - - return self.direction in (RetentionDirection.DECREASE, RetentionDirection.MIXED) - - -@dataclass(frozen=True) -class TableTypePlanEntry: - """Snowflake table-type work independent of model identity actions.""" - - model_name: str - destination: CompiledRelationLocation - copy_name: str - desired_type: str - actual_type: str | None - source: str - downgrade: bool - downgrade_policy: str - irreversible_warning: str | None = None - - -@dataclass(frozen=True) -class ModelMigrationPlanEntry: - """One declared or discovered model migration and its per-run decision.""" - - model_name: str - discovery: MigrationDiscovery - decision: MigrationDecision - compatibility: MigrationCompatibility - origin_model: str | None - origin: CompiledRelationLocation - destination: CompiledRelationLocation - target_name: str | None - origin_version_hash: str = "" - origin_is_transient: bool = False - stage_is_transient: bool | None = None - transfer: MigrationTransfer | None = None - transfer_fallback: MigrationTransfer | None = None - promotion: MigrationPromotion | None = None - completed_at: datetime | None = None - compatibility_findings: tuple[str, ...] = () - message: str | None = None - origin_tracked: bool = False - missing_origin_policy: MissingMigrationOriginPolicy = MissingMigrationOriginPolicy.ALLOW - - @property - def origin_missing(self) -> bool: - """Return whether the declared origin was not found in this target.""" - - return self.decision == MigrationDecision.ORIGIN_MISSING - - @property - def origin_hidden(self) -> bool: - """Return whether the missing origin has build history in this target.""" - - return self.origin_missing and self.origin_tracked - - @property - def storage_transition(self) -> str | None: - """Return the Snowflake table-type transition label, when both types are known.""" - - if self.stage_is_transient is None: - return None - labels: tuple[str, str] = ("permanent", "transient") - return f"{labels[self.origin_is_transient]} -> {labels[self.stage_is_transient]}" - - @property - def target_label(self) -> str: - """Return the target name shown in migration text, naming the unnamed target 'default'.""" - - return self.target_name or "default" - - @property - def blocks_build(self) -> bool: - """Return whether this migration must stop a build before any execution.""" - - if self.origin_missing: - return ( - self.origin_tracked - or self.missing_origin_policy == MissingMigrationOriginPolicy.DENY - ) - return self.decision.blocks_build or ( - self.decision.moves_data and self.compatibility == MigrationCompatibility.INCOMPATIBLE - ) - - -@dataclass(frozen=True) -class OldNameViewPlanEntry: - """What one build does at the old name of one migrated model.""" - - model_name: str - origin: CompiledRelationLocation - destination: CompiledRelationLocation - action: OldNameViewAction - target_name: str | None - retention: str | None = None - expires_at: datetime | None = None - column_aliases: tuple[tuple[str, str], ...] = () - stores_history: bool = False - migration_event_id: str | None = None - records_requirement: bool = False - reason: str | None = None - grants_copied: int | None = None - grants_supported: bool = False - archived: bool = False - - @property - def runs_steps(self) -> bool: - """Return whether this build archives the old relation or creates the view.""" - - return self.action in (OldNameViewAction.ARCHIVE_AND_VIEW, OldNameViewAction.VIEW_ONLY) - - -@dataclass(frozen=True) -class OldNameView: - """One compatibility view recorded at an old name and not recorded as dropped.""" - - destination_model: str - old: CompiledRelationLocation - new: CompiledRelationLocation - expires_at: datetime | None - column_aliases: tuple[tuple[str, str], ...] - migration_event_id: str - target_name: str | None - name_reused_by: str | None = None - - -@dataclass(frozen=True) -class ColumnMigrationPlanEntry: - """One declared or detected in-place column rename and its per-run decision.""" - - model_name: str - destination: CompiledRelationLocation - origin_column: str - destination_column: str - discovery: MigrationDiscovery - decision: ColumnMigrationDecision - target_name: str | None - completed_at: datetime | None = None - message: str | None = None - missing_origin_policy: MissingMigrationOriginPolicy = MissingMigrationOriginPolicy.ALLOW - - @property - def origin_missing(self) -> bool: - """Return whether the declared origin column was not found in this target.""" - - return self.decision == ColumnMigrationDecision.SOURCE_MISSING - - @property - def blocks_build(self) -> bool: - """Return whether this column migration must stop a build before any execution.""" - - if self.origin_missing: - return self.missing_origin_policy == MissingMigrationOriginPolicy.DENY - return self.decision.blocks_build - - -@dataclass(frozen=True) -class ModelPlanEntry: - """Per-model execution plan entry with action, reason, and resolved artifacts.""" - - key: CompiledObjectKey - name: str - relative_path: Path - materialization_type: MaterializationType - action: PlanAction - reason: PlanReason - destination: CompiledRelationLocation - fingerprint_query_sql: str - resolved_sql: str - logical_ddl: str - migration_fingerprint: str | None = None - incremental_strategy: str | None = None - incremental_mode: str | None = None - microbatch_strategy: str | None = None - cursor_watermark_mode: str | None = None - cursor_column: str | None = None - cursor_type: str | None = None - cursor_grain: str | None = None - cursor_start: str | None = None - cursor_end: str | None = None - lookback: str | None = None - cursor_bounds: CursorBounds | None = None - empty_cursor_inputs: tuple[str, ...] = () - waiting_on_empty_inputs: bool = False - cursor_input_relations: tuple[CursorInputRelation, ...] = field(default_factory=tuple) - batch_size: str | None = None - batch_concurrency: int = 1 - unaccounted_partition_policy: str | None = None - microbatch_range: CursorBounds | None = None - microbatch_limit: int | None = None - declared_microbatch_limit_action: MicrobatchLimitAction | None = None - microbatch_safety_limit: int | None = None - microbatch_safety_limit_action: MicrobatchLimitAction | None = None - microbatch_limit_count: int | None = None - microbatch_limit_action: MicrobatchLimitAction | None = None - microbatch_limit_warning: str | None = None - start_cursor_override: str | None = None - end_cursor_override: str | None = None - future_cursor_config: FutureCursorsConfig | None = None - start_cursor_config: StartCursorsConfig | None = None - invocation_time: datetime | None = None - unique_key: tuple[str, ...] = field(default_factory=tuple) - merge_exclude_columns: tuple[str, ...] = field(default_factory=tuple) - snapshot_strategy: str | None = None - updated_at_column: str | None = None - check_columns: tuple[str, ...] = field(default_factory=tuple) - observed_at_column: str | None = None - historical_input: str | None = None - valid_from_column: str | None = None - valid_to_column: str | None = None - initial_valid_from: str | None = None - invalidate_hard_deletes: bool = False - snapshot_full_refresh: str | None = None - snapshot_schema_change: str | None = None - table_type: str = "transient" - on_schema_change: OnSchemaChange | None = None - type_enforcement: bool = False - declared_columns: tuple[ColumnInfo, ...] = field(default_factory=tuple) - contract_enforced: bool = False - contract_columns: tuple[ColumnInfo, ...] = field(default_factory=tuple) - contract_dynamic_columns: tuple[SchemaDynamicColumnFamily, ...] = field(default_factory=tuple) - pre_hooks: object = None - post_hooks: object = None - previous_query_sql: str | None = None - fingerprint_metadata_json: str | None = None - previous_metadata_json: str | None = None - fingerprint_version_hash: str | None = None - previous_version_hash: str | None = None - query_changed: bool = False - config_changed: bool = False - schema_actions: tuple[SchemaAction, ...] = field(default_factory=tuple) - schema_findings: tuple[SchemaFinding, ...] = field(default_factory=tuple) - backfill: BackfillResult = field( - default_factory=lambda: BackfillResult(action=BackfillAction.FORWARD_ONLY) - ) - changed_functions: tuple[str, ...] = field(default_factory=tuple) - custom_materialization_name: str | None = None - custom_config: dict[str, object] = field(default_factory=dict) - custom_placeholders: dict[str, str] = field(default_factory=dict) - column_rename_hints: tuple[ColumnRenameHint, ...] = field(default_factory=tuple) - - -@dataclass(frozen=True) -class CloneSourcePlanEntry: - """One managed source relation selected for target cloning.""" - - key: CompiledObjectKey - name: str - destination: CompiledRelationLocation - - -@dataclass(frozen=True) -class SeedPlanEntry: - """Per-seed execution plan entry.""" - - key: CompiledObjectKey - name: str - destination: CompiledRelationLocation - file_path: Path - columns: tuple[ColumnInfo, ...] - csv_settings: SeedCsvSettings - fingerprint_definition: str = "" - fingerprint_version_hash: str = "" - fingerprint_metadata_json: str = "{}" - action: PlanAction = PlanAction.LOAD_SEED - reason: PlanReason = PlanReason.FIRST_RUN - - -@dataclass(frozen=True) -class SourceLoadPlanEntry: - """Per-managed-source loader execution plan entry.""" - - key: CompiledObjectKey - name: str - loader: str - destination: str - resource_kind: ExecutionResourceKind = ExecutionResourceKind.SOURCE - write_strategy: SourceWriteStrategy | None = None - cursor_column: str | None = None - unique_key: tuple[str, ...] = field(default_factory=tuple) - is_reload: bool = False - integration_kind: str | None = None - - -@dataclass(frozen=True) -class FunctionPlanEntry: - """Per-SQL-function execution plan entry.""" - - key: CompiledObjectKey - name: str - relative_path: Path - destination: CompiledRelationLocation - arguments: tuple[object, ...] - returns: str - body_sql: str - fingerprint_query_sql: str - fingerprint_destination: CompiledRelationLocation - return_columns: tuple[FunctionReturnColumn, ...] = field(default_factory=tuple) - language: FunctionLanguage = FunctionLanguage.SQL - source_file_path: Path | None = None - runtime_version: str | None = None - entry_point: str | None = None - packages: tuple[str, ...] = field(default_factory=tuple) - previous_query_sql: str | None = None - reason: PlanReason = PlanReason.NO_CHANGE - - -@dataclass(frozen=True) -class AuditPlanEntry: - """Per-audit execution plan entry with resolved SQL and scheduling metadata.""" - - key: CompiledObjectKey - name: str - definition_name: str - resolved_sql: str - unresolved_sql: str - attachment_kind: AuditAttachmentKind - severity: AuditSeverity - requested_run_scope: AuditRunScope - effective_run_scope: AuditRunScope - evaluation_mode: AuditEvaluationMode = AuditEvaluationMode.VIOLATIONS - value_column: str | None = None - sample_count_column: str | None = None - sample_unit: str | None = None - thresholds: MeasurementThresholds | None = None - minimum_samples: int | None = None - evidence_resolved_sql: str | None = None - evidence_unresolved_sql: str | None = None - evidence_limit: int | None = None - scope_deps: tuple[CompiledObjectKey, ...] = field(default_factory=tuple) - attached_target_kind: AttachedAuditTargetKind | None = None - attached_target_name: str | None = None - attached_column_name: str | None = None - always_run: bool = False - description: str | None = None - reads_outside_target_lineage: bool = False - - -@dataclass(frozen=True) -class ChainStep: - """One step in a chained test execution sequence.""" - - model_name: str - resolved_sql: str - expected_cte_sql: str | None = None - lifted_ctes: tuple[tuple[str, str], ...] = field(default_factory=tuple) - comparison_body_sql: str | None = None - expected_columns: tuple[str, ...] | None = None - expected_lifted_ctes: tuple[tuple[str, str], ...] = field(default_factory=tuple) - - -@dataclass(frozen=True) -class SqlTestPlanEntry: - """Per-test execution plan entry with chained resolution.""" - - key: CompiledObjectKey - name: str - source_path: Path | None = None - block_index: int | None = None - parent_name: str | None = None - case_name: str | None = None - case_index: int | None = None - case_fingerprint: str | None = None - parameter_schema: tuple[SqlTestParameterDeclaration, ...] = field(default_factory=tuple) - parameter_values: tuple[tuple[str, SqlValue], ...] = field(default_factory=tuple) - mock_ref_names: tuple[str, ...] = field(default_factory=tuple) - mock_source_names: tuple[str, ...] = field(default_factory=tuple) - mock_seed_names: tuple[str, ...] = field(default_factory=tuple) - mock_dbt_ref_names: tuple[str, ...] = field(default_factory=tuple) - mock_table_function_names: tuple[str, ...] = field(default_factory=tuple) - chain: tuple[ChainStep, ...] = field(default_factory=tuple) - assertions: tuple[SqlTestAssertionStep, ...] = field(default_factory=tuple) - scope_deps: tuple[CompiledObjectKey, ...] = field(default_factory=tuple) - function_deps: tuple[CompiledObjectKey, ...] = field(default_factory=tuple) - sql_analysis_enabled: bool = True - - -@dataclass(frozen=True) -class ScenarioRelationMap: - """Resolved scenario hash prefix and scenario-owned relation names.""" - - scenario_name: str - hash_prefix: str - artifacts: tuple[ScenarioArtifactName, ...] = field(default_factory=tuple) - - -@dataclass(frozen=True) -class ScenarioRelationPlan: - """Resolved scenario relation locations for model/source/seed ref resolution.""" - - scenario_name: str - relation_map: ScenarioRelationMap - model_locations: dict[str, CompiledRelationLocation] = field(default_factory=dict) - seed_locations: dict[str, CompiledRelationLocation] = field(default_factory=dict) - project_source_map: dict[str, SourceEntry] = field(default_factory=dict) - source_map: dict[str, SourceEntry] = field(default_factory=dict) - source_fixture_locations: dict[str, CompiledRelationLocation] = field(default_factory=dict) - ref_fixture_locations: dict[str, CompiledRelationLocation] = field(default_factory=dict) - seed_fixture_locations: dict[str, CompiledRelationLocation] = field(default_factory=dict) - dbt_ref_fixture_locations: dict[str, CompiledRelationLocation] = field(default_factory=dict) - - -@dataclass(frozen=True) -class ScenarioFixturePlan: - """Self-contained fixture SQL planned for scenario materialization.""" - - kind: ScenarioArtifactKind - logical_name: str - destination: CompiledRelationLocation - sql: str - - -@dataclass(frozen=True) -class ScenarioExpectedExpectationPlan: - """Expected-output comparison inputs for one scenario target model.""" - - model_name: str - actual_destination: CompiledRelationLocation - expected_sql: str - - -@dataclass(frozen=True) -class ScenarioAssertionExpectationPlan: - """Zero-row assertion SQL for one scenario assertion CTE.""" - - name: str - sql: str - - -@dataclass(frozen=True) -class ScenarioExecutionPlan: - """Dry-run execution plan for a SQL scenario graph slice.""" - - key: CompiledObjectKey - name: str - graph_plan: ScenarioGraphPlan - relation_plan: ScenarioRelationPlan - fixture_plans: tuple[ScenarioFixturePlan, ...] = field(default_factory=tuple) - seed_entries: tuple[SeedPlanEntry, ...] = field(default_factory=tuple) - function_entries: tuple[FunctionPlanEntry, ...] = field(default_factory=tuple) - model_entries: tuple[ModelPlanEntry, ...] = field(default_factory=tuple) - hook_functions: tuple[DiscoveredHookFunction, ...] = field(default_factory=tuple) - expected_expectations: tuple[ScenarioExpectedExpectationPlan, ...] = field( - default_factory=tuple - ) - assertion_expectations: tuple[ScenarioAssertionExpectationPlan, ...] = field( - default_factory=tuple - ) - - -@dataclass(frozen=True) -class PlanOutput: - """Complete execution plan produced by the planner.""" - - execution_order: tuple[CompiledObjectKey, ...] = field(default_factory=tuple) - model_entries: tuple[ModelPlanEntry, ...] = field(default_factory=tuple) - seed_entries: tuple[SeedPlanEntry, ...] = field(default_factory=tuple) - source_load_entries: tuple[SourceLoadPlanEntry, ...] = field(default_factory=tuple) - function_entries: tuple[FunctionPlanEntry, ...] = field(default_factory=tuple) - audit_entries: tuple[AuditPlanEntry, ...] = field(default_factory=tuple) - test_entries: tuple[SqlTestPlanEntry, ...] = field(default_factory=tuple) - available_test_case_names: tuple[str, ...] = field(default_factory=tuple) - selected_keys: frozenset[CompiledObjectKey] = field(default_factory=frozenset) - warnings: tuple[PlanWarning, ...] = field(default_factory=tuple) - retention_entries: tuple[RetentionPlanEntry, ...] = field(default_factory=tuple) - table_type_entries: tuple[TableTypePlanEntry, ...] = field(default_factory=tuple) - migration_entries: tuple[ModelMigrationPlanEntry, ...] = field(default_factory=tuple) - column_migration_entries: tuple[ColumnMigrationPlanEntry, ...] = field(default_factory=tuple) - old_name_view_entries: tuple[OldNameViewPlanEntry, ...] = field(default_factory=tuple) - old_name_views: tuple[OldNameView, ...] = field(default_factory=tuple) - upstream_deps: dict[CompiledObjectKey, tuple[CompiledObjectKey, ...]] = field( - default_factory=dict - ) - downstream_deps: dict[CompiledObjectKey, tuple[CompiledObjectKey, ...]] = field( - default_factory=dict - ) - model_locations: dict[str, CompiledRelationLocation] = field(default_factory=dict) - seed_locations: dict[str, CompiledRelationLocation] = field(default_factory=dict) - function_locations: dict[str, CompiledRelationLocation] = field(default_factory=dict) - source_map: dict[str, SourceEntry] = field(default_factory=dict) - source_read_map: dict[str, SourceEntry] = field(default_factory=dict) - python_source_read_map: dict[str, SourceEntry] = field(default_factory=dict) - hook_functions: tuple[DiscoveredHookFunction, ...] = field(default_factory=tuple) - enforce_explicit_references: bool = True - provider_usages: tuple[PlanProviderUsage, ...] = field(default_factory=tuple) - source_freshness: DirectSourceFreshnessPlanningResult | None = None - python_identity_fingerprints: dict[tuple[str, str], Fingerprint] = field(default_factory=dict) - metadata: dict[str, object] = field(default_factory=dict) - cursor_overrides: CursorOverrides | None = None - - @property - def python_source_entries(self) -> dict[str, SourceEntry]: - """Return the source entries Python reads resolve, as a selected SQL read sees them.""" - - return self.python_source_read_map or self.source_read_map or self.source_map diff --git a/src/sqlbuild/compiler/planner/types.py b/src/sqlbuild/compiler/planner/types.py deleted file mode 100644 index 75c869829c..0000000000 --- a/src/sqlbuild/compiler/planner/types.py +++ /dev/null @@ -1,142 +0,0 @@ -"""Planner domain types.""" - -from __future__ import annotations - -from enum import StrEnum - - -class BackfillAction(StrEnum): - FULL = "full" - BOUNDED = "bounded" - FORWARD_ONLY = "forward" - - -class OnSchemaChange(StrEnum): - IGNORE = "ignore" - FAIL = "fail" - APPEND_NEW_COLUMNS = "append_new_columns" - SYNC_ALL_COLUMNS = "sync_all_columns" - - -class ContractPolicy(StrEnum): - NONE = "none" - ENFORCED = "enforced" - - -class RetentionDirection(StrEnum): - INCREASE = "increase" - DECREASE = "decrease" - MIXED = "mixed" - MATCH = "match" - APPLY_AFTER_CREATE = "apply_after_create" - - -class RetentionPlanPhase(StrEnum): - PRE = "pre" - POST = "post" - AFTER_CREATE = "after_create" - NONE = "none" - - -class PlanAction(StrEnum): - CREATE_VIEW = "create_view" - CREATE_TABLE = "create_table" - INCREMENTAL_APPEND = "incremental_append" - INCREMENTAL_DELETE_INSERT = "incremental_delete_insert" - INCREMENTAL_MERGE = "incremental_merge" - SNAPSHOT = "snapshot" - LOAD_SEED = "load_seed" - SKIP = "skip" - CUSTOM = "custom" - - -class PlanReason(StrEnum): - FIRST_RUN = "first_run" - RENAMED = "renamed" - FULL_REFRESH = "full_refresh" - QUERY_CHANGED = "query_changed" - FUNCTION_CHANGED = "function_changed" - CONFIG_CHANGED = "config_changed" - SCHEMA_CHANGED = "schema_changed" - NORMAL_INCREMENTAL = "normal_incremental" - NO_CHANGE = "no_change" - DISABLED = "disabled" - SOURCE_FRESHNESS_ERROR = "source_freshness_error" - EXTERNAL_UPSTREAM_FAILED = "external_upstream_failed" - - -class IncrementalStrategy(StrEnum): - APPEND = "append" - DELETE_INSERT = "delete_insert" - MERGE = "merge" - - -class MicrobatchStrategy(StrEnum): - """How an ordinary microbatch range is selected.""" - - ROLLING_WINDOW = "rolling_window" - WATERMARK = "watermark" - - -class IncrementalMode(StrEnum): - FULL = "full" - MICROBATCH = "microbatch" - - -class SnapshotStrategy(StrEnum): - TIMESTAMP = "timestamp" - CHECK = "check" - - -class HistoricalInput(StrEnum): - SNAPSHOT = "snapshot" - CHANGES = "changes" - - -class InitialValidFrom(StrEnum): - UPDATED_AT = "updated_at" - OBSERVED_AT = "observed_at" - EXECUTION_TIME = "execution_time" - - -class SnapshotSchemaChangePolicy(StrEnum): - DENY = "deny" - REQUIRE_CONFIRMATION = "require_confirmation" - APPEND_NEW_COLUMNS = "append_new_columns" - - -class CursorType(StrEnum): - TIMESTAMP = "timestamp" - INTEGER = "integer" - - -class CursorGrain(StrEnum): - SECOND = "second" - MINUTE = "minute" - HOUR = "hour" - DAY = "day" - MONTH = "month" - YEAR = "year" - - -class MaterializationType(StrEnum): - VIEW = "view" - TABLE = "table" - INCREMENTAL = "incremental" - SNAPSHOT = "snapshot" - SEED = "seed" - CUSTOM = "custom" - - @classmethod - def is_table_backed(cls, *, materialized: object | None) -> bool: - """Return whether a materialization owns a physical managed table.""" - - return materialized in {cls.TABLE, cls.INCREMENTAL, cls.SNAPSHOT} - - -class ScenarioArtifactKind(StrEnum): - SOURCE = "source" - REF = "ref" - SEED = "seed" - DBT_REF = "dbt_ref" - MODEL = "model" diff --git a/src/sqlbuild/cursor_algebra/constants.py b/src/sqlbuild/cursor_algebra/constants.py index 094a033b7e..684a9e6354 100644 --- a/src/sqlbuild/cursor_algebra/constants.py +++ b/src/sqlbuild/cursor_algebra/constants.py @@ -1,6 +1,6 @@ """Canonical cursor grain metadata.""" -from sqlbuild.compiler.planner.types import CursorGrain +from sqlbuild._native import CursorGrain GRAIN_ORDER: dict[CursorGrain, int] = { CursorGrain.SECOND: 0, diff --git a/src/sqlbuild/executor/auditing/_helpers/execution.py b/src/sqlbuild/executor/auditing/_helpers/execution.py index 4161711549..4a0d5d7310 100644 --- a/src/sqlbuild/executor/auditing/_helpers/execution.py +++ b/src/sqlbuild/executor/auditing/_helpers/execution.py @@ -10,6 +10,7 @@ from typing import Any, cast from sqlbuild._native import ( + AuditPlanEntry, CostContext, SourceEntry, assert_no_unresolved_sql_markers, @@ -23,7 +24,6 @@ AuditRunScope, ) from sqlbuild.compiler.compile.models import CompiledRelationLocation -from sqlbuild.compiler.planner.models import AuditPlanEntry from sqlbuild.diagnostics.main.diagnostics_context import diagnostics_context from sqlbuild.executor.auditing.constants import ( DEFAULT_EVIDENCE_ROW_LIMIT, diff --git a/src/sqlbuild/executor/auditing/_helpers/result_projection.py b/src/sqlbuild/executor/auditing/_helpers/result_projection.py index f61f547cba..80d7baec19 100644 --- a/src/sqlbuild/executor/auditing/_helpers/result_projection.py +++ b/src/sqlbuild/executor/auditing/_helpers/result_projection.py @@ -13,11 +13,16 @@ from datetime import UTC, datetime from typing import Any -from sqlbuild._native import AuditIdentity, MeasurementThresholdBound, build_audit_gate_identity +from sqlbuild._native import ( + AuditIdentity, + AuditPlanEntry, + MeasurementThresholdBound, + PlanOutput, + build_audit_gate_identity, +) from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter from sqlbuild.compiler.auditing.types import AuditEvaluationMode, ThresholdOperator from sqlbuild.compiler.compile.models import CompiledRelationLocation -from sqlbuild.compiler.planner.models import AuditPlanEntry, PlanOutput from sqlbuild.executor.audit_results.constants import AUDIT_RESULT_SCHEMA_VERSION from sqlbuild.executor.audit_results.exceptions import AuditResultStorageError from sqlbuild.executor.audit_results.main._write import write_audit_result_records diff --git a/src/sqlbuild/executor/auditing/main/_execute.py b/src/sqlbuild/executor/auditing/main/_execute.py index 2e5f1472a0..6ff96df6ed 100644 --- a/src/sqlbuild/executor/auditing/main/_execute.py +++ b/src/sqlbuild/executor/auditing/main/_execute.py @@ -4,11 +4,10 @@ from typing import Any -from sqlbuild._native import SourceEntry +from sqlbuild._native import AuditPlanEntry, SourceEntry from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter from sqlbuild.compiler.auditing.types import AuditRunScope from sqlbuild.compiler.compile.models import CompiledRelationLocation -from sqlbuild.compiler.planner.models import AuditPlanEntry from sqlbuild.executor.auditing._helpers.execution import execute_audit_impl from sqlbuild.executor.auditing.models import AuditExecutionResult diff --git a/src/sqlbuild/executor/auditing/main/_execute_reporting_errors.py b/src/sqlbuild/executor/auditing/main/_execute_reporting_errors.py index c98c402be6..e1233f76d6 100644 --- a/src/sqlbuild/executor/auditing/main/_execute_reporting_errors.py +++ b/src/sqlbuild/executor/auditing/main/_execute_reporting_errors.py @@ -4,11 +4,10 @@ from typing import Any -from sqlbuild._native import SourceEntry +from sqlbuild._native import AuditPlanEntry, SourceEntry from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter from sqlbuild.compiler.auditing.types import AuditRunScope from sqlbuild.compiler.compile.models import CompiledRelationLocation -from sqlbuild.compiler.planner.models import AuditPlanEntry from sqlbuild.executor.auditing._helpers.execution import execute_audit_impl, failed_audit_result from sqlbuild.executor.auditing.models import AuditExecutionResult diff --git a/src/sqlbuild/executor/auditing/main/_project_results.py b/src/sqlbuild/executor/auditing/main/_project_results.py index 21eacefa87..93cfe5cbd3 100644 --- a/src/sqlbuild/executor/auditing/main/_project_results.py +++ b/src/sqlbuild/executor/auditing/main/_project_results.py @@ -4,8 +4,8 @@ from typing import Any +from sqlbuild._native import PlanOutput from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter -from sqlbuild.compiler.planner.models import PlanOutput from sqlbuild.executor.auditing._helpers.result_projection import project_audit_result_batch_impl from sqlbuild.executor.auditing.models import AuditExecutionResult, AuditResultProjection diff --git a/src/sqlbuild/executor/auditing/main/audit_result_publication_scope.py b/src/sqlbuild/executor/auditing/main/audit_result_publication_scope.py index af77daa140..9ac2c247fd 100644 --- a/src/sqlbuild/executor/auditing/main/audit_result_publication_scope.py +++ b/src/sqlbuild/executor/auditing/main/audit_result_publication_scope.py @@ -5,7 +5,7 @@ from collections.abc import Iterator from contextlib import contextmanager -from sqlbuild.compiler.planner.models import PlanOutput +from sqlbuild._native import PlanOutput from sqlbuild.executor.auditing._helpers.result_projection import ( audit_result_publication_scope_impl, ) diff --git a/src/sqlbuild/executor/build/_helpers/end_audits.py b/src/sqlbuild/executor/build/_helpers/end_audits.py index 9671bbf12f..4d2e8e8a07 100644 --- a/src/sqlbuild/executor/build/_helpers/end_audits.py +++ b/src/sqlbuild/executor/build/_helpers/end_audits.py @@ -4,11 +4,10 @@ from typing import Any -from sqlbuild._native import SourceEntry, audit_resource_id +from sqlbuild._native import AuditPlanEntry, SourceEntry, audit_resource_id from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter from sqlbuild.compiler.auditing.types import AuditOutcome, AuditRunScope from sqlbuild.compiler.compile.models import CompiledRelationLocation -from sqlbuild.compiler.planner.models import AuditPlanEntry from sqlbuild.executor.auditing.main._execute_reporting_errors import ( execute_audit_reporting_errors, ) diff --git a/src/sqlbuild/executor/build/_helpers/external_source_loads.py b/src/sqlbuild/executor/build/_helpers/external_source_loads.py index 063650d68b..32fcfa2642 100644 --- a/src/sqlbuild/executor/build/_helpers/external_source_loads.py +++ b/src/sqlbuild/executor/build/_helpers/external_source_loads.py @@ -4,12 +4,11 @@ from collections.abc import Callable -from sqlbuild._native import SourceEntry +from sqlbuild._native import PlanOutput, SourceEntry, SourceLoadPlanEntry from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter from sqlbuild.compiler.compile.models import CompiledObjectKey from sqlbuild.compiler.discovery.models import DiscoveredLoaderFunction from sqlbuild.compiler.discovery.types import LoaderConnectionMode -from sqlbuild.compiler.planner.models import PlanOutput, SourceLoadPlanEntry from sqlbuild.errors.contracts.exceptions import ExecutorInputError from sqlbuild.executor.build._helpers.source_node import execute_build_source_node from sqlbuild.executor.build.models import ( diff --git a/src/sqlbuild/executor/build/_helpers/indexes.py b/src/sqlbuild/executor/build/_helpers/indexes.py index d1978b36a3..04df581d06 100644 --- a/src/sqlbuild/executor/build/_helpers/indexes.py +++ b/src/sqlbuild/executor/build/_helpers/indexes.py @@ -2,8 +2,8 @@ from __future__ import annotations +from sqlbuild._native import AuditPlanEntry, PlanOutput from sqlbuild.compiler.auditing.types import AuditAttachmentKind -from sqlbuild.compiler.planner.models import AuditPlanEntry, PlanOutput from sqlbuild.executor.build.models import BuildIndexes diff --git a/src/sqlbuild/executor/build/_helpers/retention.py b/src/sqlbuild/executor/build/_helpers/retention.py index 0b2dd5f845..cff1e935bd 100644 --- a/src/sqlbuild/executor/build/_helpers/retention.py +++ b/src/sqlbuild/executor/build/_helpers/retention.py @@ -4,7 +4,17 @@ from typing import Any -from sqlbuild._native import TableType +from sqlbuild._native import ( + IncrementalMode, + ModelPlanEntry, + PlanAction, + PlanOutput, + RetentionDirection, + RetentionPlanEntry, + RetentionPlanPhase, + TableType, + TableTypePlanEntry, +) from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter from sqlbuild.adapter.contract.models import ( RelationInfo, @@ -16,18 +26,6 @@ from sqlbuild.adapter.relations.main.resolve_qualified_name_parts import ( resolve_qualified_name_parts, ) -from sqlbuild.compiler.planner.models import ( - ModelPlanEntry, - PlanOutput, - RetentionPlanEntry, - TableTypePlanEntry, -) -from sqlbuild.compiler.planner.types import ( - IncrementalMode, - PlanAction, - RetentionDirection, - RetentionPlanPhase, -) from sqlbuild.errors.contracts.exceptions import ExecutorInputError from sqlbuild.runtime.observability.classes.operation_lifecycle import ( OperationAttributes, diff --git a/src/sqlbuild/executor/build/_helpers/scheduler.py b/src/sqlbuild/executor/build/_helpers/scheduler.py index 480429e6de..6d136f34c1 100644 --- a/src/sqlbuild/executor/build/_helpers/scheduler.py +++ b/src/sqlbuild/executor/build/_helpers/scheduler.py @@ -5,6 +5,11 @@ from collections.abc import Callable, Mapping from sqlbuild._native import ( + IncrementalMode, + MaterializationType, + ModelPlanEntry, + PlanAction, + PlanReason, SnapshotsConfig, assert_no_unresolved_sql_markers, replaces_incremental_relation, @@ -14,13 +19,6 @@ from sqlbuild.adapter.contract.types import TablePromotionMode from sqlbuild.compiler.compile.models import CompiledObjectKey from sqlbuild.compiler.planner.constants import MICROBATCH_END_SENTINEL, MICROBATCH_START_SENTINEL -from sqlbuild.compiler.planner.models import ModelPlanEntry -from sqlbuild.compiler.planner.types import ( - IncrementalMode, - MaterializationType, - PlanAction, - PlanReason, -) from sqlbuild.errors.contracts.exceptions import ExecutorInputError from sqlbuild.executor.build.constants import ( BUILD_CUSTOM_MATERIALIZATION_MISSING_CODE, diff --git a/src/sqlbuild/executor/build/_helpers/source_audits.py b/src/sqlbuild/executor/build/_helpers/source_audits.py index 17da4ee61c..fdce952fc4 100644 --- a/src/sqlbuild/executor/build/_helpers/source_audits.py +++ b/src/sqlbuild/executor/build/_helpers/source_audits.py @@ -4,12 +4,11 @@ from typing import Any -from sqlbuild._native import audit_resource_id +from sqlbuild._native import AuditPlanEntry, PlanOutput, audit_resource_id from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter from sqlbuild.compiler.auditing.types import AuditOutcome, AuditRunScope from sqlbuild.compiler.compile.models import CompiledObjectKey from sqlbuild.compiler.compile.types import CompiledResourceType -from sqlbuild.compiler.planner.models import AuditPlanEntry, PlanOutput from sqlbuild.executor.auditing.main._execute_reporting_errors import ( execute_audit_reporting_errors, ) diff --git a/src/sqlbuild/executor/build/_helpers/source_node.py b/src/sqlbuild/executor/build/_helpers/source_node.py index 9992c266c6..152bedf3ea 100644 --- a/src/sqlbuild/executor/build/_helpers/source_node.py +++ b/src/sqlbuild/executor/build/_helpers/source_node.py @@ -7,12 +7,11 @@ from collections.abc import Callable from typing import Any -from sqlbuild._native import CostContext, SourceEntry +from sqlbuild._native import CostContext, PlanOutput, SourceEntry, SourceLoadPlanEntry from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter from sqlbuild.adapter.contract.classes.statement_recorder import StatementRecorder from sqlbuild.compiler.compile.models import CompiledObjectKey from sqlbuild.compiler.discovery.models import DiscoveredLoaderFunction -from sqlbuild.compiler.planner.models import PlanOutput, SourceLoadPlanEntry from sqlbuild.executor.build.models import BuildCallbacks, BuildRuntimeParams from sqlbuild.executor.load.main._execute import execute_source_load from sqlbuild.executor.load.models import LoadExecutionResult, LoadRuntimeParams diff --git a/src/sqlbuild/executor/build/classes/build_scheduler.py b/src/sqlbuild/executor/build/classes/build_scheduler.py index f5fc1f4c14..3f6fd1fac2 100644 --- a/src/sqlbuild/executor/build/classes/build_scheduler.py +++ b/src/sqlbuild/executor/build/classes/build_scheduler.py @@ -15,29 +15,30 @@ from pathlib import Path from typing import Any, cast -from sqlbuild._native import CostContext, SnapshotsConfig, SourceEntry, sql_test_resource_id -from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter -from sqlbuild.adapter.contract.classes.statement_recorder import StatementRecorder -from sqlbuild.adapter.contract.models import RelationInfo -from sqlbuild.adapter.contract.types import TablePromotionMode -from sqlbuild.compiler.auditing.types import AuditOutcome -from sqlbuild.compiler.compile.models import CompiledObjectKey -from sqlbuild.compiler.compile.types import CompiledResourceType -from sqlbuild.compiler.discovery.models import DiscoveredLoaderFunction -from sqlbuild.compiler.planner.models import ( +from sqlbuild._native import ( AuditPlanEntry, + CostContext, FunctionPlanEntry, + IncrementalMode, + IncrementalStrategy, ModelPlanEntry, + PlanAction, PlanOutput, SeedPlanEntry, + SnapshotsConfig, + SourceEntry, SqlTestPlanEntry, TableTypePlanEntry, + sql_test_resource_id, ) -from sqlbuild.compiler.planner.types import ( - IncrementalMode, - IncrementalStrategy, - PlanAction, -) +from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter +from sqlbuild.adapter.contract.classes.statement_recorder import StatementRecorder +from sqlbuild.adapter.contract.models import RelationInfo +from sqlbuild.adapter.contract.types import TablePromotionMode +from sqlbuild.compiler.auditing.types import AuditOutcome +from sqlbuild.compiler.compile.models import CompiledObjectKey +from sqlbuild.compiler.compile.types import CompiledResourceType +from sqlbuild.compiler.discovery.models import DiscoveredLoaderFunction from sqlbuild.cost.models import StatementExecutionTelemetry from sqlbuild.diagnostics.main.diagnostics_context import diagnostics_context from sqlbuild.diagnostics.main.log_debug_event import log_debug_event diff --git a/src/sqlbuild/executor/build/constants.py b/src/sqlbuild/executor/build/constants.py index 5a0902b57f..f7109ec129 100644 --- a/src/sqlbuild/executor/build/constants.py +++ b/src/sqlbuild/executor/build/constants.py @@ -2,7 +2,7 @@ from __future__ import annotations -from sqlbuild.compiler.planner.types import PlanAction +from sqlbuild._native import PlanAction INCREMENTAL_ACTIONS: frozenset[PlanAction] = frozenset( { diff --git a/src/sqlbuild/executor/build/main/_execute.py b/src/sqlbuild/executor/build/main/_execute.py index 27b9768f1b..d67d05dcec 100644 --- a/src/sqlbuild/executor/build/main/_execute.py +++ b/src/sqlbuild/executor/build/main/_execute.py @@ -5,13 +5,11 @@ from dataclasses import replace from typing import Any -from sqlbuild._native import aggregate_build_result +from sqlbuild._native import PlanOutput, RetentionPlanPhase, aggregate_build_result from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter from sqlbuild.adapter.relations.main.relation_metadata_cache_scope import ( relation_metadata_cache_scope, ) -from sqlbuild.compiler.planner.models import PlanOutput -from sqlbuild.compiler.planner.types import RetentionPlanPhase from sqlbuild.executor.auditing.main._project_results import ( AuditResultProjection, project_audit_result_batch, diff --git a/src/sqlbuild/executor/build/main/_external_source_loads.py b/src/sqlbuild/executor/build/main/_external_source_loads.py index eb81adbc5e..347ba7ff9d 100644 --- a/src/sqlbuild/executor/build/main/_external_source_loads.py +++ b/src/sqlbuild/executor/build/main/_external_source_loads.py @@ -2,10 +2,10 @@ from __future__ import annotations +from sqlbuild._native import PlanOutput from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter from sqlbuild.compiler.compile.models import CompiledObjectKey from sqlbuild.compiler.discovery.models import DiscoveredLoaderFunction -from sqlbuild.compiler.planner.models import PlanOutput from sqlbuild.executor.build._helpers.external_source_loads import ( run_external_source_loads_before_connections as _run_external_source_loads_before_connections, ) diff --git a/src/sqlbuild/executor/build/models.py b/src/sqlbuild/executor/build/models.py index 5586233fc3..e308b6fd9f 100644 --- a/src/sqlbuild/executor/build/models.py +++ b/src/sqlbuild/executor/build/models.py @@ -7,19 +7,19 @@ from datetime import datetime from pathlib import Path -from sqlbuild._native import SnapshotsConfig -from sqlbuild.adapter.contract.models import LifeCycleEvent, RelationInfo -from sqlbuild.adapter.contract.types import TablePromotionMode -from sqlbuild.compiler.compile.models import CompiledObjectKey, CompiledRelationLocation -from sqlbuild.compiler.discovery.models import DiscoveredLoaderFunction -from sqlbuild.compiler.planner.models import ( +from sqlbuild._native import ( AuditPlanEntry, FunctionPlanEntry, ModelPlanEntry, SeedPlanEntry, + SnapshotsConfig, SourceLoadPlanEntry, SqlTestPlanEntry, ) +from sqlbuild.adapter.contract.models import LifeCycleEvent, RelationInfo +from sqlbuild.adapter.contract.types import TablePromotionMode +from sqlbuild.compiler.compile.models import CompiledObjectKey, CompiledRelationLocation +from sqlbuild.compiler.discovery.models import DiscoveredLoaderFunction from sqlbuild.cost.models import StatementExecutionTelemetry from sqlbuild.executor.auditing.models import AuditExecutionResult from sqlbuild.executor.build.types import BeforeModelMaterializeCallback, BuildStatus diff --git a/src/sqlbuild/executor/build/types.py b/src/sqlbuild/executor/build/types.py index d5701202f5..b8075be76a 100644 --- a/src/sqlbuild/executor/build/types.py +++ b/src/sqlbuild/executor/build/types.py @@ -6,7 +6,7 @@ from typing import TYPE_CHECKING, Any, Protocol if TYPE_CHECKING: - from sqlbuild.compiler.planner.models import ModelPlanEntry + from sqlbuild._native import ModelPlanEntry from sqlbuild.executor.scheduling.types import ExecutionStatus as ExecutionStatus from sqlbuild.runtime.contracts.types import ExecutionResourceKind as ExecutionResourceKind diff --git a/src/sqlbuild/executor/clone/_helpers/execution_items.py b/src/sqlbuild/executor/clone/_helpers/execution_items.py index f4ab512551..47537fa20c 100644 --- a/src/sqlbuild/executor/clone/_helpers/execution_items.py +++ b/src/sqlbuild/executor/clone/_helpers/execution_items.py @@ -5,17 +5,17 @@ import time from dataclasses import replace -from sqlbuild.adapter.contract.classes.statement_recorder import StatementRecorder -from sqlbuild.adapter.contract.models import RelationLookup, RetentionRequest -from sqlbuild.adapter.contract.types import RetentionScope -from sqlbuild.compiler.compile.models import CompiledObjectKey -from sqlbuild.compiler.planner.models import ( +from sqlbuild._native import ( CloneSourcePlanEntry, FunctionPlanEntry, + MaterializationType, ModelPlanEntry, SeedPlanEntry, ) -from sqlbuild.compiler.planner.types import MaterializationType +from sqlbuild.adapter.contract.classes.statement_recorder import StatementRecorder +from sqlbuild.adapter.contract.models import RelationLookup, RetentionRequest +from sqlbuild.adapter.contract.types import RetentionScope +from sqlbuild.compiler.compile.models import CompiledObjectKey from sqlbuild.executor.build.models import FunctionExecutionResult from sqlbuild.executor.clone._helpers.operations import clone_relation, recreate_view from sqlbuild.executor.clone._helpers.retention import apply_clone_retention diff --git a/src/sqlbuild/executor/clone/_helpers/fingerprinting.py b/src/sqlbuild/executor/clone/_helpers/fingerprinting.py index c890a87ab9..278329ffbb 100644 --- a/src/sqlbuild/executor/clone/_helpers/fingerprinting.py +++ b/src/sqlbuild/executor/clone/_helpers/fingerprinting.py @@ -11,6 +11,9 @@ from sqlbuild._native import ( Fingerprint, FingerprintSet, + MaterializationType, + ModelPlanEntry, + SeedPlanEntry, read_latest_fingerprints, write_fingerprints, ) @@ -22,8 +25,6 @@ NODE_TYPE_MODEL, NODE_TYPE_SEED, ) -from sqlbuild.compiler.planner.models import ModelPlanEntry, SeedPlanEntry -from sqlbuild.compiler.planner.types import MaterializationType from sqlbuild.executor.clone.models import CloneExecutionResult from sqlbuild.executor.clone.types import ( CloneAction, diff --git a/src/sqlbuild/executor/clone/_helpers/inspection.py b/src/sqlbuild/executor/clone/_helpers/inspection.py index 51ba6aebcf..69aae99c1a 100644 --- a/src/sqlbuild/executor/clone/_helpers/inspection.py +++ b/src/sqlbuild/executor/clone/_helpers/inspection.py @@ -2,9 +2,9 @@ from __future__ import annotations +from sqlbuild._native import CloneSourcePlanEntry, ModelPlanEntry, SeedPlanEntry from sqlbuild.adapter.contract.models import RelationLookup from sqlbuild.adapter.relations.main.relation_lookup import build_relation_lookup -from sqlbuild.compiler.planner.models import CloneSourcePlanEntry, ModelPlanEntry, SeedPlanEntry from sqlbuild.executor.clone.models import CloneExecutionInput from sqlbuild.runtime.observability.classes.operation_lifecycle import OperationLifecycle diff --git a/src/sqlbuild/executor/clone/_helpers/lifecycle.py b/src/sqlbuild/executor/clone/_helpers/lifecycle.py index 8cb75027d1..acbc6f5c87 100644 --- a/src/sqlbuild/executor/clone/_helpers/lifecycle.py +++ b/src/sqlbuild/executor/clone/_helpers/lifecycle.py @@ -4,15 +4,15 @@ from typing import Any -from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter -from sqlbuild.adapter.contract.classes.statement_recorder import StatementRecorder -from sqlbuild.compiler.planner.models import ( +from sqlbuild._native import ( CloneSourcePlanEntry, FunctionPlanEntry, ModelPlanEntry, + RetentionPlanPhase, SeedPlanEntry, ) -from sqlbuild.compiler.planner.types import RetentionPlanPhase +from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter +from sqlbuild.adapter.contract.classes.statement_recorder import StatementRecorder from sqlbuild.executor.clone._helpers.retention import apply_clone_namespace_retention_phase from sqlbuild.executor.clone.models import ( CloneExecutionInput, diff --git a/src/sqlbuild/executor/clone/_helpers/operations.py b/src/sqlbuild/executor/clone/_helpers/operations.py index 0c88295635..c9fa054a7d 100644 --- a/src/sqlbuild/executor/clone/_helpers/operations.py +++ b/src/sqlbuild/executor/clone/_helpers/operations.py @@ -4,12 +4,12 @@ from typing import Any +from sqlbuild._native import CloneSourcePlanEntry, ModelPlanEntry, SeedPlanEntry from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter from sqlbuild.adapter.contract.models import RelationLookup from sqlbuild.adapter.relations.main.resolve_relation_location_qualified_name import ( resolve_relation_location_qualified_name, ) -from sqlbuild.compiler.planner.models import CloneSourcePlanEntry, ModelPlanEntry, SeedPlanEntry from sqlbuild.executor.clone.main._clone_relation_operation import clone_relation_by_names from sqlbuild.executor.clone.main._recreate_view_operation import recreate_view_by_names from sqlbuild.executor.clone.models import CloneItemResult diff --git a/src/sqlbuild/executor/clone/_helpers/retention.py b/src/sqlbuild/executor/clone/_helpers/retention.py index d3a6a3fb34..6324e91208 100644 --- a/src/sqlbuild/executor/clone/_helpers/retention.py +++ b/src/sqlbuild/executor/clone/_helpers/retention.py @@ -4,7 +4,7 @@ from typing import Any -from sqlbuild._native import ResolvedTimeTravelRetention +from sqlbuild._native import ResolvedTimeTravelRetention, RetentionPlanPhase from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter from sqlbuild.adapter.contract.models import ( RenderedRetentionChange, @@ -13,7 +13,6 @@ ) from sqlbuild.adapter.contract.types import BuiltinAdapter, RetentionScope from sqlbuild.compiler.compile.types import CompiledProjectView -from sqlbuild.compiler.planner.types import RetentionPlanPhase from sqlbuild.errors.contracts.exceptions import ExecutorInputError from sqlbuild.runtime.observability.classes.operation_lifecycle import ( OperationAttributes, diff --git a/src/sqlbuild/executor/clone/main/_fingerprinting.py b/src/sqlbuild/executor/clone/main/_fingerprinting.py index 8de777f8f9..1c7acb0be2 100644 --- a/src/sqlbuild/executor/clone/main/_fingerprinting.py +++ b/src/sqlbuild/executor/clone/main/_fingerprinting.py @@ -4,8 +4,8 @@ from typing import Any +from sqlbuild._native import ModelPlanEntry, SeedPlanEntry from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter -from sqlbuild.compiler.planner.models import ModelPlanEntry, SeedPlanEntry from sqlbuild.executor.clone._helpers.fingerprinting import copy_clone_fingerprints as _copy from sqlbuild.executor.clone.models import CloneExecutionResult from sqlbuild.executor.clone.types import CloneFingerprintProgressCallback diff --git a/src/sqlbuild/executor/clone/main/execute.py b/src/sqlbuild/executor/clone/main/execute.py index f796b50209..af25a28b55 100644 --- a/src/sqlbuild/executor/clone/main/execute.py +++ b/src/sqlbuild/executor/clone/main/execute.py @@ -2,14 +2,9 @@ from __future__ import annotations +from sqlbuild._native import CloneSourcePlanEntry, FunctionPlanEntry, ModelPlanEntry, SeedPlanEntry from sqlbuild.adapter.contract.models import RelationLookup from sqlbuild.compiler.compile.models import CompiledObjectKey -from sqlbuild.compiler.planner.models import ( - CloneSourcePlanEntry, - FunctionPlanEntry, - ModelPlanEntry, - SeedPlanEntry, -) from sqlbuild.executor.clone._helpers.canonical_progress import clone_item_enrichment from sqlbuild.executor.clone._helpers.execution_items import ( execute_clone_destination_item, diff --git a/src/sqlbuild/executor/clone/models.py b/src/sqlbuild/executor/clone/models.py index 2dc71e87b7..19ffdebe91 100644 --- a/src/sqlbuild/executor/clone/models.py +++ b/src/sqlbuild/executor/clone/models.py @@ -5,15 +5,10 @@ from dataclasses import dataclass, field from typing import Any +from sqlbuild._native import CloneSourcePlanEntry, FunctionPlanEntry, ModelPlanEntry, SeedPlanEntry from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter from sqlbuild.adapter.contract.models import LifeCycleEvent, RetentionRequest from sqlbuild.compiler.compile.models import CompiledObjectKey, CompiledRelationLocation -from sqlbuild.compiler.planner.models import ( - CloneSourcePlanEntry, - FunctionPlanEntry, - ModelPlanEntry, - SeedPlanEntry, -) from sqlbuild.executor.clone.types import CloneAction, CloneItemCallback, CloneStatus diff --git a/src/sqlbuild/executor/custom/models.py b/src/sqlbuild/executor/custom/models.py index 8b6111a2ed..993b5caf43 100644 --- a/src/sqlbuild/executor/custom/models.py +++ b/src/sqlbuild/executor/custom/models.py @@ -7,10 +7,10 @@ from dataclasses import dataclass, field from typing import Any +from sqlbuild._native import SchemaFinding from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter from sqlbuild.adapter.contract.classes.statement_recorder import StatementRecorder from sqlbuild.adapter.contract.models import ColumnInfo, RelationInfo -from sqlbuild.compiler.planner.models import SchemaFinding from sqlbuild.executor.auditing.models import AuditExecutionResult from sqlbuild.executor.custom.main._execute_sql import execute_sql_with_recording from sqlbuild.executor.custom.main._qualify_relation import qualify_custom_relation diff --git a/src/sqlbuild/executor/diff/_helpers/bounds.py b/src/sqlbuild/executor/diff/_helpers/bounds.py index 50fd17346b..426af6a8af 100644 --- a/src/sqlbuild/executor/diff/_helpers/bounds.py +++ b/src/sqlbuild/executor/diff/_helpers/bounds.py @@ -5,10 +5,9 @@ from datetime import UTC, datetime from typing import Any -from sqlbuild._native import get_config_str +from sqlbuild._native import Duration, get_config_str from sqlbuild.adapter.contract.models import CursorValue from sqlbuild.adapter.contract.types import CursorKind -from sqlbuild.compiler.planner.models import Duration from sqlbuild.errors.contracts.exceptions import ExecutorInputError diff --git a/src/sqlbuild/executor/functions/main/_execute.py b/src/sqlbuild/executor/functions/main/_execute.py index 2061b580c4..ed4d2c305f 100644 --- a/src/sqlbuild/executor/functions/main/_execute.py +++ b/src/sqlbuild/executor/functions/main/_execute.py @@ -8,6 +8,7 @@ from sqlbuild._native import ( Fingerprint, + FunctionPlanEntry, compute_function_definition_hash, error_code, error_help, @@ -19,7 +20,6 @@ from sqlbuild.adapter.contract.classes.statement_recorder import StatementRecorder from sqlbuild.adapter.contract.models import FunctionDefinition from sqlbuild.compiler.compile.types import FunctionLanguage -from sqlbuild.compiler.planner.models import FunctionPlanEntry from sqlbuild.errors.contracts.exceptions import ExecutorInputError from sqlbuild.executor.build.models import FunctionExecutionResult from sqlbuild.executor.functions.constants import ( diff --git a/src/sqlbuild/executor/janitor/models.py b/src/sqlbuild/executor/janitor/models.py index 965584d612..1eb83014a3 100644 --- a/src/sqlbuild/executor/janitor/models.py +++ b/src/sqlbuild/executor/janitor/models.py @@ -5,9 +5,8 @@ from dataclasses import dataclass, field from datetime import datetime +from sqlbuild._native import OldNameViewDropReason, OldNameViewHistory from sqlbuild.adapter.contract.models import RelationInfo, RelationLookup -from sqlbuild.compiler.migrations.models import OldNameViewHistory -from sqlbuild.compiler.migrations.types import OldNameViewDropReason @dataclass(frozen=True, eq=False) diff --git a/src/sqlbuild/executor/load/_helpers/schema.py b/src/sqlbuild/executor/load/_helpers/schema.py index 40b3b2842b..9b7c93205b 100644 --- a/src/sqlbuild/executor/load/_helpers/schema.py +++ b/src/sqlbuild/executor/load/_helpers/schema.py @@ -4,12 +4,11 @@ from typing import Any -from sqlbuild._native import SourceEntry +from sqlbuild._native import ContractPolicy, SourceEntry from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter from sqlbuild.adapter.contract.classes.statement_recorder import StatementRecorder from sqlbuild.adapter.contract.models import ColumnInfo from sqlbuild.adapter.type_system.main.types_equal import types_equal -from sqlbuild.compiler.planner.types import ContractPolicy from sqlbuild.errors.contracts.exceptions import ExecutorInputError diff --git a/src/sqlbuild/executor/load/_helpers/staging.py b/src/sqlbuild/executor/load/_helpers/staging.py index 6e566cb5c4..351b1a2db2 100644 --- a/src/sqlbuild/executor/load/_helpers/staging.py +++ b/src/sqlbuild/executor/load/_helpers/staging.py @@ -4,14 +4,13 @@ from typing import Any -from sqlbuild._native import SourceEntry, fit_auxiliary_relation_name +from sqlbuild._native import ContractPolicy, SourceEntry, fit_auxiliary_relation_name from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter from sqlbuild.adapter.contract.classes.statement_recorder import StatementRecorder from sqlbuild.adapter.contract.types import LoaderLogicalType from sqlbuild.adapter.relations.main.resolve_qualified_name_parts import ( resolve_qualified_name_parts, ) -from sqlbuild.compiler.planner.types import ContractPolicy from sqlbuild.executor.load._helpers.rows import ( build_rows_sql, iter_loader_row_batches, diff --git a/src/sqlbuild/executor/migrations/_helpers/columns.py b/src/sqlbuild/executor/migrations/_helpers/columns.py index 8c62013c05..57284c5fb2 100644 --- a/src/sqlbuild/executor/migrations/_helpers/columns.py +++ b/src/sqlbuild/executor/migrations/_helpers/columns.py @@ -5,14 +5,18 @@ from datetime import UTC, datetime from typing import Any -from sqlbuild._native import deterministic_column_migration_event_id, write_migration_state_event +from sqlbuild._native import ( + ColumnMigrationDecision, + ColumnMigrationEvent, + ColumnMigrationPlanEntry, + MigrationRelation, + deterministic_column_migration_event_id, + write_migration_state_event, +) from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter from sqlbuild.adapter.contract.classes.statement_recorder import StatementRecorder from sqlbuild.adapter.contract.models import ColumnInfo from sqlbuild.compiler.migrations.constants import MIGRATION_WRITE_ATTEMPTS -from sqlbuild.compiler.migrations.models import ColumnMigrationEvent, MigrationRelation -from sqlbuild.compiler.migrations.types import ColumnMigrationDecision -from sqlbuild.compiler.planner.models import ColumnMigrationPlanEntry from sqlbuild.executor.migrations._helpers.relation_for_location import ( migration_relation_for_location, ) diff --git a/src/sqlbuild/executor/migrations/_helpers/naming.py b/src/sqlbuild/executor/migrations/_helpers/naming.py index 03d29b933c..86056211fa 100644 --- a/src/sqlbuild/executor/migrations/_helpers/naming.py +++ b/src/sqlbuild/executor/migrations/_helpers/naming.py @@ -12,7 +12,6 @@ from sqlbuild.adapter.relations.main.resolve_qualified_name_parts import ( resolve_qualified_name_parts, ) -from sqlbuild.compiler.planner.models import ModelMigrationPlanEntry from sqlbuild.errors.contracts.exceptions import ExecutorInputError from sqlbuild.executor.migrations.constants import ( MIGRATION_NAME_ATTEMPTS, diff --git a/src/sqlbuild/executor/migrations/_helpers/old_name_facts.py b/src/sqlbuild/executor/migrations/_helpers/old_name_facts.py index b05e84d7d1..12186bec4d 100644 --- a/src/sqlbuild/executor/migrations/_helpers/old_name_facts.py +++ b/src/sqlbuild/executor/migrations/_helpers/old_name_facts.py @@ -6,6 +6,10 @@ from typing import Any from sqlbuild._native import ( + MigrationRelation, + OldNameViewEvent, + OldNameViewEventType, + OldNameViewPlanEntry, deterministic_old_name_view_event_id, read_migration_state_events, stored_old_name_view_columns, @@ -13,9 +17,6 @@ ) from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter from sqlbuild.compiler.migrations.constants import MIGRATION_WRITE_ATTEMPTS -from sqlbuild.compiler.migrations.models import MigrationRelation, OldNameViewEvent -from sqlbuild.compiler.migrations.types import OldNameViewEventType -from sqlbuild.compiler.planner.models import OldNameViewPlanEntry from sqlbuild.executor.migrations._helpers.relation_for_location import ( migration_relation_for_location, ) diff --git a/src/sqlbuild/executor/migrations/_helpers/old_name_views.py b/src/sqlbuild/executor/migrations/_helpers/old_name_views.py index 304f590be0..34de345f4a 100644 --- a/src/sqlbuild/executor/migrations/_helpers/old_name_views.py +++ b/src/sqlbuild/executor/migrations/_helpers/old_name_views.py @@ -24,13 +24,6 @@ from sqlbuild.adapter.type_system.main.normalize_relation_type import normalize_relation_type from sqlbuild.compiler.compile.models import CompiledRelationLocation from sqlbuild.compiler.migrations.constants import COLUMN_MIGRATION_TABLE_NAME -from sqlbuild.compiler.migrations.models import ( - ColumnMigrationEvent, - MigrationRelation, - OldNameViewEvent, -) -from sqlbuild.compiler.migrations.types import OldNameViewEventType -from sqlbuild.compiler.planner.models import OldNameView, OldNameViewPlanEntry, PlanOutput from sqlbuild.errors.contracts.exceptions import ExecutorInputError from sqlbuild.executor.migrations._helpers.old_name_facts import ( old_name_fact, diff --git a/src/sqlbuild/executor/migrations/_helpers/promotion.py b/src/sqlbuild/executor/migrations/_helpers/promotion.py index 9e3dac3889..033bdc4c82 100644 --- a/src/sqlbuild/executor/migrations/_helpers/promotion.py +++ b/src/sqlbuild/executor/migrations/_helpers/promotion.py @@ -5,13 +5,20 @@ from datetime import UTC, datetime from typing import Any -from sqlbuild._native import deterministic_migration_event_id, write_migration_state_event +from sqlbuild._native import ( + MigrationDecision, + MigrationEvent, + MigrationRelation, + ModelMigrationPlanEntry, + OldNameViewEvent, + OldNameViewEventType, + OldNameViewPlanEntry, + deterministic_migration_event_id, + write_migration_state_event, +) from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter from sqlbuild.adapter.contract.classes.statement_recorder import StatementRecorder from sqlbuild.compiler.migrations.constants import MIGRATION_WRITE_ATTEMPTS -from sqlbuild.compiler.migrations.models import MigrationEvent, MigrationRelation, OldNameViewEvent -from sqlbuild.compiler.migrations.types import MigrationDecision, OldNameViewEventType -from sqlbuild.compiler.planner.models import ModelMigrationPlanEntry, OldNameViewPlanEntry from sqlbuild.executor.migrations._helpers.old_name_facts import ( old_name_fact, old_name_state_table_sql, diff --git a/src/sqlbuild/executor/migrations/_helpers/relation_for_location.py b/src/sqlbuild/executor/migrations/_helpers/relation_for_location.py index b298a89606..4131202980 100644 --- a/src/sqlbuild/executor/migrations/_helpers/relation_for_location.py +++ b/src/sqlbuild/executor/migrations/_helpers/relation_for_location.py @@ -2,8 +2,8 @@ from __future__ import annotations +from sqlbuild._native import MigrationRelation from sqlbuild.compiler.compile.models import CompiledRelationLocation -from sqlbuild.compiler.migrations.models import MigrationRelation def migration_relation_for_location(location: CompiledRelationLocation) -> MigrationRelation: diff --git a/src/sqlbuild/executor/migrations/_helpers/staging.py b/src/sqlbuild/executor/migrations/_helpers/staging.py index b954e24ad0..5ed55d54af 100644 --- a/src/sqlbuild/executor/migrations/_helpers/staging.py +++ b/src/sqlbuild/executor/migrations/_helpers/staging.py @@ -5,10 +5,10 @@ import logging from typing import Any +from sqlbuild._native import ModelMigrationPlanEntry from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter from sqlbuild.adapter.contract.models import ColumnInfo, MigrationStagePlan, RelationInfo from sqlbuild.adapter.contract.types import MigrationTransfer -from sqlbuild.compiler.planner.models import ModelMigrationPlanEntry from sqlbuild.errors.contracts.exceptions import ExecutorInputError from sqlbuild.executor.migrations.models import MigrationArtifactNames diff --git a/src/sqlbuild/executor/migrations/main/_apply.py b/src/sqlbuild/executor/migrations/main/_apply.py index 7c4fe28546..c58a46598d 100644 --- a/src/sqlbuild/executor/migrations/main/_apply.py +++ b/src/sqlbuild/executor/migrations/main/_apply.py @@ -7,15 +7,15 @@ from datetime import UTC, datetime from typing import Any -from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter -from sqlbuild.adapter.contract.classes.statement_recorder import StatementRecorder -from sqlbuild.adapter.contract.types import MigrationTransfer -from sqlbuild.compiler.migrations.models import MigrationEvent -from sqlbuild.compiler.planner.models import ( +from sqlbuild._native import ( + MigrationEvent, ModelMigrationPlanEntry, OldNameViewPlanEntry, PlanOutput, ) +from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter +from sqlbuild.adapter.contract.classes.statement_recorder import StatementRecorder +from sqlbuild.adapter.contract.types import MigrationTransfer from sqlbuild.errors.contracts.exceptions import ExecutorInputError from sqlbuild.executor.migrations._helpers.naming import resolve_artifact_names from sqlbuild.executor.migrations._helpers.promotion import ( diff --git a/src/sqlbuild/executor/migrations/main/_apply_columns.py b/src/sqlbuild/executor/migrations/main/_apply_columns.py index 4229d79fc9..a9d6254b88 100644 --- a/src/sqlbuild/executor/migrations/main/_apply_columns.py +++ b/src/sqlbuild/executor/migrations/main/_apply_columns.py @@ -6,8 +6,8 @@ from collections.abc import Callable from typing import Any +from sqlbuild._native import ColumnMigrationPlanEntry, PlanOutput from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter -from sqlbuild.compiler.planner.models import ColumnMigrationPlanEntry, PlanOutput from sqlbuild.errors.contracts.exceptions import ExecutorInputError from sqlbuild.executor.migrations._helpers.columns import apply_model_column_renames diff --git a/src/sqlbuild/executor/migrations/main/_bound_old_name_view_guard.py b/src/sqlbuild/executor/migrations/main/_bound_old_name_view_guard.py index 820a3897f6..fe4f97cec8 100644 --- a/src/sqlbuild/executor/migrations/main/_bound_old_name_view_guard.py +++ b/src/sqlbuild/executor/migrations/main/_bound_old_name_view_guard.py @@ -6,9 +6,9 @@ from functools import partial from typing import Any +from sqlbuild._native import PlanOutput from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter from sqlbuild.compiler.compile.models import CompiledRelationLocation -from sqlbuild.compiler.planner.models import PlanOutput from sqlbuild.executor.migrations._helpers.old_name_views import views_reading from sqlbuild.executor.migrations.classes.bound_old_name_views import BoundOldNameViews from sqlbuild.executor.migrations.models import OldNameViewSource diff --git a/src/sqlbuild/executor/migrations/main/_compatibility_view_sql.py b/src/sqlbuild/executor/migrations/main/_compatibility_view_sql.py index c16178ce37..3d2aecb8ba 100644 --- a/src/sqlbuild/executor/migrations/main/_compatibility_view_sql.py +++ b/src/sqlbuild/executor/migrations/main/_compatibility_view_sql.py @@ -4,9 +4,9 @@ from typing import Any +from sqlbuild._native import MigrationRelation, OldNameViewHistory from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter from sqlbuild.compiler.compile.models import CompiledRelationLocation -from sqlbuild.compiler.migrations.models import MigrationRelation, OldNameViewHistory from sqlbuild.executor.migrations._helpers.old_name_views import render_old_name_view_select from sqlbuild.executor.migrations.models import OldNameViewSource diff --git a/src/sqlbuild/executor/migrations/main/_complete_old_name_views.py b/src/sqlbuild/executor/migrations/main/_complete_old_name_views.py index 3457c71a0b..63ce40982b 100644 --- a/src/sqlbuild/executor/migrations/main/_complete_old_name_views.py +++ b/src/sqlbuild/executor/migrations/main/_complete_old_name_views.py @@ -4,9 +4,9 @@ from typing import Any +from sqlbuild._native import PlanOutput from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter from sqlbuild.compiler.compile.models import CompiledRelationLocation -from sqlbuild.compiler.planner.models import PlanOutput from sqlbuild.executor.migrations._helpers.old_name_views import ( rebind_old_name_views_atomically, run_planned_old_name_steps, diff --git a/src/sqlbuild/executor/old_name_views/_helpers/janitor.py b/src/sqlbuild/executor/old_name_views/_helpers/janitor.py index 299d04eb72..e9251027d0 100644 --- a/src/sqlbuild/executor/old_name_views/_helpers/janitor.py +++ b/src/sqlbuild/executor/old_name_views/_helpers/janitor.py @@ -6,6 +6,11 @@ from typing import Any from sqlbuild._native import ( + OldNameViewDropReason, + OldNameViewEvent, + OldNameViewEventType, + OldNameViewHistory, + OldNameViewStatus, deterministic_old_name_view_event_id, write_migration_state_event, ) diff --git a/src/sqlbuild/executor/pipeline/_helpers/auditing.py b/src/sqlbuild/executor/pipeline/_helpers/auditing.py index 9cc0dbe756..91696eb327 100644 --- a/src/sqlbuild/executor/pipeline/_helpers/auditing.py +++ b/src/sqlbuild/executor/pipeline/_helpers/auditing.py @@ -15,10 +15,9 @@ from typing import Any, cast from uuid import uuid4 -from sqlbuild._native import CostContext, audit_resource_id +from sqlbuild._native import AuditPlanEntry, CostContext, PlanOutput, audit_resource_id from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter from sqlbuild.compiler.auditing.types import AuditOutcome, AuditRunScope -from sqlbuild.compiler.planner.models import AuditPlanEntry, PlanOutput from sqlbuild.diagnostics.classes.diagnostic_record_redactor import DiagnosticRecordRedactor from sqlbuild.diagnostics.main.diagnostics_context import diagnostics_context from sqlbuild.executor.auditing.main._execute import execute_audit diff --git a/src/sqlbuild/executor/pipeline/_helpers/connections.py b/src/sqlbuild/executor/pipeline/_helpers/connections.py index adcbb3d091..e960bd874e 100644 --- a/src/sqlbuild/executor/pipeline/_helpers/connections.py +++ b/src/sqlbuild/executor/pipeline/_helpers/connections.py @@ -8,9 +8,8 @@ from contextvars import copy_context from typing import Any -from sqlbuild._native import BuildPhaseTimingTracker +from sqlbuild._native import BuildPhaseTimingTracker, PlanOutput from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter -from sqlbuild.compiler.planner.models import PlanOutput from sqlbuild.executor.build.models import BuildCallbacks from sqlbuild.executor.pipeline._helpers.schema_preparation import prepare_build_schemas from sqlbuild.executor.pipeline.models import BuildConnectionPreparation diff --git a/src/sqlbuild/executor/pipeline/_helpers/graph_width.py b/src/sqlbuild/executor/pipeline/_helpers/graph_width.py index 887e67dec7..53c776211f 100644 --- a/src/sqlbuild/executor/pipeline/_helpers/graph_width.py +++ b/src/sqlbuild/executor/pipeline/_helpers/graph_width.py @@ -2,7 +2,7 @@ from __future__ import annotations -from sqlbuild.compiler.planner.models import PlanOutput +from sqlbuild._native import PlanOutput def runnable_graph_width(*, plan: PlanOutput) -> int: diff --git a/src/sqlbuild/executor/pipeline/_helpers/scenario.py b/src/sqlbuild/executor/pipeline/_helpers/scenario.py index 9141520410..cf20fcf2f4 100644 --- a/src/sqlbuild/executor/pipeline/_helpers/scenario.py +++ b/src/sqlbuild/executor/pipeline/_helpers/scenario.py @@ -17,6 +17,7 @@ from sqlbuild._native import ( NativeProgressProjector, + ScenarioExecutionPlan, SqlLexicalSyntax, current_native_progress_projector, error_code, @@ -29,7 +30,6 @@ from sqlbuild.adapter.contract.types import TablePromotionMode from sqlbuild.compiler.compile.models import CompiledSqlScenario from sqlbuild.compiler.pipeline.models import CompilePipelineResult -from sqlbuild.compiler.planner.models import ScenarioExecutionPlan from sqlbuild.diagnostics.main.log_debug_event import log_debug_event from sqlbuild.executor.pipeline._helpers.connections import ( close_connections, diff --git a/src/sqlbuild/executor/pipeline/_helpers/schema_preparation.py b/src/sqlbuild/executor/pipeline/_helpers/schema_preparation.py index d8ac9b14b9..9110443534 100644 --- a/src/sqlbuild/executor/pipeline/_helpers/schema_preparation.py +++ b/src/sqlbuild/executor/pipeline/_helpers/schema_preparation.py @@ -4,10 +4,9 @@ from typing import Any -from sqlbuild._native import SourceEntry +from sqlbuild._native import PlanOutput, SourceEntry from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter from sqlbuild.adapter.contract.classes.statement_recorder import StatementRecorder -from sqlbuild.compiler.planner.models import PlanOutput def prepare_build_schemas( diff --git a/src/sqlbuild/executor/pipeline/_helpers/seeding.py b/src/sqlbuild/executor/pipeline/_helpers/seeding.py index 0808a3a4cc..e608c77529 100644 --- a/src/sqlbuild/executor/pipeline/_helpers/seeding.py +++ b/src/sqlbuild/executor/pipeline/_helpers/seeding.py @@ -8,9 +8,9 @@ from contextvars import copy_context from typing import Any, cast +from sqlbuild._native import PlanOutput, SeedPlanEntry from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter from sqlbuild.adapter.contract.classes.statement_recorder import StatementRecorder -from sqlbuild.compiler.planner.models import PlanOutput, SeedPlanEntry from sqlbuild.executor.build.models import SeedExecutionResult from sqlbuild.executor.scheduling.types import ExecutionStatus from sqlbuild.executor.seed.main._execute import execute_seed diff --git a/src/sqlbuild/executor/pipeline/_helpers/testing.py b/src/sqlbuild/executor/pipeline/_helpers/testing.py index 81577014c0..3bdb0bf93e 100644 --- a/src/sqlbuild/executor/pipeline/_helpers/testing.py +++ b/src/sqlbuild/executor/pipeline/_helpers/testing.py @@ -12,7 +12,7 @@ from typing import Any, cast from uuid import uuid4 -from sqlbuild._native import sql_test_resource_id +from sqlbuild._native import FunctionPlanEntry, PlanOutput, SqlTestPlanEntry, sql_test_resource_id from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter from sqlbuild.adapter.contract.classes.statement_recorder import StatementRecorder from sqlbuild.adapter.contract.models import FunctionInfo @@ -21,7 +21,6 @@ CompiledRelationLocation, ) from sqlbuild.compiler.compile.types import FunctionLanguage -from sqlbuild.compiler.planner.models import FunctionPlanEntry, PlanOutput, SqlTestPlanEntry from sqlbuild.executor.build.models import FunctionExecutionResult from sqlbuild.executor.functions.main._execute import execute_function from sqlbuild.executor.pipeline._helpers.connections import ( diff --git a/src/sqlbuild/executor/pipeline/main/run.py b/src/sqlbuild/executor/pipeline/main/run.py index 37c3a1fb96..eba798e2f5 100644 --- a/src/sqlbuild/executor/pipeline/main/run.py +++ b/src/sqlbuild/executor/pipeline/main/run.py @@ -6,9 +6,8 @@ from dataclasses import replace from typing import Any -from sqlbuild._native import BuildPhaseTimingTracker, CostContext, SettingsConfig +from sqlbuild._native import BuildPhaseTimingTracker, CostContext, PlanOutput, SettingsConfig from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter -from sqlbuild.compiler.planner.models import PlanOutput from sqlbuild.diagnostics.main.diagnostics_context import diagnostics_context from sqlbuild.executor.build.main._execute import execute_build_plan from sqlbuild.executor.build.main._external_source_loads import ( diff --git a/src/sqlbuild/executor/pipeline/models.py b/src/sqlbuild/executor/pipeline/models.py index 6f5b5f519f..9190a261b0 100644 --- a/src/sqlbuild/executor/pipeline/models.py +++ b/src/sqlbuild/executor/pipeline/models.py @@ -6,7 +6,7 @@ from dataclasses import dataclass from typing import Any -from sqlbuild.compiler.planner.models import AuditPlanEntry, SqlTestPlanEntry +from sqlbuild._native import AuditPlanEntry, SqlTestPlanEntry from sqlbuild.executor.auditing.models import AuditExecutionResult from sqlbuild.executor.build.models import ( BuildCallbacks, diff --git a/src/sqlbuild/executor/run/_helpers/execution/final_audits.py b/src/sqlbuild/executor/run/_helpers/execution/final_audits.py index 8393fc33fe..1c9e0dc1ed 100644 --- a/src/sqlbuild/executor/run/_helpers/execution/final_audits.py +++ b/src/sqlbuild/executor/run/_helpers/execution/final_audits.py @@ -4,11 +4,10 @@ from typing import Any -from sqlbuild._native import Fingerprint, SourceEntry +from sqlbuild._native import AuditPlanEntry, Fingerprint, SourceEntry from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter from sqlbuild.compiler.auditing.types import AuditOutcome, AuditRunScope from sqlbuild.compiler.compile.models import CompiledRelationLocation -from sqlbuild.compiler.planner.models import AuditPlanEntry from sqlbuild.executor.auditing.main._execute import execute_audit from sqlbuild.executor.auditing.models import AuditExecutionResult from sqlbuild.executor.run._helpers.reuse.audit import ( diff --git a/src/sqlbuild/executor/run/_helpers/execution/hook_phases.py b/src/sqlbuild/executor/run/_helpers/execution/hook_phases.py index ae1d8ae8a2..6841a6b8f4 100644 --- a/src/sqlbuild/executor/run/_helpers/execution/hook_phases.py +++ b/src/sqlbuild/executor/run/_helpers/execution/hook_phases.py @@ -4,8 +4,8 @@ from collections.abc import Mapping +from sqlbuild._native import ModelPlanEntry from sqlbuild.adapter.contract.classes.statement_recorder import StatementRecorder -from sqlbuild.compiler.planner.models import ModelPlanEntry from sqlbuild.diagnostics.main.diagnostics_context import diagnostics_context from sqlbuild.executor.auditing.models import AuditExecutionResult from sqlbuild.executor.run._helpers.execution.hooks import execute_hooks, render_hooks diff --git a/src/sqlbuild/executor/run/_helpers/execution/results.py b/src/sqlbuild/executor/run/_helpers/execution/results.py index 502bb15301..44c120fa46 100644 --- a/src/sqlbuild/executor/run/_helpers/execution/results.py +++ b/src/sqlbuild/executor/run/_helpers/execution/results.py @@ -4,14 +4,17 @@ from dataclasses import replace -from sqlbuild._native import error_code, error_help, error_message, format_waiting_on_empty_inputs -from sqlbuild.adapter.contract.classes.statement_recorder import StatementRecorder -from sqlbuild.compiler.planner.models import ( +from sqlbuild._native import ( CursorBounds, FutureCursorSafetyEvidence, MaximumStartSafetyEvidence, ModelPlanEntry, + error_code, + error_help, + error_message, + format_waiting_on_empty_inputs, ) +from sqlbuild.adapter.contract.classes.statement_recorder import StatementRecorder from sqlbuild.executor.auditing.models import AuditExecutionResult from sqlbuild.executor.run.constants import ( RUN_AUDIT_FAILED_CODE, diff --git a/src/sqlbuild/executor/run/_helpers/execution/table_targets.py b/src/sqlbuild/executor/run/_helpers/execution/table_targets.py index f93a559bb0..4885ea39c1 100644 --- a/src/sqlbuild/executor/run/_helpers/execution/table_targets.py +++ b/src/sqlbuild/executor/run/_helpers/execution/table_targets.py @@ -2,7 +2,7 @@ from __future__ import annotations -from sqlbuild._native import fit_auxiliary_relation_name +from sqlbuild._native import ModelPlanEntry, fit_auxiliary_relation_name from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter from sqlbuild.adapter.relations.main.resolve_qualified_name_parts import ( resolve_qualified_name_parts, @@ -10,7 +10,6 @@ from sqlbuild.adapter.relations.main.resolve_relation_location_qualified_name import ( resolve_relation_location_qualified_name, ) -from sqlbuild.compiler.planner.models import ModelPlanEntry from sqlbuild.executor.run.constants import STAGING_RELATION_SUFFIX from sqlbuild.executor.run.models import TableTargets diff --git a/src/sqlbuild/executor/run/_helpers/materializations/custom.py b/src/sqlbuild/executor/run/_helpers/materializations/custom.py index 1e59e4f1e8..bebc9efea5 100644 --- a/src/sqlbuild/executor/run/_helpers/materializations/custom.py +++ b/src/sqlbuild/executor/run/_helpers/materializations/custom.py @@ -7,7 +7,7 @@ from dataclasses import replace from typing import Any, cast -from sqlbuild._native import SourceEntry +from sqlbuild._native import AuditPlanEntry, ModelPlanEntry, PlanReason, SourceEntry from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter from sqlbuild.adapter.contract.classes.statement_recorder import StatementRecorder from sqlbuild.adapter.contract.models import ColumnInfo, RelationInfo @@ -16,8 +16,6 @@ ) from sqlbuild.compiler.auditing.types import AuditRunScope from sqlbuild.compiler.compile.models import CompiledRelationLocation -from sqlbuild.compiler.planner.models import AuditPlanEntry, ModelPlanEntry -from sqlbuild.compiler.planner.types import PlanReason from sqlbuild.diagnostics.main.diagnostics_context import diagnostics_context from sqlbuild.errors.contracts.exceptions import ExecutorInputError from sqlbuild.executor.auditing.main._execute import execute_audit diff --git a/src/sqlbuild/executor/run/_helpers/materializations/incremental.py b/src/sqlbuild/executor/run/_helpers/materializations/incremental.py index 7b0bbd7b04..e0c222de4f 100644 --- a/src/sqlbuild/executor/run/_helpers/materializations/incremental.py +++ b/src/sqlbuild/executor/run/_helpers/materializations/incremental.py @@ -6,6 +6,10 @@ from typing import Any from sqlbuild._native import ( + CursorBounds, + IncrementalStrategy, + ModelPlanEntry, + OnSchemaChange, TableType, fit_auxiliary_relation_name, future_cursor_cap_warning, @@ -22,8 +26,6 @@ resolve_relation_location_qualified_name, ) from sqlbuild.adapter.type_system.main.types_equal import types_equal -from sqlbuild.compiler.planner.models import CursorBounds, ModelPlanEntry -from sqlbuild.compiler.planner.types import IncrementalStrategy, OnSchemaChange from sqlbuild.diagnostics.main.diagnostics_context import diagnostics_context from sqlbuild.errors.contracts.exceptions import ExecutorInputError from sqlbuild.executor.auditing.models import AuditExecutionResult diff --git a/src/sqlbuild/executor/run/_helpers/materializations/microbatch.py b/src/sqlbuild/executor/run/_helpers/materializations/microbatch.py index 1201893013..b0b733a938 100644 --- a/src/sqlbuild/executor/run/_helpers/materializations/microbatch.py +++ b/src/sqlbuild/executor/run/_helpers/materializations/microbatch.py @@ -13,9 +13,18 @@ from sqlbuild._native import ( AlignedInterval, + BackfillAction, + CursorBounds, + CursorGrain, + CursorType, DateValue, + Duration, + IncrementalStrategy, IntegerValue, MicrobatchLimitAction, + MicrobatchStrategy, + ModelPlanEntry, + OnSchemaChange, TableType, TimestampValue, aligned_timestamp_bounds, @@ -52,19 +61,6 @@ from sqlbuild.adapter.relations.main.resolve_relation_location_qualified_name import ( resolve_relation_location_qualified_name, ) -from sqlbuild.compiler.planner.models import ( - CursorBounds, - Duration, - ModelPlanEntry, -) -from sqlbuild.compiler.planner.types import ( - BackfillAction, - CursorGrain, - CursorType, - IncrementalStrategy, - MicrobatchStrategy, - OnSchemaChange, -) from sqlbuild.cursor_algebra.types import Bound, BoundSentinel, CursorScalar from sqlbuild.diagnostics.main.diagnostics_context import diagnostics_context from sqlbuild.diagnostics.main.log_debug_event import log_debug_event diff --git a/src/sqlbuild/executor/run/_helpers/materializations/snapshot.py b/src/sqlbuild/executor/run/_helpers/materializations/snapshot.py index f2ca9f2636..97b93b887a 100644 --- a/src/sqlbuild/executor/run/_helpers/materializations/snapshot.py +++ b/src/sqlbuild/executor/run/_helpers/materializations/snapshot.py @@ -4,7 +4,16 @@ from typing import Any -from sqlbuild._native import SnapshotsConfig, fit_auxiliary_relation_name +from sqlbuild._native import ( + HistoricalInput, + InitialValidFrom, + ModelPlanEntry, + PlanReason, + SnapshotSchemaChangePolicy, + SnapshotsConfig, + SnapshotStrategy, + fit_auxiliary_relation_name, +) from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter from sqlbuild.adapter.contract.classes.statement_recorder import StatementRecorder from sqlbuild.adapter.contract.models import ( @@ -26,14 +35,6 @@ SNAPSHOT_DEFAULT_VALID_FROM_COLUMN, SNAPSHOT_DEFAULT_VALID_TO_COLUMN, ) -from sqlbuild.compiler.planner.models import ModelPlanEntry -from sqlbuild.compiler.planner.types import ( - HistoricalInput, - InitialValidFrom, - PlanReason, - SnapshotSchemaChangePolicy, - SnapshotStrategy, -) from sqlbuild.diagnostics.main.diagnostics_context import diagnostics_context from sqlbuild.errors.contracts.exceptions import ExecutorInputError from sqlbuild.executor.auditing.models import AuditExecutionResult diff --git a/src/sqlbuild/executor/run/_helpers/materializations/table_cursor.py b/src/sqlbuild/executor/run/_helpers/materializations/table_cursor.py index d8eb5b41a5..9aca5f50e1 100644 --- a/src/sqlbuild/executor/run/_helpers/materializations/table_cursor.py +++ b/src/sqlbuild/executor/run/_helpers/materializations/table_cursor.py @@ -5,12 +5,13 @@ from dataclasses import replace from sqlbuild._native import ( + CursorBounds, + ModelPlanEntry, empty_cursor_window, future_cursor_cap_warning, replaces_incremental_relation, ) from sqlbuild.adapter.relations.main.cached_relation_exists import cached_relation_exists -from sqlbuild.compiler.planner.models import CursorBounds, ModelPlanEntry from sqlbuild.errors.contracts.exceptions import ExecutorInputError from sqlbuild.executor.run._helpers.validation.cursor_bounds import ( build_runtime_cursor_spec, diff --git a/src/sqlbuild/executor/run/_helpers/materializations/view.py b/src/sqlbuild/executor/run/_helpers/materializations/view.py index 695d7b0691..5e44371070 100644 --- a/src/sqlbuild/executor/run/_helpers/materializations/view.py +++ b/src/sqlbuild/executor/run/_helpers/materializations/view.py @@ -5,7 +5,7 @@ from collections.abc import Mapping from typing import Any -from sqlbuild._native import SourceEntry +from sqlbuild._native import AuditPlanEntry, ModelPlanEntry, SourceEntry from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter from sqlbuild.adapter.contract.classes.statement_recorder import StatementRecorder from sqlbuild.adapter.relations.main.resolve_relation_location_qualified_name import ( @@ -14,7 +14,6 @@ from sqlbuild.compiler.auditing.types import AuditOutcome, AuditRunScope from sqlbuild.compiler.compile.models import CompiledRelationLocation from sqlbuild.compiler.discovery.models import DiscoveredHookFunction -from sqlbuild.compiler.planner.models import AuditPlanEntry, ModelPlanEntry from sqlbuild.diagnostics.main.diagnostics_context import diagnostics_context from sqlbuild.executor.auditing.main._execute import execute_audit from sqlbuild.executor.auditing.models import AuditExecutionResult diff --git a/src/sqlbuild/executor/run/_helpers/reuse/audit.py b/src/sqlbuild/executor/run/_helpers/reuse/audit.py index 3f6659a223..5f5b0ee20d 100644 --- a/src/sqlbuild/executor/run/_helpers/reuse/audit.py +++ b/src/sqlbuild/executor/run/_helpers/reuse/audit.py @@ -2,9 +2,8 @@ from __future__ import annotations -from sqlbuild._native import build_audit_gate_identity +from sqlbuild._native import AuditPlanEntry, build_audit_gate_identity from sqlbuild.compiler.auditing.types import AuditOutcome, AuditRunScope, AuditSeverity -from sqlbuild.compiler.planner.models import AuditPlanEntry from sqlbuild.executor.auditing.models import AuditExecutionResult from sqlbuild.executor.run._helpers.reuse.fingerprint_metadata import ( reuse_from_audit_gate_reuse_decision, diff --git a/src/sqlbuild/executor/run/_helpers/reuse/fingerprint_metadata.py b/src/sqlbuild/executor/run/_helpers/reuse/fingerprint_metadata.py index 1f3a10a36c..66b600ba3a 100644 --- a/src/sqlbuild/executor/run/_helpers/reuse/fingerprint_metadata.py +++ b/src/sqlbuild/executor/run/_helpers/reuse/fingerprint_metadata.py @@ -7,11 +7,15 @@ from collections.abc import Mapping from typing import cast -from sqlbuild._native import AuditGateIdentity, AuditIdentity, build_audit_gate_identity +from sqlbuild._native import ( + AuditGateIdentity, + AuditIdentity, + AuditPlanEntry, + build_audit_gate_identity, +) from sqlbuild.compiler.auditing.types import AuditOutcome, AuditSeverity from sqlbuild.compiler.fingerprints.constants import AUDIT_GATE_METADATA_KEY from sqlbuild.compiler.planner.constants import MIGRATION_FINGERPRINT_METADATA_KEY -from sqlbuild.compiler.planner.models import AuditPlanEntry from sqlbuild.executor.auditing.models import AuditExecutionResult from sqlbuild.executor.run.models import ( AuditGateMetadata, diff --git a/src/sqlbuild/executor/run/_helpers/reuse/fingerprinting.py b/src/sqlbuild/executor/run/_helpers/reuse/fingerprinting.py index fa2713af38..7eec2be047 100644 --- a/src/sqlbuild/executor/run/_helpers/reuse/fingerprinting.py +++ b/src/sqlbuild/executor/run/_helpers/reuse/fingerprinting.py @@ -6,10 +6,15 @@ from datetime import UTC, datetime from typing import Any -from sqlbuild._native import Fingerprint, compute_query_hash, write_fingerprint +from sqlbuild._native import ( + AuditPlanEntry, + Fingerprint, + ModelPlanEntry, + compute_query_hash, + write_fingerprint, +) from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter from sqlbuild.compiler.fingerprints.constants import NODE_TYPE_MODEL -from sqlbuild.compiler.planner.models import AuditPlanEntry, ModelPlanEntry from sqlbuild.executor.auditing.models import AuditExecutionResult from sqlbuild.executor.run._helpers.reuse.fingerprint_metadata import ( model_fingerprint_metadata_with_audit_gate, diff --git a/src/sqlbuild/executor/run/_helpers/validation/contracts.py b/src/sqlbuild/executor/run/_helpers/validation/contracts.py index e18317ac5b..ae8fefe93b 100644 --- a/src/sqlbuild/executor/run/_helpers/validation/contracts.py +++ b/src/sqlbuild/executor/run/_helpers/validation/contracts.py @@ -2,11 +2,10 @@ from __future__ import annotations -from sqlbuild._native import SchemaDynamicColumnFamily, matching_dynamic_families +from sqlbuild._native import ModelPlanEntry, SchemaDynamicColumnFamily, matching_dynamic_families from sqlbuild.adapter.contract.models import ColumnInfo from sqlbuild.adapter.contract.types import TypeDialect from sqlbuild.adapter.type_system.main.types_equal import types_equal -from sqlbuild.compiler.planner.models import ModelPlanEntry from sqlbuild.errors.contracts.exceptions import ExecutorInputError from sqlbuild.executor.run.constants import ( RUNTIME_CONTRACT_EXTRA_COLUMN_CODE, diff --git a/src/sqlbuild/executor/run/_helpers/validation/cursor_bounds.py b/src/sqlbuild/executor/run/_helpers/validation/cursor_bounds.py index 135b18c79e..e0e68cbaa4 100644 --- a/src/sqlbuild/executor/run/_helpers/validation/cursor_bounds.py +++ b/src/sqlbuild/executor/run/_helpers/validation/cursor_bounds.py @@ -10,8 +10,20 @@ from sqlbuild._native import ( AlignedInterval, + BackfillAction, + CursorBounds, + CursorGrain, + CursorInputEvidence, + CursorInputRelation, + CursorType, DateValue, + Duration, + EmptyCursorInputDecision, IntegerValue, + MaximumStartPolicyInputs, + MicrobatchStrategy, + ModelCursorSnapshot, + ModelPlanEntry, StartCursorsConfig, TimestampValue, apply_maximum_start_policy, @@ -39,22 +51,6 @@ from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter from sqlbuild.adapter.relations.main.cached_relation_exists import cached_relation_exists from sqlbuild.compiler.planner.constants import MICROBATCH_END_SENTINEL, MICROBATCH_START_SENTINEL -from sqlbuild.compiler.planner.models import ( - CursorBounds, - CursorInputEvidence, - CursorInputRelation, - Duration, - EmptyCursorInputDecision, - MaximumStartPolicyInputs, - ModelCursorSnapshot, - ModelPlanEntry, -) -from sqlbuild.compiler.planner.types import ( - BackfillAction, - CursorGrain, - CursorType, - MicrobatchStrategy, -) from sqlbuild.cursor_algebra.constants import GRAIN_ORDER from sqlbuild.cursor_algebra.types import BoundSentinel, CursorScalar from sqlbuild.errors.contracts.exceptions import ExecutorInputError diff --git a/src/sqlbuild/executor/run/main/_execute.py b/src/sqlbuild/executor/run/main/_execute.py index a546af9a14..f7545193b5 100644 --- a/src/sqlbuild/executor/run/main/_execute.py +++ b/src/sqlbuild/executor/run/main/_execute.py @@ -5,7 +5,7 @@ from dataclasses import replace from typing import Any -from sqlbuild._native import Fingerprint, setting_help +from sqlbuild._native import Fingerprint, ModelPlanEntry, setting_help from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter from sqlbuild.adapter.contract.classes.statement_recorder import StatementRecorder from sqlbuild.adapter.contract.models import ColumnInfo @@ -16,7 +16,6 @@ STAGED_PROMOTION_MODE, TABLE_PROMOTION_MODE_SETTING_KEY, ) -from sqlbuild.compiler.planner.models import ModelPlanEntry from sqlbuild.diagnostics.main.diagnostics_context import diagnostics_context from sqlbuild.errors.contracts.exceptions import ExecutorInputError from sqlbuild.executor.auditing.models import AuditExecutionResult diff --git a/src/sqlbuild/executor/run/main/_table_targets.py b/src/sqlbuild/executor/run/main/_table_targets.py index 69539ca521..7981a8595f 100644 --- a/src/sqlbuild/executor/run/main/_table_targets.py +++ b/src/sqlbuild/executor/run/main/_table_targets.py @@ -2,8 +2,8 @@ from __future__ import annotations +from sqlbuild._native import ModelPlanEntry from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter -from sqlbuild.compiler.planner.models import ModelPlanEntry from sqlbuild.executor.run._helpers.execution.table_targets import ( resolve_table_targets as _resolve_table_targets, ) diff --git a/src/sqlbuild/executor/run/models.py b/src/sqlbuild/executor/run/models.py index 9ee0cf18f1..6079ae2a79 100644 --- a/src/sqlbuild/executor/run/models.py +++ b/src/sqlbuild/executor/run/models.py @@ -9,10 +9,18 @@ from typing import Any, cast from sqlbuild._native import ( + AuditPlanEntry, + CursorBounds, + CursorGrain, + CursorType, DateValue, + Duration, + FutureCursorSafetyEvidence, FutureCursorsConfig, IntegerValue, + MaximumStartSafetyEvidence, MicrobatchLimitAction, + ModelPlanEntry, SourceEntry, StartCursorsConfig, TimestampValue, @@ -24,15 +32,6 @@ from sqlbuild.adapter.contract.models import LifeCycleEvent, QueryResult from sqlbuild.compiler.compile.models import CompiledRelationLocation from sqlbuild.compiler.discovery.models import DiscoveredHookFunction, PythonHookEntry -from sqlbuild.compiler.planner.models import ( - AuditPlanEntry, - CursorBounds, - Duration, - FutureCursorSafetyEvidence, - MaximumStartSafetyEvidence, - ModelPlanEntry, -) -from sqlbuild.compiler.planner.types import CursorGrain, CursorType from sqlbuild.compiler.python_nodes.types import SkipMode from sqlbuild.cursor_algebra.exceptions import CursorAlgebraError from sqlbuild.cursor_algebra.types import BoundSentinel, CursorScalar diff --git a/src/sqlbuild/executor/scenario/_helpers/capture/core.py b/src/sqlbuild/executor/scenario/_helpers/capture/core.py index 7cd3e0e670..8208c2be38 100644 --- a/src/sqlbuild/executor/scenario/_helpers/capture/core.py +++ b/src/sqlbuild/executor/scenario/_helpers/capture/core.py @@ -5,8 +5,8 @@ from pathlib import Path from typing import Any +from sqlbuild._native import ScenarioExecutionPlan from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter -from sqlbuild.compiler.planner.models import ScenarioExecutionPlan from sqlbuild.executor.build.models import SeedExecutionResult from sqlbuild.executor.scenario._helpers.lifecycle.failures import first_failure_details from sqlbuild.executor.scenario._helpers.lifecycle.fixtures import ( diff --git a/src/sqlbuild/executor/scenario/_helpers/execution/interrupts.py b/src/sqlbuild/executor/scenario/_helpers/execution/interrupts.py index a51ea11a8f..81fec1cca5 100644 --- a/src/sqlbuild/executor/scenario/_helpers/execution/interrupts.py +++ b/src/sqlbuild/executor/scenario/_helpers/execution/interrupts.py @@ -4,8 +4,8 @@ from typing import Any +from sqlbuild._native import ScenarioExecutionPlan from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter -from sqlbuild.compiler.planner.models import ScenarioExecutionPlan from sqlbuild.executor.scenario.classes.scenario_interrupts import ScenarioInterrupts from sqlbuild.executor.scenario.main._cleanup import execute_scenario_cleanup diff --git a/src/sqlbuild/executor/scenario/_helpers/execution/model_execution.py b/src/sqlbuild/executor/scenario/_helpers/execution/model_execution.py index b4c113d910..08df6f107c 100644 --- a/src/sqlbuild/executor/scenario/_helpers/execution/model_execution.py +++ b/src/sqlbuild/executor/scenario/_helpers/execution/model_execution.py @@ -5,9 +5,9 @@ import threading from typing import Any +from sqlbuild._native import ModelPlanEntry, ScenarioExecutionPlan from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter from sqlbuild.adapter.contract.types import TablePromotionMode -from sqlbuild.compiler.planner.models import ModelPlanEntry, ScenarioExecutionPlan from sqlbuild.executor.run.models import ModelExecutionResult from sqlbuild.executor.scenario.exceptions import ScenarioStopRequested from sqlbuild.executor.scenario.main._execute import execute_scenario_model diff --git a/src/sqlbuild/executor/scenario/_helpers/execution/run.py b/src/sqlbuild/executor/scenario/_helpers/execution/run.py index 46012a9668..4eb7161627 100644 --- a/src/sqlbuild/executor/scenario/_helpers/execution/run.py +++ b/src/sqlbuild/executor/scenario/_helpers/execution/run.py @@ -4,8 +4,8 @@ from typing import Any +from sqlbuild._native import ScenarioExecutionPlan, ScenarioRelationMap from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter -from sqlbuild.compiler.planner.models import ScenarioExecutionPlan, ScenarioRelationMap from sqlbuild.executor.build.models import SeedExecutionResult from sqlbuild.executor.run.models import ModelExecutionResult from sqlbuild.executor.scenario._helpers.execution.interrupts import ( diff --git a/src/sqlbuild/executor/scenario/_helpers/lifecycle/cleanup.py b/src/sqlbuild/executor/scenario/_helpers/lifecycle/cleanup.py index 6575d7cb45..c80973fa69 100644 --- a/src/sqlbuild/executor/scenario/_helpers/lifecycle/cleanup.py +++ b/src/sqlbuild/executor/scenario/_helpers/lifecycle/cleanup.py @@ -7,6 +7,14 @@ from dataclasses import replace from typing import Any +from sqlbuild._native import ( + MaterializationType, + ModelPlanEntry, + ScenarioArtifactKind, + ScenarioExecutionPlan, + ScenarioFixturePlan, + SeedPlanEntry, +) from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter from sqlbuild.adapter.contract.models import RelationInfo, RelationLookup from sqlbuild.adapter.contract.types import RelationType @@ -18,13 +26,6 @@ ) from sqlbuild.adapter.type_system.main.normalize_relation_type import normalize_relation_type from sqlbuild.compiler.compile.models import CompiledRelationLocation -from sqlbuild.compiler.planner.models import ( - ModelPlanEntry, - ScenarioExecutionPlan, - ScenarioFixturePlan, - SeedPlanEntry, -) -from sqlbuild.compiler.planner.types import MaterializationType, ScenarioArtifactKind from sqlbuild.diagnostics.main.log_debug_event import log_debug_event from sqlbuild.executor.run.main._table_targets import resolve_table_targets from sqlbuild.executor.run.main._type_enforcement_relation_name import ( diff --git a/src/sqlbuild/executor/scenario/_helpers/lifecycle/expectations.py b/src/sqlbuild/executor/scenario/_helpers/lifecycle/expectations.py index 3b9b7cb9d4..74997a5385 100644 --- a/src/sqlbuild/executor/scenario/_helpers/lifecycle/expectations.py +++ b/src/sqlbuild/executor/scenario/_helpers/lifecycle/expectations.py @@ -4,12 +4,12 @@ from typing import Any -from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter -from sqlbuild.compiler.planner.models import ( +from sqlbuild._native import ( ScenarioAssertionExpectationPlan, ScenarioExecutionPlan, ScenarioExpectedExpectationPlan, ) +from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter from sqlbuild.executor.scenario.main._assertion_expectations import ( execute_scenario_assertion_expectation, ) diff --git a/src/sqlbuild/executor/scenario/_helpers/lifecycle/fixtures.py b/src/sqlbuild/executor/scenario/_helpers/lifecycle/fixtures.py index 7d90a5b52e..4ceed41ace 100644 --- a/src/sqlbuild/executor/scenario/_helpers/lifecycle/fixtures.py +++ b/src/sqlbuild/executor/scenario/_helpers/lifecycle/fixtures.py @@ -5,9 +5,9 @@ from dataclasses import replace from typing import Any +from sqlbuild._native import ScenarioFixturePlan, SeedPlanEntry from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter from sqlbuild.adapter.contract.classes.statement_recorder import StatementRecorder -from sqlbuild.compiler.planner.models import ScenarioFixturePlan, SeedPlanEntry from sqlbuild.executor.build.models import SeedExecutionResult from sqlbuild.executor.scenario.constants import SCENARIO_EXEC_SEED_FAILED from sqlbuild.executor.scenario.main._fixtures import execute_scenario_fixture diff --git a/src/sqlbuild/executor/scenario/_helpers/local/execution.py b/src/sqlbuild/executor/scenario/_helpers/local/execution.py index 7c8c4c7501..4d4b5d8e0e 100644 --- a/src/sqlbuild/executor/scenario/_helpers/local/execution.py +++ b/src/sqlbuild/executor/scenario/_helpers/local/execution.py @@ -6,21 +6,24 @@ from pathlib import Path from typing import Any -from sqlbuild._native import SourceEntry, error_code, error_help, error_message -from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter -from sqlbuild.adapter.contract.classes.statement_recorder import StatementRecorder -from sqlbuild.adapter.contract.types import TablePromotionMode -from sqlbuild.compiler.compile.models import CompiledRelationLocation -from sqlbuild.compiler.compile.types import FunctionLanguage -from sqlbuild.compiler.planner.models import ( +from sqlbuild._native import ( FunctionPlanEntry, ModelPlanEntry, + ScenarioArtifactKind, ScenarioAssertionExpectationPlan, ScenarioExecutionPlan, ScenarioExpectedExpectationPlan, ScenarioRelationPlan, + SourceEntry, + error_code, + error_help, + error_message, ) -from sqlbuild.compiler.planner.types import ScenarioArtifactKind +from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter +from sqlbuild.adapter.contract.classes.statement_recorder import StatementRecorder +from sqlbuild.adapter.contract.types import TablePromotionMode +from sqlbuild.compiler.compile.models import CompiledRelationLocation +from sqlbuild.compiler.compile.types import FunctionLanguage from sqlbuild.executor.build.models import FunctionExecutionResult from sqlbuild.executor.functions.main._execute import execute_function from sqlbuild.executor.run.models import ModelExecutionResult diff --git a/src/sqlbuild/executor/scenario/_helpers/local/snapshots.py b/src/sqlbuild/executor/scenario/_helpers/local/snapshots.py index e24624c68a..313c83e736 100644 --- a/src/sqlbuild/executor/scenario/_helpers/local/snapshots.py +++ b/src/sqlbuild/executor/scenario/_helpers/local/snapshots.py @@ -5,8 +5,7 @@ from pathlib import Path from typing import Any -from sqlbuild._native import error_code, error_message -from sqlbuild.compiler.planner.types import ScenarioArtifactKind +from sqlbuild._native import ScenarioArtifactKind, error_code, error_message from sqlbuild.errors.contracts.exceptions import ExecutorInputError from sqlbuild.executor.scenario._helpers.snapshots.core import ( is_scenario_snapshot_fresh, diff --git a/src/sqlbuild/executor/scenario/_helpers/snapshots/core.py b/src/sqlbuild/executor/scenario/_helpers/snapshots/core.py index 78e7096b66..5f0fa53810 100644 --- a/src/sqlbuild/executor/scenario/_helpers/snapshots/core.py +++ b/src/sqlbuild/executor/scenario/_helpers/snapshots/core.py @@ -11,13 +11,14 @@ from typing import Any from uuid import UUID -from sqlbuild._native import error_code, error_message -from sqlbuild.compiler.planner.models import ( +from sqlbuild._native import ( + ScenarioArtifactKind, ScenarioExecutionPlan, ScenarioFixturePlan, SeedPlanEntry, + error_code, + error_message, ) -from sqlbuild.compiler.planner.types import ScenarioArtifactKind from sqlbuild.errors.contracts.exceptions import ExecutorInputError from sqlbuild.executor.scenario.constants import ( JSONL_BLANK_LINE, diff --git a/src/sqlbuild/executor/scenario/classes/prepared_scenario_schemas.py b/src/sqlbuild/executor/scenario/classes/prepared_scenario_schemas.py index d8e316f66b..da5df38d9b 100644 --- a/src/sqlbuild/executor/scenario/classes/prepared_scenario_schemas.py +++ b/src/sqlbuild/executor/scenario/classes/prepared_scenario_schemas.py @@ -6,10 +6,10 @@ import threading from typing import Any +from sqlbuild._native import ScenarioExecutionPlan from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter from sqlbuild.adapter.contract.classes.statement_recorder import StatementRecorder from sqlbuild.compiler.compile.models import CompiledRelationLocation -from sqlbuild.compiler.planner.models import ScenarioExecutionPlan from sqlbuild.diagnostics.main.log_debug_event import log_debug_event _DEBUG_LOGGER: logging.Logger = logging.getLogger("sqlbuild.execution") diff --git a/src/sqlbuild/executor/scenario/classes/running_scenarios.py b/src/sqlbuild/executor/scenario/classes/running_scenarios.py index 174fbe19ff..08f48fb20e 100644 --- a/src/sqlbuild/executor/scenario/classes/running_scenarios.py +++ b/src/sqlbuild/executor/scenario/classes/running_scenarios.py @@ -6,8 +6,8 @@ import threading from typing import Any +from sqlbuild._native import ScenarioExecutionPlan from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter -from sqlbuild.compiler.planner.models import ScenarioExecutionPlan from sqlbuild.diagnostics.main.log_debug_event import log_debug_event _DEBUG_LOGGER: logging.Logger = logging.getLogger("sqlbuild.execution") diff --git a/src/sqlbuild/executor/scenario/main/_assertion_expectations.py b/src/sqlbuild/executor/scenario/main/_assertion_expectations.py index 9b1a1a17ca..c328feb06a 100644 --- a/src/sqlbuild/executor/scenario/main/_assertion_expectations.py +++ b/src/sqlbuild/executor/scenario/main/_assertion_expectations.py @@ -4,8 +4,8 @@ from typing import Any +from sqlbuild._native import ScenarioAssertionExpectationPlan from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter -from sqlbuild.compiler.planner.models import ScenarioAssertionExpectationPlan from sqlbuild.executor.scenario.constants import ( SCENARIO_EXEC_ASSERTION_ERRORED, SCENARIO_EXEC_ASSERTION_FAILED, diff --git a/src/sqlbuild/executor/scenario/main/_capture_steps.py b/src/sqlbuild/executor/scenario/main/_capture_steps.py index 1054cee4b6..0149702f29 100644 --- a/src/sqlbuild/executor/scenario/main/_capture_steps.py +++ b/src/sqlbuild/executor/scenario/main/_capture_steps.py @@ -5,8 +5,8 @@ from pathlib import Path from typing import Any +from sqlbuild._native import ScenarioExecutionPlan from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter -from sqlbuild.compiler.planner.models import ScenarioExecutionPlan from sqlbuild.executor.scenario._helpers.capture.core import execute_scenario_snapshot_capture_steps from sqlbuild.executor.scenario.models import ( ScenarioCaptureSettings, diff --git a/src/sqlbuild/executor/scenario/main/_cleanup.py b/src/sqlbuild/executor/scenario/main/_cleanup.py index 78e1b5f68d..291cd7ea0d 100644 --- a/src/sqlbuild/executor/scenario/main/_cleanup.py +++ b/src/sqlbuild/executor/scenario/main/_cleanup.py @@ -4,10 +4,9 @@ from typing import Any +from sqlbuild._native import MaterializationType, ScenarioExecutionPlan from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter from sqlbuild.adapter.contract.classes.statement_recorder import StatementRecorder -from sqlbuild.compiler.planner.models import ScenarioExecutionPlan -from sqlbuild.compiler.planner.types import MaterializationType from sqlbuild.diagnostics.main.diagnostics_context import diagnostics_context from sqlbuild.executor.scenario._helpers.lifecycle.cleanup import ( collect_scenario_cleanup_targets, diff --git a/src/sqlbuild/executor/scenario/main/_cleanup_interrupted.py b/src/sqlbuild/executor/scenario/main/_cleanup_interrupted.py index 75ad2e6a55..da119bd1a6 100644 --- a/src/sqlbuild/executor/scenario/main/_cleanup_interrupted.py +++ b/src/sqlbuild/executor/scenario/main/_cleanup_interrupted.py @@ -4,8 +4,8 @@ from typing import Any +from sqlbuild._native import ScenarioExecutionPlan from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter -from sqlbuild.compiler.planner.models import ScenarioExecutionPlan from sqlbuild.executor.scenario._helpers.execution.interrupts import ( cleanup_interrupted_scenarios as _cleanup_interrupted_scenarios, ) diff --git a/src/sqlbuild/executor/scenario/main/_execute.py b/src/sqlbuild/executor/scenario/main/_execute.py index f629db08d1..e3e0d90ce6 100644 --- a/src/sqlbuild/executor/scenario/main/_execute.py +++ b/src/sqlbuild/executor/scenario/main/_execute.py @@ -5,10 +5,9 @@ from dataclasses import replace from typing import Any +from sqlbuild._native import MaterializationType, ModelPlanEntry, PlanAction, ScenarioExecutionPlan from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter from sqlbuild.adapter.contract.types import TablePromotionMode -from sqlbuild.compiler.planner.models import ModelPlanEntry, ScenarioExecutionPlan -from sqlbuild.compiler.planner.types import MaterializationType, PlanAction from sqlbuild.executor.run.main._execute import execute_table_entry, execute_view_entry from sqlbuild.executor.run.models import ModelExecutionResult, ModelMaterializationContext from sqlbuild.executor.scenario.constants import SCENARIO_EXEC_MODEL_FAILED diff --git a/src/sqlbuild/executor/scenario/main/_expected_expectations.py b/src/sqlbuild/executor/scenario/main/_expected_expectations.py index d848eb1477..933d59cc38 100644 --- a/src/sqlbuild/executor/scenario/main/_expected_expectations.py +++ b/src/sqlbuild/executor/scenario/main/_expected_expectations.py @@ -4,12 +4,11 @@ from typing import Any -from sqlbuild._native import build_scenario_expected_comparison_sql +from sqlbuild._native import ScenarioExpectedExpectationPlan, build_scenario_expected_comparison_sql from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter from sqlbuild.adapter.relations.main.resolve_relation_location_qualified_name import ( resolve_relation_location_qualified_name, ) -from sqlbuild.compiler.planner.models import ScenarioExpectedExpectationPlan from sqlbuild.executor.scenario.constants import ( SCENARIO_EXEC_EXPECTED_ERRORED, SCENARIO_EXEC_EXPECTED_FAILED, diff --git a/src/sqlbuild/executor/scenario/main/_fixtures.py b/src/sqlbuild/executor/scenario/main/_fixtures.py index 5fb4ec4969..7a03b0915e 100644 --- a/src/sqlbuild/executor/scenario/main/_fixtures.py +++ b/src/sqlbuild/executor/scenario/main/_fixtures.py @@ -4,12 +4,12 @@ from typing import Any +from sqlbuild._native import ScenarioFixturePlan from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter from sqlbuild.adapter.contract.classes.statement_recorder import StatementRecorder from sqlbuild.adapter.relations.main.resolve_relation_location_qualified_name import ( resolve_relation_location_qualified_name, ) -from sqlbuild.compiler.planner.models import ScenarioFixturePlan from sqlbuild.diagnostics.main.diagnostics_context import diagnostics_context from sqlbuild.executor.scenario.constants import SCENARIO_EXEC_FIXTURE_FAILED from sqlbuild.executor.scenario.models import ScenarioFixtureExecutionResult diff --git a/src/sqlbuild/executor/scenario/main/_local.py b/src/sqlbuild/executor/scenario/main/_local.py index ffd7242821..388ce36cbf 100644 --- a/src/sqlbuild/executor/scenario/main/_local.py +++ b/src/sqlbuild/executor/scenario/main/_local.py @@ -4,9 +4,9 @@ from pathlib import Path +from sqlbuild._native import ScenarioExecutionPlan from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter from sqlbuild.adapter.contract.types import TablePromotionMode -from sqlbuild.compiler.planner.models import ScenarioExecutionPlan from sqlbuild.executor.scenario._helpers.local.execution import ( execute_local_scenario_load_only_run as _execute_local_scenario_load_only_run, ) diff --git a/src/sqlbuild/executor/scenario/main/_run.py b/src/sqlbuild/executor/scenario/main/_run.py index 16e6f533ee..05a4f5faca 100644 --- a/src/sqlbuild/executor/scenario/main/_run.py +++ b/src/sqlbuild/executor/scenario/main/_run.py @@ -4,8 +4,8 @@ from typing import Any +from sqlbuild._native import ScenarioExecutionPlan from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter -from sqlbuild.compiler.planner.models import ScenarioExecutionPlan from sqlbuild.executor.scenario._helpers.execution.run import execute_scenario_run_steps from sqlbuild.executor.scenario.models import ScenarioRunOptions, ScenarioRunResult diff --git a/src/sqlbuild/executor/scenario/main/_snapshots.py b/src/sqlbuild/executor/scenario/main/_snapshots.py index e8f1975e65..980693ca12 100644 --- a/src/sqlbuild/executor/scenario/main/_snapshots.py +++ b/src/sqlbuild/executor/scenario/main/_snapshots.py @@ -4,7 +4,7 @@ from pathlib import Path -from sqlbuild.compiler.planner.models import ScenarioExecutionPlan +from sqlbuild._native import ScenarioExecutionPlan from sqlbuild.executor.scenario._helpers.snapshots.core import ( classify_scenario_snapshot_state as _classify_scenario_snapshot_state, ) diff --git a/src/sqlbuild/executor/scenario/models.py b/src/sqlbuild/executor/scenario/models.py index 4f147ac77f..46019d1e31 100644 --- a/src/sqlbuild/executor/scenario/models.py +++ b/src/sqlbuild/executor/scenario/models.py @@ -5,15 +5,16 @@ from dataclasses import dataclass, field from pathlib import Path -from sqlbuild._native import SqlLexicalSyntax -from sqlbuild.adapter.contract.models import LifeCycleEvent -from sqlbuild.adapter.contract.types import TablePromotionMode -from sqlbuild.compiler.planner.models import ( - CompiledRelationLocation, +from sqlbuild._native import ( + MaterializationType, + ScenarioArtifactKind, ScenarioExecutionPlan, ScenarioRelationMap, + SqlLexicalSyntax, ) -from sqlbuild.compiler.planner.types import MaterializationType, ScenarioArtifactKind +from sqlbuild.adapter.contract.models import LifeCycleEvent +from sqlbuild.adapter.contract.types import TablePromotionMode +from sqlbuild.compiler.compile.models import CompiledRelationLocation from sqlbuild.executor.build.models import FunctionExecutionResult, SeedExecutionResult from sqlbuild.executor.run.models import ModelExecutionResult from sqlbuild.executor.scenario.classes.prepared_scenario_schemas import PreparedScenarioSchemas diff --git a/src/sqlbuild/executor/seed/_helpers/fingerprinting.py b/src/sqlbuild/executor/seed/_helpers/fingerprinting.py index c72250718b..0172752a18 100644 --- a/src/sqlbuild/executor/seed/_helpers/fingerprinting.py +++ b/src/sqlbuild/executor/seed/_helpers/fingerprinting.py @@ -6,10 +6,9 @@ from datetime import UTC, datetime from typing import Any -from sqlbuild._native import Fingerprint, write_fingerprint +from sqlbuild._native import Fingerprint, SeedPlanEntry, write_fingerprint from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter from sqlbuild.compiler.fingerprints.constants import NODE_TYPE_SEED -from sqlbuild.compiler.planner.models import SeedPlanEntry def try_write_seed_fingerprint( diff --git a/src/sqlbuild/executor/seed/main/_execute.py b/src/sqlbuild/executor/seed/main/_execute.py index e159b62808..b4ddb026cd 100644 --- a/src/sqlbuild/executor/seed/main/_execute.py +++ b/src/sqlbuild/executor/seed/main/_execute.py @@ -5,12 +5,12 @@ import time from typing import Any +from sqlbuild._native import SeedPlanEntry from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter from sqlbuild.adapter.contract.classes.statement_recorder import StatementRecorder from sqlbuild.adapter.relations.main.resolve_relation_location_qualified_name import ( resolve_relation_location_qualified_name, ) -from sqlbuild.compiler.planner.models import SeedPlanEntry from sqlbuild.executor.build.models import SeedExecutionResult from sqlbuild.executor.scheduling.types import ExecutionStatus from sqlbuild.executor.seed._helpers.fingerprinting import try_write_seed_fingerprint diff --git a/src/sqlbuild/executor/testing/_helpers/difference_samples.py b/src/sqlbuild/executor/testing/_helpers/difference_samples.py index 70bb3a2911..9d6f02f246 100644 --- a/src/sqlbuild/executor/testing/_helpers/difference_samples.py +++ b/src/sqlbuild/executor/testing/_helpers/difference_samples.py @@ -6,9 +6,8 @@ from dataclasses import replace from typing import Any, Final -from sqlbuild._native import render_sql_test_difference_sample_sql +from sqlbuild._native import ChainStep, SqlTestPlanEntry, render_sql_test_difference_sample_sql from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter -from sqlbuild.compiler.planner.models import ChainStep, SqlTestPlanEntry from sqlbuild.diagnostics.classes.diagnostic_record_redactor import DiagnosticRecordRedactor from sqlbuild.diagnostics.main.log_debug_event import log_debug_event from sqlbuild.executor.testing.models import ( diff --git a/src/sqlbuild/executor/testing/_helpers/expected_column_probe.py b/src/sqlbuild/executor/testing/_helpers/expected_column_probe.py index c82b524ce9..27df21d025 100644 --- a/src/sqlbuild/executor/testing/_helpers/expected_column_probe.py +++ b/src/sqlbuild/executor/testing/_helpers/expected_column_probe.py @@ -5,9 +5,12 @@ import logging from typing import Any -from sqlbuild._native import describe_missing_expected_columns, render_sql_test_comparison_sql +from sqlbuild._native import ( + SqlTestPlanEntry, + describe_missing_expected_columns, + render_sql_test_comparison_sql, +) from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter -from sqlbuild.compiler.planner.models import SqlTestPlanEntry from sqlbuild.diagnostics.classes.diagnostic_record_redactor import DiagnosticRecordRedactor from sqlbuild.diagnostics.main.log_debug_event import log_debug_event diff --git a/src/sqlbuild/executor/testing/main/_execute.py b/src/sqlbuild/executor/testing/main/_execute.py index f0c31495c7..69951e0ae0 100644 --- a/src/sqlbuild/executor/testing/main/_execute.py +++ b/src/sqlbuild/executor/testing/main/_execute.py @@ -4,10 +4,9 @@ from typing import Any -from sqlbuild._native import render_sql_test_comparison_sql +from sqlbuild._native import SqlTestPlanEntry, render_sql_test_comparison_sql from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter from sqlbuild.compiler.compile.exceptions import CompileInputError -from sqlbuild.compiler.planner.models import SqlTestPlanEntry from sqlbuild.executor.testing._helpers.difference_samples import add_difference_samples from sqlbuild.executor.testing._helpers.expected_column_probe import ( missing_expected_columns_message, diff --git a/src/sqlbuild/integrations/rivers/constants.py b/src/sqlbuild/integrations/rivers/constants.py index d2f5ad3451..cf147dd8fd 100644 --- a/src/sqlbuild/integrations/rivers/constants.py +++ b/src/sqlbuild/integrations/rivers/constants.py @@ -2,8 +2,8 @@ from __future__ import annotations +from sqlbuild._native import MaterializationType from sqlbuild.compiler.dag.types import NodeKind -from sqlbuild.compiler.planner.types import MaterializationType RIVERS_DEPLOYMENT_ENVIRONMENT_VARIABLE: str = "RIVERS_DEPLOYMENT" RIVERS_DEVELOPMENT_DEPLOYMENT: str = "dev" diff --git a/src/sqlbuild/microbatches/classes/direct_store.py b/src/sqlbuild/microbatches/classes/direct_store.py index 40393c4fa7..e29016878d 100644 --- a/src/sqlbuild/microbatches/classes/direct_store.py +++ b/src/sqlbuild/microbatches/classes/direct_store.py @@ -6,9 +6,9 @@ from dataclasses import replace from typing import Any +from sqlbuild._native import ModelPlanEntry from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter from sqlbuild.compiler.compile.models import CompiledRelationLocation -from sqlbuild.compiler.planner.models import ModelPlanEntry from sqlbuild.microbatches._helpers.sql import ( build_existing_event_ids_sql, build_insert_many_sql, diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/scenario/helpers.py b/tests/e2e/src/sqlbuild/cli/commands/main/scenario/helpers.py index d3e3787872..7706e484ee 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/scenario/helpers.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/scenario/helpers.py @@ -14,10 +14,10 @@ from pathlib import Path from typing import IO, NamedTuple, cast +from sqlbuild._native import ScenarioArtifactKind from sqlbuild.compiler.frontier.constants import ( COMPILE_REUSE_DISABLE_ENV_VAR as REUSE_DISABLE_ENV_VAR, ) -from sqlbuild.compiler.planner.types import ScenarioArtifactKind from sqlbuild.executor.scenario._helpers.snapshots.core import ( build_scenario_snapshot_input_fingerprint, ) diff --git a/tests/integration/src/sqlbuild/adapters/snowflake/test_table_type_conversion.py b/tests/integration/src/sqlbuild/adapters/snowflake/test_table_type_conversion.py index 9b609c35f3..b5b9186ed8 100644 --- a/tests/integration/src/sqlbuild/adapters/snowflake/test_table_type_conversion.py +++ b/tests/integration/src/sqlbuild/adapters/snowflake/test_table_type_conversion.py @@ -6,10 +6,9 @@ import pytest -from sqlbuild._native import TableType, table_type_copy_name +from sqlbuild._native import TableType, TableTypePlanEntry, table_type_copy_name from sqlbuild.adapter.contract.models import RelationInfo from sqlbuild.compiler.compile.models import CompiledRelationLocation -from sqlbuild.compiler.planner.models import TableTypePlanEntry from sqlbuild.executor.build._helpers.retention import apply_table_type_conversion from sqlbuild.executor.clone.main._clone_relation_operation import clone_relation_by_names from tests.integration.src.sqlbuild.adapters.snowflake._test_types import ( diff --git a/tests/integration/src/sqlbuild/cli/commands/main/_test_types.py b/tests/integration/src/sqlbuild/cli/commands/main/_test_types.py index 088cb42294..8b95a482f2 100644 --- a/tests/integration/src/sqlbuild/cli/commands/main/_test_types.py +++ b/tests/integration/src/sqlbuild/cli/commands/main/_test_types.py @@ -4,7 +4,7 @@ from dataclasses import dataclass, field from pathlib import Path -from sqlbuild.compiler.planner.models import CursorOverrides +from sqlbuild._native import CursorOverrides @dataclass(frozen=True) diff --git a/tests/integration/src/sqlbuild/cli/commands/main/helpers.py b/tests/integration/src/sqlbuild/cli/commands/main/helpers.py index cfc9777725..09311ab352 100644 --- a/tests/integration/src/sqlbuild/cli/commands/main/helpers.py +++ b/tests/integration/src/sqlbuild/cli/commands/main/helpers.py @@ -21,6 +21,7 @@ CustomHostObserver, NativeQueryColumns, NativeSqlTestArtifact, + NativeSqlTestPlanningError, PlannedStaticSqlTests, PreparedCompileArtifacts, ) @@ -33,7 +34,6 @@ ) from sqlbuild.adapter.contract.types import RetentionChangePhase from sqlbuild.adapters.duckdb.classes.duckdb_adapter import DuckDbAdapter -from sqlbuild.compiler.planner.exceptions import NativeSqlTestPlanningError from tests.integration.src.sqlbuild.cli.commands.main._test_types import ( RepeatedJsonParseTestCase, ) diff --git a/tests/integration/src/sqlbuild/cli/commands/main/old_name_views/helpers.py b/tests/integration/src/sqlbuild/cli/commands/main/old_name_views/helpers.py index 0b236fa5de..76a1faa8c7 100644 --- a/tests/integration/src/sqlbuild/cli/commands/main/old_name_views/helpers.py +++ b/tests/integration/src/sqlbuild/cli/commands/main/old_name_views/helpers.py @@ -9,8 +9,7 @@ import pytest -from sqlbuild.compiler.migrations.models import OldNameViewEvent -from sqlbuild.compiler.migrations.types import OldNameViewEventType +from sqlbuild._native import OldNameViewEvent, OldNameViewEventType from sqlbuild.executor.migrations._helpers import old_name_views as old_name_steps from tests.integration.src.sqlbuild.cli.commands.main.model_migrations.helpers import ( CliRun, diff --git a/tests/integration/src/sqlbuild/cli/commands/main/test_load_command.py b/tests/integration/src/sqlbuild/cli/commands/main/test_load_command.py index faf3f957cf..d5cbf5b9b5 100644 --- a/tests/integration/src/sqlbuild/cli/commands/main/test_load_command.py +++ b/tests/integration/src/sqlbuild/cli/commands/main/test_load_command.py @@ -17,8 +17,10 @@ AuditCommandRequest, BuildCommandRequest, CliUserError, + CursorOverrides, LoadCommandRequest, PlanCommandRequest, + PlannerInputError, ScenarioTestCommandRequest, SourceEntry, TestCommandRequest, @@ -35,8 +37,6 @@ from sqlbuild.adapter.contract.types import CursorKind from sqlbuild.adapters.duckdb.classes.duckdb_adapter import DuckDbAdapter from sqlbuild.compiler.discovery.models import DiscoveredProjectInputs, DiscoveredSourceFile -from sqlbuild.compiler.planner.exceptions import PlannerInputError -from sqlbuild.compiler.planner.models import CursorOverrides from sqlbuild.executor.load.main.run import run_load_pipeline from sqlbuild.executor.load.models import ( LoadCallbacks, diff --git a/tests/integration/src/sqlbuild/cli/commands/main/test_startup_import_footprint.py b/tests/integration/src/sqlbuild/cli/commands/main/test_startup_import_footprint.py index 005153b360..d0b78d5354 100644 --- a/tests/integration/src/sqlbuild/cli/commands/main/test_startup_import_footprint.py +++ b/tests/integration/src/sqlbuild/cli/commands/main/test_startup_import_footprint.py @@ -43,7 +43,6 @@ "sqlbuild.adapter.contract.classes.base_adapter", "sqlbuild.compiler.compile.models", "sqlbuild.compiler.discovery.models", - "sqlbuild.compiler.planner.models", "sqlbuild.lint", "sqlbuild.rule_engine", "sqlbuild.runtime.compute_logs", @@ -67,7 +66,6 @@ "sqlbuild.adapter.contract.classes.base_adapter", "sqlbuild.compiler.compile.models", "sqlbuild.compiler.pipeline", - "sqlbuild.compiler.planner.models", ) REUSED_COMPILE_FORBIDDEN: tuple[str, ...] = ( *WAREHOUSE_AND_INTEGRATION_MODULES, diff --git a/tests/integration/src/sqlbuild/compiler/contracts/helpers.py b/tests/integration/src/sqlbuild/compiler/contracts/helpers.py index d392365a7e..1ab2e01904 100644 --- a/tests/integration/src/sqlbuild/compiler/contracts/helpers.py +++ b/tests/integration/src/sqlbuild/compiler/contracts/helpers.py @@ -14,9 +14,12 @@ import sqlbuild._native as native_module from sqlbuild._native import ( ColumnContractMode, + ContractPolicy, ContractValidationResult, DynamicColumnContractProof, DynamicColumnFamilyProof, + IncrementalMode, + MaterializationType, SchemaColumn, SchemaDynamicColumnFamily, SchemaModelEntry, @@ -31,7 +34,6 @@ from sqlbuild.compiler.compile.models import CompilerDiagnostic, InferredColumn from sqlbuild.compiler.compile.types import CompiledModelView, CompiledProjectView from sqlbuild.compiler.lineage.types import InferredNullability -from sqlbuild.compiler.planner.types import ContractPolicy, IncrementalMode, MaterializationType _PROJECT_TOML: str = ( 'name = "orders_contracts"\nadapter = "duckdb"\n\n[connection]\ndatabase = "orders.duckdb"\n' diff --git a/tests/integration/src/sqlbuild/compiler/pipeline/test_main.py b/tests/integration/src/sqlbuild/compiler/pipeline/test_main.py index 66bd89520c..cfe37ac6eb 100644 --- a/tests/integration/src/sqlbuild/compiler/pipeline/test_main.py +++ b/tests/integration/src/sqlbuild/compiler/pipeline/test_main.py @@ -8,6 +8,7 @@ import pytest from sqlbuild._native import ( + ModelPlanEntry, build_compiled_project, build_project_graph, discover_project_inputs, @@ -20,7 +21,6 @@ from sqlbuild.compiler.discovery.models import DiscoveredProjectInputs from sqlbuild.compiler.lineage.types import InferredNullability from sqlbuild.compiler.pipeline.models import CompilePipelineResult -from sqlbuild.compiler.planner.models import ModelPlanEntry from sqlbuild.rule_engine.exceptions import RulesError from tests.integration.src.sqlbuild.compiler.pipeline._test_types import ( AppendCursorPipelineIntegrationTestCase, diff --git a/tests/integration/src/sqlbuild/compiler/pipeline/test_measurement_audits.py b/tests/integration/src/sqlbuild/compiler/pipeline/test_measurement_audits.py index 3006b453fd..847620ca96 100644 --- a/tests/integration/src/sqlbuild/compiler/pipeline/test_measurement_audits.py +++ b/tests/integration/src/sqlbuild/compiler/pipeline/test_measurement_audits.py @@ -5,11 +5,11 @@ import pytest +from sqlbuild._native import AuditPlanEntry from sqlbuild.adapters.duckdb.classes.duckdb_adapter import DuckDbAdapter from sqlbuild.compiler.auditing.types import AuditEvaluationMode, ThresholdOperator from sqlbuild.compiler.compile.types import CompiledAuditView from sqlbuild.compiler.pipeline.models import CompilePipelineResult -from sqlbuild.compiler.planner.models import AuditPlanEntry from tests.integration.src.sqlbuild.compiler.pipeline._test_types import ( MeasurementCompileErrorIntegrationTestCase, MeasurementCompileIntegrationTestCase, diff --git a/tests/integration/src/sqlbuild/compiler/planner/_helpers/_test_types.py b/tests/integration/src/sqlbuild/compiler/planner/_helpers/_test_types.py index 1151b0c650..f0976db814 100644 --- a/tests/integration/src/sqlbuild/compiler/planner/_helpers/_test_types.py +++ b/tests/integration/src/sqlbuild/compiler/planner/_helpers/_test_types.py @@ -3,12 +3,10 @@ from sqlbuild._native import ( CursorSnapshotScope, Fingerprint, + ModelCursorSnapshot, SourceEntry, ) from sqlbuild.compiler.compile.models import CompiledObjectKey, CompiledRelationLocation -from sqlbuild.compiler.planner.models import ( - ModelCursorSnapshot, -) from sqlbuild.cursor_algebra.types import CursorScalar diff --git a/tests/integration/src/sqlbuild/compiler/planner/_helpers/resolve/helpers.py b/tests/integration/src/sqlbuild/compiler/planner/_helpers/resolve/helpers.py index 4bbb5d6290..ed7e7658fb 100644 --- a/tests/integration/src/sqlbuild/compiler/planner/_helpers/resolve/helpers.py +++ b/tests/integration/src/sqlbuild/compiler/planner/_helpers/resolve/helpers.py @@ -7,11 +7,13 @@ from typing import Any, cast from sqlbuild._native import ( + BackfillAction, BackfillResult, CollectionRendering, CompileAdapterContext, CompileModelConfig, CursorOverridePair, + ModelCursorSnapshot, ModelPlanContext, SourceEntry, WarehouseSnapshot, @@ -33,10 +35,6 @@ CompiledProjectView, CompiledResourceType, ) -from sqlbuild.compiler.planner.models import ( - ModelCursorSnapshot, -) -from sqlbuild.compiler.planner.types import BackfillAction from sqlbuild.compiler.references.types import SqlReferenceKind diff --git a/tests/integration/src/sqlbuild/compiler/planner/_helpers/resolve/test_cursor_planning.py b/tests/integration/src/sqlbuild/compiler/planner/_helpers/resolve/test_cursor_planning.py index 4b3e677317..bef84b5758 100644 --- a/tests/integration/src/sqlbuild/compiler/planner/_helpers/resolve/test_cursor_planning.py +++ b/tests/integration/src/sqlbuild/compiler/planner/_helpers/resolve/test_cursor_planning.py @@ -5,7 +5,7 @@ import pytest -from sqlbuild.compiler.planner.models import CursorBounds, ModelPlanEntry, PlanOutput +from sqlbuild._native import CursorBounds, ModelPlanEntry, PlanOutput from tests.integration.src.sqlbuild.compiler.planner._helpers.resolve._test_types import ( CursorPlannerTestCase, ) diff --git a/tests/integration/src/sqlbuild/compiler/planner/_helpers/resolve/test_orchestration.py b/tests/integration/src/sqlbuild/compiler/planner/_helpers/resolve/test_orchestration.py index b55a6cc093..ef12ba2995 100644 --- a/tests/integration/src/sqlbuild/compiler/planner/_helpers/resolve/test_orchestration.py +++ b/tests/integration/src/sqlbuild/compiler/planner/_helpers/resolve/test_orchestration.py @@ -7,6 +7,7 @@ import pytest from sqlbuild._native import ( + BackfillAction, BackfillResult, CursorOverridePair, ModelPlanContext, @@ -20,7 +21,6 @@ ) from sqlbuild.compiler.compile.types import CompiledModelView, CompiledProjectView from sqlbuild.compiler.discovery.models import SqlHookEntry -from sqlbuild.compiler.planner.types import BackfillAction from tests.integration.src.sqlbuild.compiler.planner._helpers.resolve._test_types import ( OrchestrationTestCase, ) diff --git a/tests/integration/src/sqlbuild/compiler/planner/_helpers/resolve/test_resolve.py b/tests/integration/src/sqlbuild/compiler/planner/_helpers/resolve/test_resolve.py index 4d3a358a46..d85c979d49 100644 --- a/tests/integration/src/sqlbuild/compiler/planner/_helpers/resolve/test_resolve.py +++ b/tests/integration/src/sqlbuild/compiler/planner/_helpers/resolve/test_resolve.py @@ -7,6 +7,9 @@ import pytest from sqlbuild._native import ( + CursorType, + ModelCursorSnapshot, + PlannerInputError, SourceColumnEntry, SourceEntry, WarehouseSnapshot, @@ -16,11 +19,6 @@ CompiledRelationLocation, ) from sqlbuild.compiler.compile.types import CompiledModelView -from sqlbuild.compiler.planner.exceptions import PlannerInputError -from sqlbuild.compiler.planner.models import ( - ModelCursorSnapshot, -) -from sqlbuild.compiler.planner.types import CursorType from tests.integration.src.sqlbuild.compiler.planner._helpers.resolve._test_types import ( ResolveAndExecuteTestCase, ResolveBoundedOverrideTestCase, diff --git a/tests/integration/src/sqlbuild/compiler/planner/_helpers/sql_test_assembly/test_execute_chain.py b/tests/integration/src/sqlbuild/compiler/planner/_helpers/sql_test_assembly/test_execute_chain.py index b185ed8256..86b5f22be6 100644 --- a/tests/integration/src/sqlbuild/compiler/planner/_helpers/sql_test_assembly/test_execute_chain.py +++ b/tests/integration/src/sqlbuild/compiler/planner/_helpers/sql_test_assembly/test_execute_chain.py @@ -7,9 +7,11 @@ import pytest from sqlbuild._native import ( + ChainStep, CompiledDirectLogicSqlTestPayload, CompileSqlTestCte, DiscoveredSqlTestBlock, + SqlTestPlanEntry, build_compiled_project, discover_project_inputs, ) @@ -17,7 +19,6 @@ from sqlbuild.compiler.compile.models import CompiledObjectKey, CompiledProject, CompiledSqlTest from sqlbuild.compiler.compile.types import CompiledProjectView, CompiledResourceType, SqlTestMode from sqlbuild.compiler.discovery.models import DiscoveredSqlTestFile -from sqlbuild.compiler.planner.models import ChainStep, SqlTestPlanEntry from sqlbuild.executor.testing.main._execute import execute_sql_test from sqlbuild.executor.testing.models import SqlTestExecutionResult from sqlbuild.executor.testing.types import SqlTestOutcome diff --git a/tests/integration/src/sqlbuild/compiler/planner/_helpers/test_audit_entry.py b/tests/integration/src/sqlbuild/compiler/planner/_helpers/test_audit_entry.py index db0b98771e..8c0e94e6a4 100644 --- a/tests/integration/src/sqlbuild/compiler/planner/_helpers/test_audit_entry.py +++ b/tests/integration/src/sqlbuild/compiler/planner/_helpers/test_audit_entry.py @@ -5,7 +5,13 @@ import pytest -from sqlbuild._native import DiscoveredAuditBlock, DiscoveredAuditFile, SourceEntry, plan_audit +from sqlbuild._native import ( + AuditPlanEntry, + DiscoveredAuditBlock, + DiscoveredAuditFile, + SourceEntry, + plan_audit, +) from sqlbuild.adapters.duckdb.classes.duckdb_adapter import DuckDbAdapter from sqlbuild.compiler.compile.models import ( CompiledAudit, @@ -13,7 +19,6 @@ CompiledRelationLocation, ) from sqlbuild.compiler.compile.types import CompiledAuditView, CompiledResourceType -from sqlbuild.compiler.planner.models import AuditPlanEntry from tests.integration.src.sqlbuild.compiler.planner._helpers._test_types import ( ExecuteAuditTestCase, ) diff --git a/tests/integration/src/sqlbuild/compiler/planner/_helpers/test_fixture_planning.py b/tests/integration/src/sqlbuild/compiler/planner/_helpers/test_fixture_planning.py index 8f26166f98..ee30f642a9 100644 --- a/tests/integration/src/sqlbuild/compiler/planner/_helpers/test_fixture_planning.py +++ b/tests/integration/src/sqlbuild/compiler/planner/_helpers/test_fixture_planning.py @@ -8,6 +8,7 @@ from sqlbuild._native import ( RelationFixtureCompletion, RelationFixturePlanningContext, + SqlTestPlanEntry, SqlTestSelection, build_compiled_project, build_relation_fixture_completion, @@ -18,7 +19,6 @@ from sqlbuild.adapters.duckdb.classes.duckdb_adapter import DuckDbAdapter from sqlbuild.compiler.compile.types import CompiledProjectView, CompiledResourceType from sqlbuild.compiler.discovery.models import DiscoveredProjectInputs -from sqlbuild.compiler.planner.models import SqlTestPlanEntry from tests.integration.src.sqlbuild.compiler.planner._helpers._test_types import ( FixturePlanningIntegrationTestCase, FixtureSchemaIntegrationTestCase, diff --git a/tests/integration/src/sqlbuild/compiler/planner/_helpers/test_warehouse_cursor_reads.py b/tests/integration/src/sqlbuild/compiler/planner/_helpers/test_warehouse_cursor_reads.py index e92bdd06ff..35ceca3c1c 100644 --- a/tests/integration/src/sqlbuild/compiler/planner/_helpers/test_warehouse_cursor_reads.py +++ b/tests/integration/src/sqlbuild/compiler/planner/_helpers/test_warehouse_cursor_reads.py @@ -7,6 +7,8 @@ from sqlbuild._native import ( CursorSnapshotScope, + ModelCursorSnapshot, + PlannerInputError, WarehouseSnapshot, gather_warehouse_snapshot, ) @@ -14,10 +16,6 @@ from sqlbuild.adapters.duckdb.classes.duckdb_adapter import DuckDbAdapter from sqlbuild.compiler.compile.models import CompiledObjectKey from sqlbuild.compiler.compile.types import CompiledProjectView, CompiledResourceType -from sqlbuild.compiler.planner.exceptions import PlannerInputError -from sqlbuild.compiler.planner.models import ( - ModelCursorSnapshot, -) from tests.integration.src.sqlbuild.compiler.planner._helpers._test_types import ( ConcurrentCursorReadsTestCase, EligibleMaxFailureTestCase, diff --git a/tests/integration/src/sqlbuild/compiler/planner/_helpers/test_warehouse_snapshot.py b/tests/integration/src/sqlbuild/compiler/planner/_helpers/test_warehouse_snapshot.py index 9e975e0e21..13acc78eb7 100644 --- a/tests/integration/src/sqlbuild/compiler/planner/_helpers/test_warehouse_snapshot.py +++ b/tests/integration/src/sqlbuild/compiler/planner/_helpers/test_warehouse_snapshot.py @@ -7,8 +7,11 @@ import pytest from sqlbuild._native import ( + CursorOverrides, CursorSnapshotScope, Fingerprint, + ModelCursorSnapshot, + PlannerInputError, SchemaColumn, SchemaModelEntry, TimestampValue, @@ -30,11 +33,6 @@ CompiledProjectView, CompiledResourceType, ) -from sqlbuild.compiler.planner.exceptions import PlannerInputError -from sqlbuild.compiler.planner.models import ( - CursorOverrides, - ModelCursorSnapshot, -) from tests.integration.src.sqlbuild.compiler.planner._helpers._test_types import ( GatherCursorSnapshotTestCase, GatherEmptySnapshotTestCase, diff --git a/tests/integration/src/sqlbuild/compiler/planner/main/_test_types.py b/tests/integration/src/sqlbuild/compiler/planner/main/_test_types.py index b6e7da3b27..acc3e29866 100644 --- a/tests/integration/src/sqlbuild/compiler/planner/main/_test_types.py +++ b/tests/integration/src/sqlbuild/compiler/planner/main/_test_types.py @@ -1,14 +1,12 @@ from dataclasses import dataclass, field from sqlbuild._native import ( - WarningSeverity, -) -from sqlbuild.compiler.compile.types import FunctionLanguage -from sqlbuild.compiler.planner.types import ( BackfillAction, PlanAction, PlanReason, + WarningSeverity, ) +from sqlbuild.compiler.compile.types import FunctionLanguage @dataclass(frozen=True) diff --git a/tests/integration/src/sqlbuild/compiler/planner/main/helpers.py b/tests/integration/src/sqlbuild/compiler/planner/main/helpers.py index 8ec57443cb..ca2f0c1946 100644 --- a/tests/integration/src/sqlbuild/compiler/planner/main/helpers.py +++ b/tests/integration/src/sqlbuild/compiler/planner/main/helpers.py @@ -26,6 +26,7 @@ PlannerOverrides, PlannerPolicies, PlannerSelection, + PlanOutput, SchemaSeedEntry, SettingsConfig, SourceEntry, @@ -58,7 +59,6 @@ ) from sqlbuild.compiler.discovery.models import DiscoveredProjectInputs, DiscoveredSourceFile from sqlbuild.compiler.pipeline.models import CompilePipelineResult -from sqlbuild.compiler.planner.models import PlanOutput from sqlbuild.compiler.references.types import SqlReferenceKind from sqlbuild.runtime.contracts.models import ConnectionHooks from tests.integration.src.sqlbuild.compiler.planner.main._test_types import ( diff --git a/tests/integration/src/sqlbuild/compiler/planner/main/test_build_execution_plan.py b/tests/integration/src/sqlbuild/compiler/planner/main/test_build_execution_plan.py index 1c8de80ebd..b005a94aba 100644 --- a/tests/integration/src/sqlbuild/compiler/planner/main/test_build_execution_plan.py +++ b/tests/integration/src/sqlbuild/compiler/planner/main/test_build_execution_plan.py @@ -9,10 +9,17 @@ import sqlbuild._native as identity_planning from sqlbuild._native import ( + BackfillAction, + CursorBounds, CursorsConfig, FutureCursorAction, FutureCursorsConfig, LocalConfig, + ModelPlanEntry, + PlanAction, + PlannerInputError, + PlanOutput, + PlanReason, PlanWarning, ProjectConfig, ResolvedTableType, @@ -27,17 +34,6 @@ from sqlbuild.adapter.contract.types import BuiltinAdapter from sqlbuild.adapters.duckdb.classes.duckdb_adapter import DuckDbAdapter from sqlbuild.compiler.compile.types import CompiledProjectView -from sqlbuild.compiler.planner.exceptions import PlannerInputError -from sqlbuild.compiler.planner.models import ( - CursorBounds, - ModelPlanEntry, - PlanOutput, -) -from sqlbuild.compiler.planner.types import ( - BackfillAction, - PlanAction, - PlanReason, -) from tests.integration.src.sqlbuild.compiler.planner.main._test_types import ( BuildExecutionPlanTestCase, FormatPlanIntegrationTestCase, diff --git a/tests/integration/src/sqlbuild/compiler/planner/main/test_snowflake_inspection_budget.py b/tests/integration/src/sqlbuild/compiler/planner/main/test_snowflake_inspection_budget.py index 30a4a9bcd0..3494144aac 100644 --- a/tests/integration/src/sqlbuild/compiler/planner/main/test_snowflake_inspection_budget.py +++ b/tests/integration/src/sqlbuild/compiler/planner/main/test_snowflake_inspection_budget.py @@ -10,10 +10,10 @@ import pytest +from sqlbuild._native import PlanReason from sqlbuild.adapter.contract.exceptions import AdapterUserError from sqlbuild.adapter.relations.constants import INSPECTION_IN_LIST_LIMIT from sqlbuild.compiler.pipeline.models import CompilePipelineResult -from sqlbuild.compiler.planner.types import PlanReason from tests.integration.src.sqlbuild.compiler.planner.main._test_types import ( SnowflakeCursorBoundsBudgetTestCase, SnowflakeFreshTargetTestCase, diff --git a/tests/integration/src/sqlbuild/executor/build/concurrent/helpers.py b/tests/integration/src/sqlbuild/executor/build/concurrent/helpers.py index b4a63cf3c8..5d706ab5c5 100644 --- a/tests/integration/src/sqlbuild/executor/build/concurrent/helpers.py +++ b/tests/integration/src/sqlbuild/executor/build/concurrent/helpers.py @@ -7,12 +7,16 @@ from pathlib import Path from typing import Any, cast -from sqlbuild._native import CompilePipelineOptions, discover_project_inputs, run_compile_pipeline +from sqlbuild._native import ( + CompilePipelineOptions, + PlanOutput, + discover_project_inputs, + run_compile_pipeline, +) from sqlbuild.adapter.contract.types import TablePromotionMode from sqlbuild.adapters.duckdb.classes.duckdb_adapter import DuckDbAdapter from sqlbuild.compiler.discovery.models import DiscoveredProjectInputs from sqlbuild.compiler.pipeline.models import CompilePipelineResult -from sqlbuild.compiler.planner.models import PlanOutput from sqlbuild.executor.build.main._execute import execute_build_plan from sqlbuild.executor.build.models import ( BuildCustomizations, diff --git a/tests/integration/src/sqlbuild/executor/build/concurrent/test_ordering_invariant.py b/tests/integration/src/sqlbuild/executor/build/concurrent/test_ordering_invariant.py index 1748ca2566..e757c9d29f 100644 --- a/tests/integration/src/sqlbuild/executor/build/concurrent/test_ordering_invariant.py +++ b/tests/integration/src/sqlbuild/executor/build/concurrent/test_ordering_invariant.py @@ -9,12 +9,16 @@ import pytest -from sqlbuild._native import CompilePipelineOptions, discover_project_inputs, run_compile_pipeline +from sqlbuild._native import ( + CompilePipelineOptions, + PlanOutput, + discover_project_inputs, + run_compile_pipeline, +) from sqlbuild.adapter.contract.types import TablePromotionMode from sqlbuild.adapters.duckdb.classes.duckdb_adapter import DuckDbAdapter from sqlbuild.compiler.discovery.models import DiscoveredProjectInputs from sqlbuild.compiler.pipeline.models import CompilePipelineResult -from sqlbuild.compiler.planner.models import PlanOutput from sqlbuild.executor.build.main._execute import execute_build_plan from sqlbuild.executor.build.models import ( BuildCallbacks, diff --git a/tests/integration/src/sqlbuild/executor/build/custom/_test_types.py b/tests/integration/src/sqlbuild/executor/build/custom/_test_types.py index cd5026db34..ea47f3a373 100644 --- a/tests/integration/src/sqlbuild/executor/build/custom/_test_types.py +++ b/tests/integration/src/sqlbuild/executor/build/custom/_test_types.py @@ -1,9 +1,8 @@ from dataclasses import dataclass, field +from sqlbuild._native import AuditPlanEntry, PlanReason from sqlbuild.adapter.contract.models import RelationInfo from sqlbuild.compiler.auditing.types import AuditOutcome -from sqlbuild.compiler.planner.models import AuditPlanEntry -from sqlbuild.compiler.planner.types import PlanReason from sqlbuild.executor.build.types import BuildStatus from sqlbuild.executor.run.types import ExecutionPhase from sqlbuild.executor.scheduling.types import ExecutionStatus diff --git a/tests/integration/src/sqlbuild/executor/build/custom/helpers.py b/tests/integration/src/sqlbuild/executor/build/custom/helpers.py index 3b622651e7..be4b320c00 100644 --- a/tests/integration/src/sqlbuild/executor/build/custom/helpers.py +++ b/tests/integration/src/sqlbuild/executor/build/custom/helpers.py @@ -6,6 +6,13 @@ from pathlib import Path from typing import Any, cast +from sqlbuild._native import ( + AuditPlanEntry, + MaterializationType, + ModelPlanEntry, + PlanAction, + PlanReason, +) from sqlbuild.adapter.contract.models import RelationInfo from sqlbuild.adapters.duckdb.classes.duckdb_adapter import DuckDbAdapter from sqlbuild.compiler.auditing.types import ( @@ -19,8 +26,6 @@ ) from sqlbuild.compiler.compile.types import CompiledResourceType from sqlbuild.compiler.discovery.models import DiscoveredHookFunction -from sqlbuild.compiler.planner.models import AuditPlanEntry, ModelPlanEntry -from sqlbuild.compiler.planner.types import MaterializationType, PlanAction, PlanReason from sqlbuild.executor.auditing.models import AuditExecutionResult from sqlbuild.executor.custom.models import ( MaterializationContext, @@ -427,13 +432,13 @@ def run_scheduler_build( from sqlbuild._native import ( CompilePipelineOptions, + PlanOutput, discover_project_inputs, run_compile_pipeline, ) from sqlbuild.adapter.contract.types import TablePromotionMode from sqlbuild.compiler.discovery.models import DiscoveredProjectInputs from sqlbuild.compiler.pipeline.models import CompilePipelineResult - from sqlbuild.compiler.planner.models import PlanOutput from sqlbuild.executor.build.main._execute import execute_build_plan from sqlbuild.executor.build.models import ( BuildCustomizations, diff --git a/tests/integration/src/sqlbuild/executor/build/custom/test_advanced_custom_materialization.py b/tests/integration/src/sqlbuild/executor/build/custom/test_advanced_custom_materialization.py index 06f7212128..0ab10600c6 100644 --- a/tests/integration/src/sqlbuild/executor/build/custom/test_advanced_custom_materialization.py +++ b/tests/integration/src/sqlbuild/executor/build/custom/test_advanced_custom_materialization.py @@ -9,12 +9,11 @@ import duckdb import pytest +from sqlbuild._native import AuditPlanEntry, ModelPlanEntry, PlanReason from sqlbuild.adapter.contract.models import RelationInfo from sqlbuild.adapter.contract.types import LifeCycleEventKind from sqlbuild.adapters.duckdb.classes.duckdb_adapter import DuckDbAdapter from sqlbuild.compiler.compile.models import CompiledRelationLocation -from sqlbuild.compiler.planner.models import AuditPlanEntry, ModelPlanEntry -from sqlbuild.compiler.planner.types import PlanReason from sqlbuild.executor.build.models import BuildExecutionResult from sqlbuild.executor.build.types import BuildStatus from sqlbuild.executor.custom.models import MaterializationContext, MaterializationResult diff --git a/tests/integration/src/sqlbuild/executor/build/custom/test_custom_materialization.py b/tests/integration/src/sqlbuild/executor/build/custom/test_custom_materialization.py index b984b2a510..6717bfb894 100644 --- a/tests/integration/src/sqlbuild/executor/build/custom/test_custom_materialization.py +++ b/tests/integration/src/sqlbuild/executor/build/custom/test_custom_materialization.py @@ -9,12 +9,11 @@ import duckdb import pytest +from sqlbuild._native import AuditPlanEntry, ModelPlanEntry, PlanReason from sqlbuild.adapters.duckdb.classes.duckdb_adapter import DuckDbAdapter from sqlbuild.compiler.auditing.types import AuditOutcome from sqlbuild.compiler.compile.models import CompiledRelationLocation from sqlbuild.compiler.discovery.models import DiscoveredHookFunction, PythonHookEntry, SqlHookEntry -from sqlbuild.compiler.planner.models import AuditPlanEntry, ModelPlanEntry -from sqlbuild.compiler.planner.types import PlanReason from sqlbuild.executor.custom.models import ( MaterializationContext, MaterializationResult, diff --git a/tests/integration/src/sqlbuild/executor/build/helpers.py b/tests/integration/src/sqlbuild/executor/build/helpers.py index a98e098f01..8e6cae918d 100644 --- a/tests/integration/src/sqlbuild/executor/build/helpers.py +++ b/tests/integration/src/sqlbuild/executor/build/helpers.py @@ -13,6 +13,8 @@ from sqlbuild._native import ( CompilePipelineOptions, MicrobatchLimitAction, + PlanOutput, + TableTypePlanEntry, discover_project_inputs, run_compile_pipeline, ) @@ -21,7 +23,6 @@ from sqlbuild.adapters.duckdb.classes.duckdb_adapter import DuckDbAdapter from sqlbuild.compiler.discovery.models import DiscoveredProjectInputs from sqlbuild.compiler.pipeline.models import CompilePipelineResult -from sqlbuild.compiler.planner.models import PlanOutput, TableTypePlanEntry from sqlbuild.executor.build.main._execute import execute_build_plan from sqlbuild.executor.build.models import ( BuildExecutionResult, diff --git a/tests/integration/src/sqlbuild/executor/build/test_snapshot_execution.py b/tests/integration/src/sqlbuild/executor/build/test_snapshot_execution.py index ba35552546..c3924f4fbe 100644 --- a/tests/integration/src/sqlbuild/executor/build/test_snapshot_execution.py +++ b/tests/integration/src/sqlbuild/executor/build/test_snapshot_execution.py @@ -8,11 +8,16 @@ import pytest -from sqlbuild._native import CompilePipelineOptions, discover_project_inputs, run_compile_pipeline +from sqlbuild._native import ( + CompilePipelineOptions, + ModelPlanEntry, + PlanOutput, + discover_project_inputs, + run_compile_pipeline, +) from sqlbuild.adapters.duckdb.classes.duckdb_adapter import DuckDbAdapter from sqlbuild.compiler.discovery.models import DiscoveredProjectInputs from sqlbuild.compiler.pipeline.models import CompilePipelineResult -from sqlbuild.compiler.planner.models import ModelPlanEntry, PlanOutput from sqlbuild.executor.build.models import BuildExecutionResult from sqlbuild.executor.build.types import BuildStatus from sqlbuild.executor.scheduling.types import ExecutionStatus diff --git a/tests/integration/src/sqlbuild/executor/migrations/_helpers/helpers.py b/tests/integration/src/sqlbuild/executor/migrations/_helpers/helpers.py index c588ec16b1..4674c6ce68 100644 --- a/tests/integration/src/sqlbuild/executor/migrations/_helpers/helpers.py +++ b/tests/integration/src/sqlbuild/executor/migrations/_helpers/helpers.py @@ -2,13 +2,16 @@ from __future__ import annotations -from sqlbuild._native import MigrationCompatibility, MigrationDiscovery +from sqlbuild._native import ( + MigrationCompatibility, + MigrationDecision, + MigrationDiscovery, + ModelMigrationPlanEntry, +) from sqlbuild.adapter.contract.models import MigrationStagePlan from sqlbuild.adapter.contract.types import MigrationTransfer from sqlbuild.adapters.duckdb.classes.duckdb_adapter import DuckDbAdapter from sqlbuild.compiler.compile.models import CompiledRelationLocation -from sqlbuild.compiler.migrations.types import MigrationDecision -from sqlbuild.compiler.planner.models import ModelMigrationPlanEntry from sqlbuild.executor.migrations.models import MigrationArtifactNames diff --git a/tests/integration/src/sqlbuild/executor/run/helpers.py b/tests/integration/src/sqlbuild/executor/run/helpers.py index fc4c5bf5ea..ebb4d9253f 100644 --- a/tests/integration/src/sqlbuild/executor/run/helpers.py +++ b/tests/integration/src/sqlbuild/executor/run/helpers.py @@ -7,6 +7,13 @@ from pathlib import Path from typing import Any, cast +from sqlbuild._native import ( + AuditPlanEntry, + MaterializationType, + ModelPlanEntry, + PlanAction, + PlanReason, +) from sqlbuild.adapter.contract.models import ColumnInfo from sqlbuild.adapters.duckdb.classes.duckdb_adapter import DuckDbAdapter from sqlbuild.compiler.auditing.types import ( @@ -20,12 +27,6 @@ ) from sqlbuild.compiler.compile.types import CompiledResourceType from sqlbuild.compiler.discovery.models import DiscoveredHookFunction -from sqlbuild.compiler.planner.models import AuditPlanEntry, ModelPlanEntry -from sqlbuild.compiler.planner.types import ( - MaterializationType, - PlanAction, - PlanReason, -) from sqlbuild.executor.run.main._execute import execute_table_entry from sqlbuild.executor.run.models import ( HookContext, diff --git a/tests/integration/src/sqlbuild/executor/run/incremental/_test_types.py b/tests/integration/src/sqlbuild/executor/run/incremental/_test_types.py index d0d2f7fd06..f0a0665f71 100644 --- a/tests/integration/src/sqlbuild/executor/run/incremental/_test_types.py +++ b/tests/integration/src/sqlbuild/executor/run/incremental/_test_types.py @@ -1,9 +1,8 @@ from dataclasses import dataclass, field from datetime import datetime -from sqlbuild._native import FutureCursorsConfig +from sqlbuild._native import FutureCursorsConfig, OnSchemaChange from sqlbuild.compiler.auditing.types import AuditRunScope -from sqlbuild.compiler.planner.types import OnSchemaChange from sqlbuild.executor.run.types import ExecutionPhase from sqlbuild.executor.scheduling.types import ExecutionStatus diff --git a/tests/integration/src/sqlbuild/executor/run/incremental/helpers.py b/tests/integration/src/sqlbuild/executor/run/incremental/helpers.py index 15d4924f49..3e1ab53b3a 100644 --- a/tests/integration/src/sqlbuild/executor/run/incremental/helpers.py +++ b/tests/integration/src/sqlbuild/executor/run/incremental/helpers.py @@ -7,6 +7,17 @@ from pathlib import Path from typing import Any, cast +from sqlbuild._native import ( + AuditPlanEntry, + CursorBounds, + CursorInputRelation, + IncrementalStrategy, + MaterializationType, + ModelPlanEntry, + OnSchemaChange, + PlanAction, + PlanReason, +) from sqlbuild.adapter.contract.models import ColumnInfo from sqlbuild.adapters.duckdb.classes.duckdb_adapter import DuckDbAdapter from sqlbuild.compiler.auditing.types import ( @@ -20,19 +31,6 @@ ) from sqlbuild.compiler.compile.types import CompiledResourceType from sqlbuild.compiler.discovery.models import DiscoveredHookFunction -from sqlbuild.compiler.planner.models import ( - AuditPlanEntry, - CursorBounds, - CursorInputRelation, - ModelPlanEntry, -) -from sqlbuild.compiler.planner.types import ( - IncrementalStrategy, - MaterializationType, - OnSchemaChange, - PlanAction, - PlanReason, -) from sqlbuild.executor.run._helpers.materializations.incremental import execute_incremental_entry from sqlbuild.executor.run.models import ( HookContext, diff --git a/tests/integration/src/sqlbuild/executor/run/incremental/test_incremental_execution.py b/tests/integration/src/sqlbuild/executor/run/incremental/test_incremental_execution.py index 5bd8539c16..c8c150a114 100644 --- a/tests/integration/src/sqlbuild/executor/run/incremental/test_incremental_execution.py +++ b/tests/integration/src/sqlbuild/executor/run/incremental/test_incremental_execution.py @@ -8,12 +8,11 @@ import pytest -from sqlbuild._native import FutureCursorAction, FutureCursorsConfig +from sqlbuild._native import FutureCursorAction, FutureCursorsConfig, OnSchemaChange from sqlbuild.adapters.duckdb.classes.duckdb_adapter import DuckDbAdapter from sqlbuild.compiler.auditing.types import AuditRunScope from sqlbuild.compiler.discovery.models import DiscoveredHookFunction, PythonHookEntry, SqlHookEntry from sqlbuild.compiler.planner.constants import MICROBATCH_END_SENTINEL, MICROBATCH_START_SENTINEL -from sqlbuild.compiler.planner.types import OnSchemaChange from sqlbuild.executor.run.models import ModelExecutionResult from sqlbuild.executor.run.types import ExecutionPhase from tests.integration.src.sqlbuild.executor.run.incremental._test_types import ( diff --git a/tests/integration/src/sqlbuild/executor/run/microbatch/_test_types.py b/tests/integration/src/sqlbuild/executor/run/microbatch/_test_types.py index 0e337020e9..02106223b4 100644 --- a/tests/integration/src/sqlbuild/executor/run/microbatch/_test_types.py +++ b/tests/integration/src/sqlbuild/executor/run/microbatch/_test_types.py @@ -2,9 +2,8 @@ from dataclasses import dataclass, field from datetime import datetime -from sqlbuild._native import FutureCursorsConfig, MicrobatchLimitAction +from sqlbuild._native import FutureCursorsConfig, MicrobatchLimitAction, OnSchemaChange from sqlbuild.compiler.auditing.types import AuditRunScope -from sqlbuild.compiler.planner.types import OnSchemaChange from sqlbuild.executor.run.types import ExecutionPhase from sqlbuild.executor.scheduling.types import ExecutionStatus diff --git a/tests/integration/src/sqlbuild/executor/run/microbatch/helpers.py b/tests/integration/src/sqlbuild/executor/run/microbatch/helpers.py index 85b5114203..22327065b0 100644 --- a/tests/integration/src/sqlbuild/executor/run/microbatch/helpers.py +++ b/tests/integration/src/sqlbuild/executor/run/microbatch/helpers.py @@ -7,6 +7,17 @@ from pathlib import Path from typing import Any, cast +from sqlbuild._native import ( + AuditPlanEntry, + CursorBounds, + CursorInputRelation, + IncrementalMode, + IncrementalStrategy, + MaterializationType, + ModelPlanEntry, + PlanAction, + PlanReason, +) from sqlbuild.adapters.duckdb.classes.duckdb_adapter import DuckDbAdapter from sqlbuild.compiler.auditing.types import ( AuditAttachmentKind, @@ -22,19 +33,6 @@ MICROBATCH_END_SENTINEL, MICROBATCH_START_SENTINEL, ) -from sqlbuild.compiler.planner.models import ( - AuditPlanEntry, - CursorBounds, - CursorInputRelation, - ModelPlanEntry, -) -from sqlbuild.compiler.planner.types import ( - IncrementalMode, - IncrementalStrategy, - MaterializationType, - PlanAction, - PlanReason, -) from sqlbuild.executor.run._helpers.materializations.microbatch import execute_microbatch_entry from sqlbuild.executor.run._helpers.validation.cursor_bounds import resolve_runtime_cursor_bounds from sqlbuild.executor.run.models import ( diff --git a/tests/integration/src/sqlbuild/executor/run/microbatch/test_microbatch_execution.py b/tests/integration/src/sqlbuild/executor/run/microbatch/test_microbatch_execution.py index 51a27ff1ba..f98a5c0f54 100644 --- a/tests/integration/src/sqlbuild/executor/run/microbatch/test_microbatch_execution.py +++ b/tests/integration/src/sqlbuild/executor/run/microbatch/test_microbatch_execution.py @@ -10,7 +10,12 @@ import pytest -from sqlbuild._native import FutureCursorAction, FutureCursorsConfig, MicrobatchLimitAction +from sqlbuild._native import ( + FutureCursorAction, + FutureCursorsConfig, + MicrobatchLimitAction, + OnSchemaChange, +) from sqlbuild.adapters.duckdb.classes.duckdb_adapter import DuckDbAdapter from sqlbuild.compiler.auditing.types import AuditRunScope from sqlbuild.compiler.discovery.models import DiscoveredHookFunction, PythonHookEntry, SqlHookEntry @@ -18,7 +23,6 @@ MICROBATCH_END_SENTINEL, MICROBATCH_START_SENTINEL, ) -from sqlbuild.compiler.planner.types import OnSchemaChange from sqlbuild.executor.run._helpers.materializations import microbatch as microbatch_module from sqlbuild.executor.run.models import BatchWindow, ModelExecutionResult from sqlbuild.executor.run.types import ExecutionPhase diff --git a/tests/integration/src/sqlbuild/executor/run/microbatch/test_microbatch_reconciliation.py b/tests/integration/src/sqlbuild/executor/run/microbatch/test_microbatch_reconciliation.py index 5fe88de068..548b644586 100644 --- a/tests/integration/src/sqlbuild/executor/run/microbatch/test_microbatch_reconciliation.py +++ b/tests/integration/src/sqlbuild/executor/run/microbatch/test_microbatch_reconciliation.py @@ -8,23 +8,19 @@ import pytest from sqlbuild._native import ( + BackfillAction, BackfillResult, + CursorBounds, CursorWatermarkMode, IntegerValue, MicrobatchLimitAction, + ModelCursorSnapshot, + ModelPlanEntry, compute_cursor_bounds, ) from sqlbuild._native import cursor_sentinel_to_token as sentinel_to_token from sqlbuild.adapter.contract.classes.statement_recorder import StatementRecorder from sqlbuild.adapters.duckdb.classes.duckdb_adapter import DuckDbAdapter -from sqlbuild.compiler.planner.models import ( - CursorBounds, - ModelCursorSnapshot, - ModelPlanEntry, -) -from sqlbuild.compiler.planner.types import ( - BackfillAction, -) from sqlbuild.executor.run._helpers.materializations.microbatch import ( _clamp_intervals_to_model_domain, _count_unaccounted_intervals, diff --git a/tests/integration/src/sqlbuild/executor/run/test_audit_helpers.py b/tests/integration/src/sqlbuild/executor/run/test_audit_helpers.py index ea9d0433b9..798a903a6d 100644 --- a/tests/integration/src/sqlbuild/executor/run/test_audit_helpers.py +++ b/tests/integration/src/sqlbuild/executor/run/test_audit_helpers.py @@ -4,7 +4,7 @@ import pytest -from sqlbuild.compiler.planner.models import AuditPlanEntry +from sqlbuild._native import AuditPlanEntry from tests.integration.src.sqlbuild.executor.run._test_types import AuditSqlResolutionTestCase from tests.integration.src.sqlbuild.executor.run.helpers import build_test_audit_plan_entry diff --git a/tests/integration/src/sqlbuild/executor/run/view/helpers.py b/tests/integration/src/sqlbuild/executor/run/view/helpers.py index f182f88d24..d5fb8b989b 100644 --- a/tests/integration/src/sqlbuild/executor/run/view/helpers.py +++ b/tests/integration/src/sqlbuild/executor/run/view/helpers.py @@ -6,6 +6,13 @@ from pathlib import Path from typing import Any, cast +from sqlbuild._native import ( + AuditPlanEntry, + MaterializationType, + ModelPlanEntry, + PlanAction, + PlanReason, +) from sqlbuild.adapters.duckdb.classes.duckdb_adapter import DuckDbAdapter from sqlbuild.compiler.auditing.types import ( AuditAttachmentKind, @@ -18,12 +25,6 @@ ) from sqlbuild.compiler.compile.types import CompiledResourceType from sqlbuild.compiler.discovery.models import DiscoveredHookFunction -from sqlbuild.compiler.planner.models import AuditPlanEntry, ModelPlanEntry -from sqlbuild.compiler.planner.types import ( - MaterializationType, - PlanAction, - PlanReason, -) from sqlbuild.executor.run._helpers.materializations.view import execute_view_entry from sqlbuild.executor.run.models import ( HookContext, diff --git a/tests/integration/src/sqlbuild/executor/run/view/test_audit_helpers.py b/tests/integration/src/sqlbuild/executor/run/view/test_audit_helpers.py index 06016f62fe..18efbc5742 100644 --- a/tests/integration/src/sqlbuild/executor/run/view/test_audit_helpers.py +++ b/tests/integration/src/sqlbuild/executor/run/view/test_audit_helpers.py @@ -4,7 +4,7 @@ import pytest -from sqlbuild.compiler.planner.models import AuditPlanEntry +from sqlbuild._native import AuditPlanEntry from tests.integration.src.sqlbuild.executor.run.view._test_types import ( ViewAuditSqlResolutionTestCase, ) diff --git a/tests/integration/src/sqlbuild/executor/scenario/helpers.py b/tests/integration/src/sqlbuild/executor/scenario/helpers.py index b7d9284811..ffcb8e18f1 100644 --- a/tests/integration/src/sqlbuild/executor/scenario/helpers.py +++ b/tests/integration/src/sqlbuild/executor/scenario/helpers.py @@ -4,29 +4,25 @@ from typing import Any from sqlbuild._native import ( - ScenarioGraphPlan, -) -from sqlbuild.compiler.compile.models import ( - CompiledObjectKey, - CompiledRelationLocation, -) -from sqlbuild.compiler.compile.types import CompiledResourceType -from sqlbuild.compiler.discovery.models import DiscoveredHookFunction -from sqlbuild.compiler.planner.models import ( + MaterializationType, ModelPlanEntry, + PlanAction, + PlanReason, + ScenarioArtifactKind, ScenarioAssertionExpectationPlan, ScenarioExecutionPlan, ScenarioExpectedExpectationPlan, ScenarioFixturePlan, + ScenarioGraphPlan, ScenarioRelationMap, ScenarioRelationPlan, ) -from sqlbuild.compiler.planner.types import ( - MaterializationType, - PlanAction, - PlanReason, - ScenarioArtifactKind, +from sqlbuild.compiler.compile.models import ( + CompiledObjectKey, + CompiledRelationLocation, ) +from sqlbuild.compiler.compile.types import CompiledResourceType +from sqlbuild.compiler.discovery.models import DiscoveredHookFunction from sqlbuild.executor.run.models import HookContext SCENARIO_NAME: str = "revenue__customer_refund" diff --git a/tests/integration/src/sqlbuild/executor/scenario/test_capture_types.py b/tests/integration/src/sqlbuild/executor/scenario/test_capture_types.py index 32037dd928..0ef295f0ca 100644 --- a/tests/integration/src/sqlbuild/executor/scenario/test_capture_types.py +++ b/tests/integration/src/sqlbuild/executor/scenario/test_capture_types.py @@ -9,9 +9,9 @@ import duckdb import pytest +from sqlbuild._native import ScenarioArtifactKind from sqlbuild.adapters.duckdb.classes.duckdb_adapter import DuckDbAdapter from sqlbuild.compiler.compile.models import CompiledRelationLocation -from sqlbuild.compiler.planner.types import ScenarioArtifactKind from sqlbuild.executor.scenario._helpers.snapshots.core import read_scenario_snapshot_manifest from sqlbuild.executor.scenario.main._capture import execute_scenario_snapshot_capture from sqlbuild.executor.scenario.models import ( diff --git a/tests/integration/src/sqlbuild/executor/scenario/test_fixture_lifecycle.py b/tests/integration/src/sqlbuild/executor/scenario/test_fixture_lifecycle.py index 333fb27434..d4aa3492c7 100644 --- a/tests/integration/src/sqlbuild/executor/scenario/test_fixture_lifecycle.py +++ b/tests/integration/src/sqlbuild/executor/scenario/test_fixture_lifecycle.py @@ -5,6 +5,7 @@ import pytest +from sqlbuild._native import ScenarioExecutionPlan, ScenarioFixturePlan, SeedPlanEntry from sqlbuild.adapter.contract.models import ColumnInfo from sqlbuild.adapter.contract.types import TablePromotionMode from sqlbuild.adapters.duckdb.classes.duckdb_adapter import DuckDbAdapter @@ -14,11 +15,6 @@ ) from sqlbuild.compiler.compile.types import CompiledResourceType from sqlbuild.compiler.discovery.models import DiscoveredHookFunction, PythonHookEntry -from sqlbuild.compiler.planner.models import ( - ScenarioExecutionPlan, - ScenarioFixturePlan, - SeedPlanEntry, -) from sqlbuild.executor.build.models import SeedExecutionResult from sqlbuild.executor.run.models import ModelExecutionResult from sqlbuild.executor.scenario._helpers.execution.model_execution import execute_scenario_models diff --git a/tests/integration/src/sqlbuild/executor/testing/helpers.py b/tests/integration/src/sqlbuild/executor/testing/helpers.py index 2514914c86..ba5c2de129 100644 --- a/tests/integration/src/sqlbuild/executor/testing/helpers.py +++ b/tests/integration/src/sqlbuild/executor/testing/helpers.py @@ -6,12 +6,11 @@ from pathlib import Path from typing import Any -from sqlbuild._native import render_sql_test_comparison_sql +from sqlbuild._native import ChainStep, SqlTestPlanEntry, render_sql_test_comparison_sql from sqlbuild.adapters.duckdb.classes.duckdb_adapter import DuckDbAdapter from sqlbuild.compiler.compile.models import CompiledObjectKey from sqlbuild.compiler.compile.types import CompiledResourceType from sqlbuild.compiler.pipeline.models import CompilePipelineResult -from sqlbuild.compiler.planner.models import ChainStep, SqlTestPlanEntry from sqlbuild.executor.testing.main._execute import execute_sql_test from sqlbuild.executor.testing.models import SqlTestExecutionResult from sqlbuild.executor.testing.types import SqlTestOutcome diff --git a/tests/integration/src/sqlbuild/executor/testing/test_sql_test_build_parity.py b/tests/integration/src/sqlbuild/executor/testing/test_sql_test_build_parity.py index 10a88288a7..d58f985ab1 100644 --- a/tests/integration/src/sqlbuild/executor/testing/test_sql_test_build_parity.py +++ b/tests/integration/src/sqlbuild/executor/testing/test_sql_test_build_parity.py @@ -9,9 +9,8 @@ import pytest -from sqlbuild._native import render_sql_test_comparison_sql +from sqlbuild._native import SqlTestPlanEntry, render_sql_test_comparison_sql from sqlbuild.adapters.duckdb.classes.duckdb_adapter import DuckDbAdapter -from sqlbuild.compiler.planner.models import SqlTestPlanEntry from tests.integration.src.sqlbuild.executor.testing._test_types import ( ExpectedBooleanTestCase, SqlTestBuildParityTestCase, diff --git a/tests/integration/src/sqlbuild/executor/testing/test_sql_test_execution.py b/tests/integration/src/sqlbuild/executor/testing/test_sql_test_execution.py index 5af2fe7cee..88c05db349 100644 --- a/tests/integration/src/sqlbuild/executor/testing/test_sql_test_execution.py +++ b/tests/integration/src/sqlbuild/executor/testing/test_sql_test_execution.py @@ -7,11 +7,15 @@ import pytest -from sqlbuild._native import render_sql_test_comparison_sql, render_sql_test_difference_sample_sql +from sqlbuild._native import ( + ChainStep, + SqlTestPlanEntry, + render_sql_test_comparison_sql, + render_sql_test_difference_sample_sql, +) from sqlbuild.adapter.contract.types import StatementSizeLimit from sqlbuild.adapters.duckdb.classes.duckdb_adapter import DuckDbAdapter from sqlbuild.adapters.sqlserver.classes.sqlserver_adapter import SqlServerAdapter -from sqlbuild.compiler.planner.models import ChainStep, SqlTestPlanEntry from sqlbuild.executor.testing.main._execute import execute_sql_test from sqlbuild.executor.testing.models import ( SqlTestColumnDifference, diff --git a/tests/unit/src/sqlbuild/adapters/sqlserver/test_migration_state_sql.py b/tests/unit/src/sqlbuild/adapters/sqlserver/test_migration_state_sql.py index b580c0c654..f9cc02a30f 100644 --- a/tests/unit/src/sqlbuild/adapters/sqlserver/test_migration_state_sql.py +++ b/tests/unit/src/sqlbuild/adapters/sqlserver/test_migration_state_sql.py @@ -7,13 +7,14 @@ import pytest from sqlbuild._native import ( + MigrationDecision, MigrationDiscovery, + MigrationEvent, + MigrationRelation, read_migration_state_events, write_migration_state_event, ) from sqlbuild.adapters.sqlserver.classes.sqlserver_adapter import SqlServerAdapter -from sqlbuild.compiler.migrations.models import MigrationEvent, MigrationRelation -from sqlbuild.compiler.migrations.types import MigrationDecision from tests.unit.src.sqlbuild.adapters.sqlserver._test_types import ( SqlServerMigrationEventReadTestCase, SqlServerMigrationEventWriteTestCase, diff --git a/tests/unit/src/sqlbuild/cli/commands/_helpers/build_planning/_test_types.py b/tests/unit/src/sqlbuild/cli/commands/_helpers/build_planning/_test_types.py index 56e3fef96d..fffeb3ee94 100644 --- a/tests/unit/src/sqlbuild/cli/commands/_helpers/build_planning/_test_types.py +++ b/tests/unit/src/sqlbuild/cli/commands/_helpers/build_planning/_test_types.py @@ -1,7 +1,6 @@ from dataclasses import dataclass -from sqlbuild._native import SnapshotsConfig -from sqlbuild.compiler.planner.models import PlanOutput +from sqlbuild._native import PlanOutput, SnapshotsConfig @dataclass(frozen=True) diff --git a/tests/unit/src/sqlbuild/cli/commands/_helpers/build_planning/helpers.py b/tests/unit/src/sqlbuild/cli/commands/_helpers/build_planning/helpers.py index 2eccf2fc78..e20a2f17b8 100644 --- a/tests/unit/src/sqlbuild/cli/commands/_helpers/build_planning/helpers.py +++ b/tests/unit/src/sqlbuild/cli/commands/_helpers/build_planning/helpers.py @@ -2,23 +2,21 @@ from pathlib import Path -from sqlbuild._native import PlanSafetyGates -from sqlbuild.adapter.contract.models import RetentionRequest -from sqlbuild.adapter.contract.types import RetentionScope -from sqlbuild.compiler.compile.models import CompiledObjectKey, CompiledRelationLocation -from sqlbuild.compiler.compile.types import CompiledResourceType -from sqlbuild.compiler.planner.models import ( - ModelPlanEntry, - RetentionPlanEntry, - TableTypePlanEntry, -) -from sqlbuild.compiler.planner.types import ( +from sqlbuild._native import ( MaterializationType, + ModelPlanEntry, PlanAction, PlanReason, + PlanSafetyGates, RetentionDirection, + RetentionPlanEntry, RetentionPlanPhase, + TableTypePlanEntry, ) +from sqlbuild.adapter.contract.models import RetentionRequest +from sqlbuild.adapter.contract.types import RetentionScope +from sqlbuild.compiler.compile.models import CompiledObjectKey, CompiledRelationLocation +from sqlbuild.compiler.compile.types import CompiledResourceType def build_snapshot_full_refresh_entry( diff --git a/tests/unit/src/sqlbuild/cli/commands/_helpers/build_planning/test_full_refresh.py b/tests/unit/src/sqlbuild/cli/commands/_helpers/build_planning/test_full_refresh.py index caf2762ff8..754d7c88a3 100644 --- a/tests/unit/src/sqlbuild/cli/commands/_helpers/build_planning/test_full_refresh.py +++ b/tests/unit/src/sqlbuild/cli/commands/_helpers/build_planning/test_full_refresh.py @@ -9,10 +9,10 @@ from sqlbuild._native import ( CliUserError, ExecutionLimitsConfig, + PlanOutput, SnapshotsConfig, enforce_plan_safety_policies, ) -from sqlbuild.compiler.planner.models import PlanOutput from tests.unit.src.sqlbuild.cli.commands._helpers.build_planning._test_types import ( SnapshotFullRefreshPolicyTestCase, ) diff --git a/tests/unit/src/sqlbuild/cli/commands/_helpers/build_planning/test_plan_policies.py b/tests/unit/src/sqlbuild/cli/commands/_helpers/build_planning/test_plan_policies.py index b20ce4da2b..6a31f8b025 100644 --- a/tests/unit/src/sqlbuild/cli/commands/_helpers/build_planning/test_plan_policies.py +++ b/tests/unit/src/sqlbuild/cli/commands/_helpers/build_planning/test_plan_policies.py @@ -7,11 +7,11 @@ from sqlbuild._native import ( CliUserError, ExecutionLimitsConfig, + PlanOutput, SnapshotsConfig, dbt_interop_plan_safety_gates, enforce_plan_safety_policies, ) -from sqlbuild.compiler.planner.models import PlanOutput from tests.unit.src.sqlbuild.cli.commands._helpers.build_planning._test_types import ( DbtPlanSafetyPolicyTestCase, ) diff --git a/tests/unit/src/sqlbuild/cli/commands/_helpers/build_planning/test_retention_decrease.py b/tests/unit/src/sqlbuild/cli/commands/_helpers/build_planning/test_retention_decrease.py index 35a1638a18..3f0e3b3fb9 100644 --- a/tests/unit/src/sqlbuild/cli/commands/_helpers/build_planning/test_retention_decrease.py +++ b/tests/unit/src/sqlbuild/cli/commands/_helpers/build_planning/test_retention_decrease.py @@ -7,11 +7,11 @@ from sqlbuild._native import ( CliUserError, ExecutionLimitsConfig, + PlanOutput, + RetentionDirection, SnapshotsConfig, enforce_plan_safety_policies, ) -from sqlbuild.compiler.planner.models import PlanOutput -from sqlbuild.compiler.planner.types import RetentionDirection from tests.unit.src.sqlbuild.cli.commands._helpers.build_planning._test_types import ( RetentionDecreasePolicyTestCase, ) diff --git a/tests/unit/src/sqlbuild/cli/commands/_helpers/build_planning/test_table_type.py b/tests/unit/src/sqlbuild/cli/commands/_helpers/build_planning/test_table_type.py index 0403cc2edb..d163fc690c 100644 --- a/tests/unit/src/sqlbuild/cli/commands/_helpers/build_planning/test_table_type.py +++ b/tests/unit/src/sqlbuild/cli/commands/_helpers/build_planning/test_table_type.py @@ -7,10 +7,10 @@ from sqlbuild._native import ( CliUserError, ExecutionLimitsConfig, + PlanOutput, SnapshotsConfig, enforce_plan_safety_policies, ) -from sqlbuild.compiler.planner.models import PlanOutput from tests.unit.src.sqlbuild.cli.commands._helpers.build_planning._test_types import ( TableTypeDowngradePolicyTestCase, ) diff --git a/tests/unit/src/sqlbuild/cli/commands/_helpers/check/test_planning.py b/tests/unit/src/sqlbuild/cli/commands/_helpers/check/test_planning.py index ea74610ce7..6149b9f353 100644 --- a/tests/unit/src/sqlbuild/cli/commands/_helpers/check/test_planning.py +++ b/tests/unit/src/sqlbuild/cli/commands/_helpers/check/test_planning.py @@ -8,6 +8,7 @@ from sqlbuild._native import ( CliUserError, + PlanOutput, PythonSqlRunLifecyclePlan, SourceEntry, run_check_read_side_dependencies, @@ -16,7 +17,6 @@ from sqlbuild.adapter.relations.main import relation_lookup as relation_lookup_module from sqlbuild.compiler.compile.models import CompiledProject, CompiledRelationLocation from sqlbuild.compiler.pipeline.models import CompilePipelineResult -from sqlbuild.compiler.planner.models import PlanOutput from sqlbuild.executor.python_nodes.main import read_side as read_side_module from sqlbuild.executor.python_nodes.models import PythonNodeExecutionResult from sqlbuild.python_nodes.models import SqlResourceRef diff --git a/tests/unit/src/sqlbuild/cli/commands/_helpers/helpers.py b/tests/unit/src/sqlbuild/cli/commands/_helpers/helpers.py index 901ac2b132..341dd52cfc 100644 --- a/tests/unit/src/sqlbuild/cli/commands/_helpers/helpers.py +++ b/tests/unit/src/sqlbuild/cli/commands/_helpers/helpers.py @@ -4,9 +4,15 @@ from pathlib import Path from sqlbuild._native import ( + ChainStep, CostContext, + MaterializationType, + ModelPlanEntry, + PlanAction, + PlanReason, SourceFreshnessIdentity, SourceFreshnessRecord, + SqlTestPlanEntry, build_source_freshness_create_table_sql, ) from sqlbuild.adapter.contract.types import FrameworkType @@ -16,8 +22,6 @@ CompiledRelationLocation, ) from sqlbuild.compiler.compile.types import CompiledResourceType -from sqlbuild.compiler.planner.models import ChainStep, ModelPlanEntry, SqlTestPlanEntry -from sqlbuild.compiler.planner.types import MaterializationType, PlanAction, PlanReason from sqlbuild.cost.models import CostResourceContext diff --git a/tests/unit/src/sqlbuild/cli/commands/_helpers/test_source_freshness.py b/tests/unit/src/sqlbuild/cli/commands/_helpers/test_source_freshness.py index 44a8306b08..58400b0b0f 100644 --- a/tests/unit/src/sqlbuild/cli/commands/_helpers/test_source_freshness.py +++ b/tests/unit/src/sqlbuild/cli/commands/_helpers/test_source_freshness.py @@ -10,12 +10,12 @@ CostContext, DirectSourceFreshnessPlanningResult, DirectSourceFreshnessPropagationResult, + PlanOutput, ) from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter from sqlbuild.cli.commands._helpers.freshness.source_freshness import ( append_eligible_direct_source_freshness_records, ) -from sqlbuild.compiler.planner.models import PlanOutput from sqlbuild.cost.models import CostResourceContext from sqlbuild.executor.build.models import BuildExecutionResult from sqlbuild.executor.build.types import BuildStatus diff --git a/tests/unit/src/sqlbuild/cli/commands/main/compile/helpers.py b/tests/unit/src/sqlbuild/cli/commands/main/compile/helpers.py index b1a45adaab..a04f0cbcb6 100644 --- a/tests/unit/src/sqlbuild/cli/commands/main/compile/helpers.py +++ b/tests/unit/src/sqlbuild/cli/commands/main/compile/helpers.py @@ -6,11 +6,21 @@ from typing import Any from sqlbuild._native import ( + AuditPlanEntry, + ChainStep, CompiledFunction, CompileModelConfig, FunctionArgument, + FunctionPlanEntry, + MaterializationType, + ModelPlanEntry, + PlanAction, + PlanOutput, + PlanReason, PreparedCompileArtifacts, SeedCsvSettings, + SeedPlanEntry, + SqlTestPlanEntry, WrittenTarget, project_with, ) @@ -35,16 +45,6 @@ FunctionLanguage, ) from sqlbuild.compiler.pipeline.models import ProjectGraph -from sqlbuild.compiler.planner.models import ( - AuditPlanEntry, - ChainStep, - FunctionPlanEntry, - ModelPlanEntry, - PlanOutput, - SeedPlanEntry, - SqlTestPlanEntry, -) -from sqlbuild.compiler.planner.types import MaterializationType, PlanAction, PlanReason from sqlbuild.executor.build.models import ( BuildExecutionResult, FunctionExecutionResult, diff --git a/tests/unit/src/sqlbuild/cli/commands/main/compile/test_compile_target_writer.py b/tests/unit/src/sqlbuild/cli/commands/main/compile/test_compile_target_writer.py index 87d117101e..7a0da29847 100644 --- a/tests/unit/src/sqlbuild/cli/commands/main/compile/test_compile_target_writer.py +++ b/tests/unit/src/sqlbuild/cli/commands/main/compile/test_compile_target_writer.py @@ -11,7 +11,10 @@ import sqlbuild._native as model_chain_module import sqlbuild._native as plan_artifacts_module from sqlbuild._native import ( + ChainStep, DiagnosticPhase, + PlanOutput, + SqlTestPlanEntry, WrittenTarget, build_manifest, compiled_model_record, @@ -27,7 +30,6 @@ CompiledProjectView, CompiledResourceType, ) -from sqlbuild.compiler.planner.models import ChainStep, PlanOutput, SqlTestPlanEntry from tests.unit.src.sqlbuild.cli.commands.main.compile._test_types import ( ExpectedMessageTestCase, TargetWriterCacheTestCase, diff --git a/tests/unit/src/sqlbuild/cli/commands/main/entry/test_main.py b/tests/unit/src/sqlbuild/cli/commands/main/entry/test_main.py index 59eb70930a..b3fbacdb99 100644 --- a/tests/unit/src/sqlbuild/cli/commands/main/entry/test_main.py +++ b/tests/unit/src/sqlbuild/cli/commands/main/entry/test_main.py @@ -18,6 +18,7 @@ CompileCommandRequest, CompileLineageMode, CostCommandRequest, + CursorOverrides, DbtInitCommandRequest, DiffCommandRequest, FreshnessCommandRequest, @@ -35,7 +36,6 @@ from sqlbuild._native import run_cli as _main_with_dependencies from sqlbuild._native import run_cli as main from sqlbuild.compiler.compile.exceptions import CompileInputError -from sqlbuild.compiler.planner.models import CursorOverrides from sqlbuild.executor.pipeline.exceptions import AuditExecutionError from sqlbuild.executor.pipeline.models import AuditExecutionFailure from sqlbuild.observability import EventDispatcher, ExecutionIdentity, current_execution_identity diff --git a/tests/unit/src/sqlbuild/cli/commands/shared/_helpers/_test_types.py b/tests/unit/src/sqlbuild/cli/commands/shared/_helpers/_test_types.py index 1c03c3f7a8..969189e0a3 100644 --- a/tests/unit/src/sqlbuild/cli/commands/shared/_helpers/_test_types.py +++ b/tests/unit/src/sqlbuild/cli/commands/shared/_helpers/_test_types.py @@ -1,6 +1,6 @@ from dataclasses import dataclass, field -from sqlbuild.compiler.planner.models import PlanOutput +from sqlbuild._native import PlanOutput from sqlbuild.executor.build.models import BuildExecutionResult from sqlbuild.executor.load.models import LoadExecutionResult from sqlbuild.executor.run.models import ModelExecutionResult diff --git a/tests/unit/src/sqlbuild/cli/commands/shared/_helpers/helpers.py b/tests/unit/src/sqlbuild/cli/commands/shared/_helpers/helpers.py index 3efb0d4095..61da8acb40 100644 --- a/tests/unit/src/sqlbuild/cli/commands/shared/_helpers/helpers.py +++ b/tests/unit/src/sqlbuild/cli/commands/shared/_helpers/helpers.py @@ -5,7 +5,16 @@ import re from pathlib import Path -from sqlbuild._native import SqlTestAssertionStep +from sqlbuild._native import ( + ChainStep, + MaterializationType, + ModelPlanEntry, + PlanAction, + PlanOutput, + PlanReason, + SqlTestAssertionStep, + SqlTestPlanEntry, +) from sqlbuild.compiler.auditing.types import ( AuditAttachmentKind, AuditOutcome, @@ -14,8 +23,6 @@ ) from sqlbuild.compiler.compile.models import CompiledObjectKey, CompiledRelationLocation from sqlbuild.compiler.compile.types import CompiledResourceType -from sqlbuild.compiler.planner.models import ChainStep, ModelPlanEntry, PlanOutput, SqlTestPlanEntry -from sqlbuild.compiler.planner.types import MaterializationType, PlanAction, PlanReason from sqlbuild.executor.auditing.models import AuditExecutionResult from sqlbuild.executor.testing.models import SqlTestExecutionResult, StepResult from sqlbuild.executor.testing.types import SqlTestOutcome diff --git a/tests/unit/src/sqlbuild/cli/commands/shared/_helpers/test_progress.py b/tests/unit/src/sqlbuild/cli/commands/shared/_helpers/test_progress.py index 526c540419..88223e1e7f 100644 --- a/tests/unit/src/sqlbuild/cli/commands/shared/_helpers/test_progress.py +++ b/tests/unit/src/sqlbuild/cli/commands/shared/_helpers/test_progress.py @@ -11,6 +11,7 @@ BuildProgressCallbacks, NestedCommandProgressCallbacks, NestedProgressChildRow, + PlanOutput, build_test_expectation_rows, format_build_footer, resolve_test_name_width, @@ -19,7 +20,6 @@ from sqlbuild.adapter.contract.models import LifeCycleEvent from sqlbuild.adapter.contract.types import LifeCycleEventKind from sqlbuild.compiler.auditing.types import AuditOutcome -from sqlbuild.compiler.planner.models import PlanOutput from sqlbuild.compiler.python_nodes.types import SkipMode from sqlbuild.cost.models import StatementExecutionTelemetry from sqlbuild.executor.build.models import ( diff --git a/tests/unit/src/sqlbuild/cli/output/_helpers/test_execution_result_document.py b/tests/unit/src/sqlbuild/cli/output/_helpers/test_execution_result_document.py index f4e0ad4741..36939cb23f 100644 --- a/tests/unit/src/sqlbuild/cli/output/_helpers/test_execution_result_document.py +++ b/tests/unit/src/sqlbuild/cli/output/_helpers/test_execution_result_document.py @@ -9,7 +9,10 @@ import pytest from sqlbuild._native import ( + CursorInputEvidence, FutureCursorAction, + FutureCursorSafetyEvidence, + MaximumStartSafetyEvidence, MeasurementThresholdBound, MeasurementThresholds, SqlLogicalType, @@ -29,11 +32,6 @@ ) from sqlbuild.compiler.compile.types import AttachedAuditTargetKind from sqlbuild.compiler.discovery.models import SqlTestParameterDeclaration -from sqlbuild.compiler.planner.models import ( - CursorInputEvidence, - FutureCursorSafetyEvidence, - MaximumStartSafetyEvidence, -) from sqlbuild.executor.auditing.models import AuditExecutionResult from sqlbuild.executor.run.models import ( MicrobatchAccountingInterval, diff --git a/tests/unit/src/sqlbuild/cli/output/main/build_execution_json/test_execution_protocol_v1.py b/tests/unit/src/sqlbuild/cli/output/main/build_execution_json/test_execution_protocol_v1.py index 1d90242727..56b15cc7ce 100644 --- a/tests/unit/src/sqlbuild/cli/output/main/build_execution_json/test_execution_protocol_v1.py +++ b/tests/unit/src/sqlbuild/cli/output/main/build_execution_json/test_execution_protocol_v1.py @@ -6,9 +6,8 @@ import pytest -from sqlbuild._native import MicrobatchLimitAction, format_build_execution_json +from sqlbuild._native import MicrobatchLimitAction, PlanReason, format_build_execution_json from sqlbuild.compiler.auditing.types import AuditOutcome -from sqlbuild.compiler.planner.types import PlanReason from sqlbuild.compiler.python_nodes.types import PythonNodeKind, PythonNodeStatus from sqlbuild.executor.build.models import BuildExecutionResult, SeedExecutionResult from sqlbuild.executor.build.types import BuildStatus diff --git a/tests/unit/src/sqlbuild/cli/output/main/plan/_test_types.py b/tests/unit/src/sqlbuild/cli/output/main/plan/_test_types.py index a60630e85a..6aece1104c 100644 --- a/tests/unit/src/sqlbuild/cli/output/main/plan/_test_types.py +++ b/tests/unit/src/sqlbuild/cli/output/main/plan/_test_types.py @@ -4,11 +4,7 @@ from dataclasses import dataclass, field -from sqlbuild._native import DisplayOptions, PythonPlanEntry -from sqlbuild.compiler.planner.models import ( - CursorInputRelation, - PlanOutput, -) +from sqlbuild._native import CursorInputRelation, DisplayOptions, PlanOutput, PythonPlanEntry @dataclass(frozen=True) diff --git a/tests/unit/src/sqlbuild/cli/output/main/plan/helpers.py b/tests/unit/src/sqlbuild/cli/output/main/plan/helpers.py index c9699a1b5c..ee834131f9 100644 --- a/tests/unit/src/sqlbuild/cli/output/main/plan/helpers.py +++ b/tests/unit/src/sqlbuild/cli/output/main/plan/helpers.py @@ -5,15 +5,28 @@ from pathlib import Path from sqlbuild._native import ( + BackfillAction, BackfillResult, ColumnRenameHint, + CursorBounds, + CursorInputRelation, + FunctionPlanEntry, + MaterializationType, MicrobatchLimitAction, + ModelPlanEntry, + PlanAction, + PlanOutput, PlanProviderUsage, + PlanReason, PlanWarning, SchemaChangeKind, SchemaColumnSource, + SchemaFinding, SeedCsvSettings, + SeedPlanEntry, + SourceLoadPlanEntry, SourceWriteStrategy, + TableTypePlanEntry, WarningSeverity, ) from sqlbuild.compiler.compile.models import ( @@ -22,23 +35,6 @@ ) from sqlbuild.compiler.compile.types import CompiledResourceType, FunctionLanguage from sqlbuild.compiler.discovery.models import DiscoveredProviderUsage -from sqlbuild.compiler.planner.models import ( - CursorBounds, - CursorInputRelation, - FunctionPlanEntry, - ModelPlanEntry, - PlanOutput, - SchemaFinding, - SeedPlanEntry, - SourceLoadPlanEntry, - TableTypePlanEntry, -) -from sqlbuild.compiler.planner.types import ( - BackfillAction, - MaterializationType, - PlanAction, - PlanReason, -) def build_model_entry( diff --git a/tests/unit/src/sqlbuild/cli/output/main/plan/test_plan_text.py b/tests/unit/src/sqlbuild/cli/output/main/plan/test_plan_text.py index 35b771b1b4..01747e9e02 100644 --- a/tests/unit/src/sqlbuild/cli/output/main/plan/test_plan_text.py +++ b/tests/unit/src/sqlbuild/cli/output/main/plan/test_plan_text.py @@ -7,9 +7,16 @@ import pytest from sqlbuild._native import ( + BackfillAction, ColumnRenameHint, + CursorBounds, + CursorInputRelation, DisplayOptions, GraphResourceKind, + MaterializationType, + PlanAction, + PlanOutput, + PlanReason, PythonIdentityStatus, PythonPlanEntry, PythonRunPhase, @@ -19,17 +26,6 @@ format_plan, ) from sqlbuild.compiler.compile.types import FunctionLanguage -from sqlbuild.compiler.planner.models import ( - CursorBounds, - CursorInputRelation, - PlanOutput, -) -from sqlbuild.compiler.planner.types import ( - BackfillAction, - MaterializationType, - PlanAction, - PlanReason, -) from sqlbuild.compiler.python_nodes.types import PythonNodeKind from tests.unit.src.sqlbuild.cli.output.main.plan._test_types import ( FormatPlanColorTestCase, diff --git a/tests/unit/src/sqlbuild/cli/output/main/plan_json/_test_types.py b/tests/unit/src/sqlbuild/cli/output/main/plan_json/_test_types.py index 22ea29272f..4561411ef2 100644 --- a/tests/unit/src/sqlbuild/cli/output/main/plan_json/_test_types.py +++ b/tests/unit/src/sqlbuild/cli/output/main/plan_json/_test_types.py @@ -1,7 +1,6 @@ from dataclasses import dataclass, field -from sqlbuild._native import PythonPlanEntry -from sqlbuild.compiler.planner.models import PlanOutput +from sqlbuild._native import PlanOutput, PythonPlanEntry @dataclass(frozen=True) diff --git a/tests/unit/src/sqlbuild/cli/output/main/plan_json/test_plan_json.py b/tests/unit/src/sqlbuild/cli/output/main/plan_json/test_plan_json.py index 9a9f860197..19192c4d82 100644 --- a/tests/unit/src/sqlbuild/cli/output/main/plan_json/test_plan_json.py +++ b/tests/unit/src/sqlbuild/cli/output/main/plan_json/test_plan_json.py @@ -8,27 +8,23 @@ import pytest from sqlbuild._native import ( - FutureCursorAction, - MicrobatchLimitAction, - PythonIdentityStatus, - PythonPlanEntry, - PythonRunPhase, - WarningSeverity, - format_plan_json, -) -from sqlbuild.compiler.planner.models import ( + BackfillAction, CursorBounds, CursorInputEvidence, CursorInputRelation, + FutureCursorAction, FutureCursorSafetyEvidence, - MaximumStartSafetyEvidence, - PlanOutput, -) -from sqlbuild.compiler.planner.types import ( - BackfillAction, MaterializationType, + MaximumStartSafetyEvidence, + MicrobatchLimitAction, PlanAction, + PlanOutput, PlanReason, + PythonIdentityStatus, + PythonPlanEntry, + PythonRunPhase, + WarningSeverity, + format_plan_json, ) from sqlbuild.compiler.python_nodes.types import PythonNodeKind from tests.unit.src.sqlbuild.cli.output.main.plan.helpers import ( diff --git a/tests/unit/src/sqlbuild/cli/progress/classes/test_build_sql_test_projection.py b/tests/unit/src/sqlbuild/cli/progress/classes/test_build_sql_test_projection.py index 35846a539d..69887593a2 100644 --- a/tests/unit/src/sqlbuild/cli/progress/classes/test_build_sql_test_projection.py +++ b/tests/unit/src/sqlbuild/cli/progress/classes/test_build_sql_test_projection.py @@ -9,13 +9,15 @@ from sqlbuild._native import ( BuildProgressCallbacks, + ChainStep, NativeProgressProjector, + PlanOutput, + SqlTestPlanEntry, TerminalEventIndex, terminal_event_index_scope, ) from sqlbuild.compiler.compile.models import CompiledObjectKey from sqlbuild.compiler.compile.types import CompiledResourceType -from sqlbuild.compiler.planner.models import ChainStep, PlanOutput, SqlTestPlanEntry from sqlbuild.executor.run.models import ModelExecutionResult from sqlbuild.executor.scheduling.types import ExecutionStatus from sqlbuild.executor.testing.models import SqlTestExecutionResult, StepResult diff --git a/tests/unit/src/sqlbuild/cli/target_artifacts/main/write_runtime_target/test_runtime_target_writer.py b/tests/unit/src/sqlbuild/cli/target_artifacts/main/write_runtime_target/test_runtime_target_writer.py index e4d9c27550..e11cecf15f 100644 --- a/tests/unit/src/sqlbuild/cli/target_artifacts/main/write_runtime_target/test_runtime_target_writer.py +++ b/tests/unit/src/sqlbuild/cli/target_artifacts/main/write_runtime_target/test_runtime_target_writer.py @@ -4,8 +4,7 @@ import pytest -from sqlbuild._native import write_runtime_target -from sqlbuild.compiler.planner.models import PlanOutput +from sqlbuild._native import PlanOutput, write_runtime_target from sqlbuild.executor.build.models import BuildExecutionResult from tests.unit.src.sqlbuild.cli.commands.main.compile.helpers import ( build_runtime_target_execution_result, diff --git a/tests/unit/src/sqlbuild/compiler/auditing/main/helpers.py b/tests/unit/src/sqlbuild/compiler/auditing/main/helpers.py index 35dddf2ec8..2be628ed6d 100644 --- a/tests/unit/src/sqlbuild/compiler/auditing/main/helpers.py +++ b/tests/unit/src/sqlbuild/compiler/auditing/main/helpers.py @@ -7,7 +7,7 @@ from types import MappingProxyType from typing import cast -from sqlbuild._native import SchemaAuditInstance, SourceEntry, parse_audit_instances +from sqlbuild._native import AuditPlanEntry, SchemaAuditInstance, SourceEntry, parse_audit_instances from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter from sqlbuild.adapter.discovery.main.builtins import builtin_adapter_classes from sqlbuild.compiler.auditing.types import ( @@ -17,7 +17,6 @@ ) from sqlbuild.compiler.compile.models import CompiledObjectKey, CompiledRelationLocation from sqlbuild.compiler.compile.types import CompiledResourceType -from sqlbuild.compiler.planner.models import AuditPlanEntry def build_render_adapter(adapter_name: str | None) -> BaseAdapter | None: diff --git a/tests/unit/src/sqlbuild/compiler/auditing/main/test_identity.py b/tests/unit/src/sqlbuild/compiler/auditing/main/test_identity.py index 26ddcceb12..5d359f92c4 100644 --- a/tests/unit/src/sqlbuild/compiler/auditing/main/test_identity.py +++ b/tests/unit/src/sqlbuild/compiler/auditing/main/test_identity.py @@ -9,6 +9,7 @@ from sqlbuild._native import ( AuditGateIdentity, AuditIdentity, + AuditPlanEntry, MeasurementThresholdBound, MeasurementThresholds, build_audit_gate_identity, @@ -20,7 +21,6 @@ AuditSeverity, ThresholdOperator, ) -from sqlbuild.compiler.planner.models import AuditPlanEntry from tests.unit.src.sqlbuild.compiler.auditing.main._test_types import ( AuditGateAggregateIdentityTestCase, AuditGateIdentityTestCase, diff --git a/tests/unit/src/sqlbuild/compiler/compile/_helpers/test_microbatch_redesign.py b/tests/unit/src/sqlbuild/compiler/compile/_helpers/test_microbatch_redesign.py index 0be866e8d0..ea02a03e59 100644 --- a/tests/unit/src/sqlbuild/compiler/compile/_helpers/test_microbatch_redesign.py +++ b/tests/unit/src/sqlbuild/compiler/compile/_helpers/test_microbatch_redesign.py @@ -6,14 +6,16 @@ from sqlbuild._native import ( CompileModelConfig, + CursorBounds, + CursorInputRelation, DateValue, + ModelCursorSnapshot, TimestampValue, compute_cursor_bounds, resolve_effective_timestamp_grain, watermark_type_is_compatible, ) from sqlbuild.compiler.compile.exceptions import CompileInputError -from sqlbuild.compiler.planner.models import CursorBounds, CursorInputRelation, ModelCursorSnapshot from sqlbuild.executor.run._helpers.validation.cursor_bounds import ( resolve_effective_timestamp_grain as resolve_runtime_effective_timestamp_grain, ) diff --git a/tests/unit/src/sqlbuild/compiler/manifest/_test_types.py b/tests/unit/src/sqlbuild/compiler/manifest/_test_types.py index 1974575b1c..8aee802a07 100644 --- a/tests/unit/src/sqlbuild/compiler/manifest/_test_types.py +++ b/tests/unit/src/sqlbuild/compiler/manifest/_test_types.py @@ -1,10 +1,10 @@ from dataclasses import dataclass, field +from sqlbuild._native import ModelPlanEntry, PlanOutput from sqlbuild.compiler.compile.models import ( CompiledObjectKey, ) from sqlbuild.compiler.compile.types import CompiledModelView, CompiledProjectView -from sqlbuild.compiler.planner.models import ModelPlanEntry, PlanOutput @dataclass(frozen=True) diff --git a/tests/unit/src/sqlbuild/compiler/manifest/helpers.py b/tests/unit/src/sqlbuild/compiler/manifest/helpers.py index b408195099..5264fb5851 100644 --- a/tests/unit/src/sqlbuild/compiler/manifest/helpers.py +++ b/tests/unit/src/sqlbuild/compiler/manifest/helpers.py @@ -7,17 +7,26 @@ from typing import Any, cast from sqlbuild._native import ( + AuditPlanEntry, + ChainStep, CompiledSeed, CompiledSource, CompileModelConfig, DiscoveredSchemaFile, DiscoveredSeedFile, DiscoveredSqlHookFile, + MaterializationType, + ModelPlanEntry, + PlanAction, + PlanOutput, + PlanReason, SchemaColumn, SchemaModelEntry, SchemaSeedEntry, + SeedPlanEntry, SourceColumnEntry, SourceEntry, + SqlTestPlanEntry, build_manifest, ) from sqlbuild.compiler.auditing.types import ( @@ -40,19 +49,6 @@ CompiledSourceView, ) from sqlbuild.compiler.discovery.models import DiscoveredSourceFile -from sqlbuild.compiler.planner.models import ( - AuditPlanEntry, - ChainStep, - ModelPlanEntry, - PlanOutput, - SeedPlanEntry, - SqlTestPlanEntry, -) -from sqlbuild.compiler.planner.types import ( - MaterializationType, - PlanAction, - PlanReason, -) def model_key(name: str) -> CompiledObjectKey: diff --git a/tests/unit/src/sqlbuild/compiler/manifest/test_main.py b/tests/unit/src/sqlbuild/compiler/manifest/test_main.py index de7ac3479e..dfa53b03e7 100644 --- a/tests/unit/src/sqlbuild/compiler/manifest/test_main.py +++ b/tests/unit/src/sqlbuild/compiler/manifest/test_main.py @@ -9,17 +9,21 @@ import sqlbuild._native as _native from sqlbuild._native import ( + AuditPlanEntry, + ChainStep, DeclarationIdentity, DeclarationKind, DeclarationRecord, DiscoveredSqlHookFile, MacroMetadata, OwnershipRoot, + PlanOutput, ResourceKind, SchemaColumn, ScopeIndex, ScopeKind, SourceColumnEntry, + SqlTestPlanEntry, build_manifest, project_with, ) @@ -34,7 +38,6 @@ CompiledSourceView, ) from sqlbuild.compiler.discovery.models import SqlHookEntry -from sqlbuild.compiler.planner.models import AuditPlanEntry, ChainStep, PlanOutput, SqlTestPlanEntry from tests.unit.src.sqlbuild.compiler.manifest._test_types import ( FqnTestCase, ManifestAuditNodeTestCase, diff --git a/tests/unit/src/sqlbuild/compiler/migrations/main/_test_types.py b/tests/unit/src/sqlbuild/compiler/migrations/main/_test_types.py index 28e1acf3db..48a52f2a35 100644 --- a/tests/unit/src/sqlbuild/compiler/migrations/main/_test_types.py +++ b/tests/unit/src/sqlbuild/compiler/migrations/main/_test_types.py @@ -4,8 +4,7 @@ from dataclasses import dataclass -from sqlbuild.compiler.migrations.models import MigrationRelation -from sqlbuild.compiler.migrations.types import OldNameViewDropReason, OldNameViewEventType +from sqlbuild._native import MigrationRelation, OldNameViewDropReason, OldNameViewEventType @dataclass(frozen=True) diff --git a/tests/unit/src/sqlbuild/compiler/migrations/main/helpers.py b/tests/unit/src/sqlbuild/compiler/migrations/main/helpers.py index 89b08c4e20..0667c33bbb 100644 --- a/tests/unit/src/sqlbuild/compiler/migrations/main/helpers.py +++ b/tests/unit/src/sqlbuild/compiler/migrations/main/helpers.py @@ -5,21 +5,17 @@ from datetime import UTC, datetime, timedelta from sqlbuild._native import ( - MigrationDiscovery, - deterministic_column_migration_event_id, - deterministic_migration_event_id, - deterministic_old_name_view_event_id, -) -from sqlbuild.compiler.migrations.models import ( + ColumnMigrationDecision, ColumnMigrationEvent, + MigrationDecision, + MigrationDiscovery, MigrationEvent, MigrationRelation, OldNameViewEvent, -) -from sqlbuild.compiler.migrations.types import ( - ColumnMigrationDecision, - MigrationDecision, OldNameViewEventType, + deterministic_column_migration_event_id, + deterministic_migration_event_id, + deterministic_old_name_view_event_id, ) _STARTED_AT: datetime = datetime(2026, 1, 1, tzinfo=UTC) diff --git a/tests/unit/src/sqlbuild/compiler/migrations/main/test_column_migration_events.py b/tests/unit/src/sqlbuild/compiler/migrations/main/test_column_migration_events.py index 916b3c6873..813e179bff 100644 --- a/tests/unit/src/sqlbuild/compiler/migrations/main/test_column_migration_events.py +++ b/tests/unit/src/sqlbuild/compiler/migrations/main/test_column_migration_events.py @@ -6,12 +6,12 @@ import pytest from sqlbuild._native import ( + ColumnMigrationEvent, newest_column_migration_event_mentioning, read_migration_state_events, write_migration_state_event, ) from sqlbuild.adapters.duckdb.classes.duckdb_adapter import DuckDbAdapter -from sqlbuild.compiler.migrations.models import ColumnMigrationEvent from tests.unit.src.sqlbuild.compiler.migrations.main._test_types import ( ColumnEventStorageTestCase, NewestColumnEventTestCase, diff --git a/tests/unit/src/sqlbuild/compiler/migrations/main/test_migration_events.py b/tests/unit/src/sqlbuild/compiler/migrations/main/test_migration_events.py index 7c1e77718d..3ba92ab6bb 100644 --- a/tests/unit/src/sqlbuild/compiler/migrations/main/test_migration_events.py +++ b/tests/unit/src/sqlbuild/compiler/migrations/main/test_migration_events.py @@ -6,12 +6,12 @@ import pytest from sqlbuild._native import ( + MigrationEvent, newest_migration_event_mentioning, read_migration_state_events, write_migration_state_event, ) from sqlbuild.adapters.duckdb.classes.duckdb_adapter import DuckDbAdapter -from sqlbuild.compiler.migrations.models import MigrationEvent from tests.unit.src.sqlbuild.compiler.migrations.main._test_types import ( NewestEventTestCase, RepeatedEventWriteTestCase, diff --git a/tests/unit/src/sqlbuild/compiler/migrations/main/test_old_name_view_events.py b/tests/unit/src/sqlbuild/compiler/migrations/main/test_old_name_view_events.py index eb25c85b74..7c2bb93826 100644 --- a/tests/unit/src/sqlbuild/compiler/migrations/main/test_old_name_view_events.py +++ b/tests/unit/src/sqlbuild/compiler/migrations/main/test_old_name_view_events.py @@ -8,19 +8,17 @@ import pytest from sqlbuild._native import ( + MigrationEvent, MigrationStateInspection, + OldNameViewEvent, + OldNameViewEventType, + OldNameViewHistory, project_old_name_views, read_migration_state_events, stored_old_name_view_columns, write_migration_state_event, ) from sqlbuild.adapters.duckdb.classes.duckdb_adapter import DuckDbAdapter -from sqlbuild.compiler.migrations.models import ( - MigrationEvent, - OldNameViewEvent, - OldNameViewHistory, -) -from sqlbuild.compiler.migrations.types import OldNameViewEventType from tests.unit.src.sqlbuild.compiler.migrations.main._test_types import ( OldNameViewAbsentColumnsTestCase, OldNameViewProjectionTestCase, diff --git a/tests/unit/src/sqlbuild/compiler/migrations/main/test_pinned_event_ids.py b/tests/unit/src/sqlbuild/compiler/migrations/main/test_pinned_event_ids.py index e873a56e4a..8626cdb977 100644 --- a/tests/unit/src/sqlbuild/compiler/migrations/main/test_pinned_event_ids.py +++ b/tests/unit/src/sqlbuild/compiler/migrations/main/test_pinned_event_ids.py @@ -3,13 +3,13 @@ import pytest from sqlbuild._native import ( + MigrationRelation, + OldNameViewEventType, deterministic_column_migration_event_id, deterministic_migration_event_id, deterministic_old_name_view_event_id, source_freshness_data_version_hash, ) -from sqlbuild.compiler.migrations.models import MigrationRelation -from sqlbuild.compiler.migrations.types import OldNameViewEventType from tests.unit.src.sqlbuild.compiler.migrations.main._test_types import ( PinnedColumnEventIdTestCase, PinnedFreshnessHashTestCase, diff --git a/tests/unit/src/sqlbuild/compiler/migrations/main/test_stored_event_rows.py b/tests/unit/src/sqlbuild/compiler/migrations/main/test_stored_event_rows.py index 0b18656d46..70d8a7107e 100644 --- a/tests/unit/src/sqlbuild/compiler/migrations/main/test_stored_event_rows.py +++ b/tests/unit/src/sqlbuild/compiler/migrations/main/test_stored_event_rows.py @@ -7,10 +7,14 @@ import duckdb import pytest -from sqlbuild._native import read_migration_state_events, stored_old_name_view_columns +from sqlbuild._native import ( + ColumnMigrationEvent, + OldNameViewDropReason, + OldNameViewEvent, + read_migration_state_events, + stored_old_name_view_columns, +) from sqlbuild.adapters.duckdb.classes.duckdb_adapter import DuckDbAdapter -from sqlbuild.compiler.migrations.models import ColumnMigrationEvent, OldNameViewEvent -from sqlbuild.compiler.migrations.types import OldNameViewDropReason from tests.unit.src.sqlbuild.compiler.migrations.main._test_types import ( StoredColumnRowTestCase, StoredOldNameRowTestCase, diff --git a/tests/unit/src/sqlbuild/compiler/pipeline/main/_test_types.py b/tests/unit/src/sqlbuild/compiler/pipeline/main/_test_types.py index 794f94504b..1c39fe8ed9 100644 --- a/tests/unit/src/sqlbuild/compiler/pipeline/main/_test_types.py +++ b/tests/unit/src/sqlbuild/compiler/pipeline/main/_test_types.py @@ -1,7 +1,6 @@ from dataclasses import dataclass -from sqlbuild._native import PythonPlanEntry -from sqlbuild.compiler.planner.models import PlanOutput +from sqlbuild._native import PlanOutput, PythonPlanEntry from sqlbuild.python_nodes.models import SqlResourceRef diff --git a/tests/unit/src/sqlbuild/compiler/pipeline/main/helpers.py b/tests/unit/src/sqlbuild/compiler/pipeline/main/helpers.py index a465467b65..2efd3e6ace 100644 --- a/tests/unit/src/sqlbuild/compiler/pipeline/main/helpers.py +++ b/tests/unit/src/sqlbuild/compiler/pipeline/main/helpers.py @@ -3,7 +3,15 @@ from pathlib import Path from typing import Any, ClassVar -from sqlbuild._native import CompiledSource, SourceEntry +from sqlbuild._native import ( + CompiledSource, + MaterializationType, + ModelPlanEntry, + PlanAction, + PlanOutput, + PlanReason, + SourceEntry, +) from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter from sqlbuild.compiler.compile.models import ( CompiledObjectKey, @@ -12,8 +20,6 @@ ) from sqlbuild.compiler.compile.types import CompiledProjectView, CompiledResourceType from sqlbuild.compiler.discovery.models import DiscoveredSourceFile -from sqlbuild.compiler.planner.models import ModelPlanEntry, PlanOutput -from sqlbuild.compiler.planner.types import MaterializationType, PlanAction, PlanReason class RelationTargetTestAdapter(BaseAdapter): diff --git a/tests/unit/src/sqlbuild/compiler/pipeline/main/test_compile_phase_timings.py b/tests/unit/src/sqlbuild/compiler/pipeline/main/test_compile_phase_timings.py index 0da2b02e2e..361c760458 100644 --- a/tests/unit/src/sqlbuild/compiler/pipeline/main/test_compile_phase_timings.py +++ b/tests/unit/src/sqlbuild/compiler/pipeline/main/test_compile_phase_timings.py @@ -13,13 +13,13 @@ BuildPhaseTimingTracker, CompilePipelineOptions, PartialBuildPhaseTimings, + PlanOutput, RulesRunResult, ) from sqlbuild.compiler.compile.models import CompiledProject from sqlbuild.compiler.compile.types import CompiledProjectView from sqlbuild.compiler.pipeline._helpers import materializations from sqlbuild.compiler.pipeline.models import CompilePipelineResult -from sqlbuild.compiler.planner.models import PlanOutput from sqlbuild.runtime.contracts.models import ConnectionHooks from tests.unit.src.sqlbuild.compiler.pipeline.main._test_types import PipelinePhaseTimingTestCase diff --git a/tests/unit/src/sqlbuild/compiler/pipeline/main/test_plan_work.py b/tests/unit/src/sqlbuild/compiler/pipeline/main/test_plan_work.py index af5e7ec7e8..ef0abd1472 100644 --- a/tests/unit/src/sqlbuild/compiler/pipeline/main/test_plan_work.py +++ b/tests/unit/src/sqlbuild/compiler/pipeline/main/test_plan_work.py @@ -3,12 +3,12 @@ import pytest from sqlbuild._native import ( + PlanOutput, PythonIdentityStatus, PythonPlanEntry, PythonRunPhase, plan_has_executable_work, ) -from sqlbuild.compiler.planner.models import PlanOutput from sqlbuild.compiler.python_nodes.types import PythonNodeKind from tests.unit.src.sqlbuild.compiler.pipeline.main._test_types import PlanWorkTestCase from tests.unit.src.sqlbuild.compiler.pipeline.main.helpers import build_plan_output_with_model diff --git a/tests/unit/src/sqlbuild/compiler/pipeline/main/test_relation_targets.py b/tests/unit/src/sqlbuild/compiler/pipeline/main/test_relation_targets.py index 0bcce0d858..5f4ae4e612 100644 --- a/tests/unit/src/sqlbuild/compiler/pipeline/main/test_relation_targets.py +++ b/tests/unit/src/sqlbuild/compiler/pipeline/main/test_relation_targets.py @@ -6,12 +6,12 @@ from sqlbuild._native import ( DiscoveredPythonNode, + PlanOutput, PythonNodeGraph, SourceEntry, build_python_relation_targets, ) from sqlbuild.compiler.compile.types import CompiledProjectView -from sqlbuild.compiler.planner.models import PlanOutput from sqlbuild.compiler.python_nodes.types import PythonNodeKind from sqlbuild.python_nodes.models import SqlResourceRef from sqlbuild.refs import model, source diff --git a/tests/unit/src/sqlbuild/compiler/planner/_helpers/_test_types.py b/tests/unit/src/sqlbuild/compiler/planner/_helpers/_test_types.py index dd66df0f50..95f6312495 100644 --- a/tests/unit/src/sqlbuild/compiler/planner/_helpers/_test_types.py +++ b/tests/unit/src/sqlbuild/compiler/planner/_helpers/_test_types.py @@ -3,16 +3,25 @@ from pathlib import Path from sqlbuild._native import ( + BackfillAction, ChangeKind, + CursorBounds, Fingerprint, + MaterializationType, MissingUpstream, + OnSchemaChange, ParsedSelector, PathSelector, + PlanAction, + PlanReason, PlanWarning, ScenarioArtifactIdentity, ScenarioGraphPlan, + ScenarioRelationMap, SchemaAction, + SchemaFinding, SourceEntry, + SourceLoadPlanEntry, SqlLexicalSyntax, WarningSeverity, ) @@ -26,19 +35,6 @@ InferredColumn, ) from sqlbuild.compiler.compile.types import AttachedAuditTargetKind, CompiledProjectView -from sqlbuild.compiler.planner.models import ( - CursorBounds, - ScenarioRelationMap, - SchemaFinding, - SourceLoadPlanEntry, -) -from sqlbuild.compiler.planner.types import ( - BackfillAction, - MaterializationType, - OnSchemaChange, - PlanAction, - PlanReason, -) from sqlbuild.compiler.references.types import SqlReferenceKind from tests.unit.src.sqlbuild.compiler.planner._helpers.sql_test_assembly._test_types import ( PlanTestChainTestCase as PlanTestChainTestCase, diff --git a/tests/unit/src/sqlbuild/compiler/planner/_helpers/changes/_test_types.py b/tests/unit/src/sqlbuild/compiler/planner/_helpers/changes/_test_types.py index e5ebba7ad7..7d87bf60a9 100644 --- a/tests/unit/src/sqlbuild/compiler/planner/_helpers/changes/_test_types.py +++ b/tests/unit/src/sqlbuild/compiler/planner/_helpers/changes/_test_types.py @@ -1,21 +1,17 @@ from dataclasses import dataclass, field from sqlbuild._native import ( + BackfillAction, BackfillResult, ChangeKind, + PlanAction, + PlanReason, ReferenceRename, SchemaDynamicColumnFamily, + SchemaFinding, ) from sqlbuild.adapter.contract.models import ColumnInfo from sqlbuild.compiler.compile.models import InferredColumn -from sqlbuild.compiler.planner.models import ( - SchemaFinding, -) -from sqlbuild.compiler.planner.types import ( - BackfillAction, - PlanAction, - PlanReason, -) @dataclass(frozen=True) diff --git a/tests/unit/src/sqlbuild/compiler/planner/_helpers/changes/test_detect.py b/tests/unit/src/sqlbuild/compiler/planner/_helpers/changes/test_detect.py index 7fd6a56d13..5263aa3ea9 100644 --- a/tests/unit/src/sqlbuild/compiler/planner/_helpers/changes/test_detect.py +++ b/tests/unit/src/sqlbuild/compiler/planner/_helpers/changes/test_detect.py @@ -6,12 +6,15 @@ import pytest from sqlbuild._native import ( + BackfillAction, BackfillResult, ChangeDetectionResult, ChangeKind, Fingerprint, + PlanAction, PlannerChangeResults, PlannerScope, + PlanReason, PlanWarning, WarehouseFingerprints, WarehouseSnapshot, @@ -27,11 +30,6 @@ from sqlbuild.compiler.compile.models import InferredColumn from sqlbuild.compiler.compile.types import CompiledModelView from sqlbuild.compiler.discovery.models import SqlHookEntry -from sqlbuild.compiler.planner.types import ( - BackfillAction, - PlanAction, - PlanReason, -) from tests.unit.src.sqlbuild.compiler.planner._helpers.changes._test_helpers import ( build_function_caller_metadata_case, build_metadata_json_with_audit_gate, diff --git a/tests/unit/src/sqlbuild/compiler/planner/_helpers/changes/test_schema.py b/tests/unit/src/sqlbuild/compiler/planner/_helpers/changes/test_schema.py index 0cbf307b5f..b0e45d6a94 100644 --- a/tests/unit/src/sqlbuild/compiler/planner/_helpers/changes/test_schema.py +++ b/tests/unit/src/sqlbuild/compiler/planner/_helpers/changes/test_schema.py @@ -6,11 +6,11 @@ SchemaChangeKind, SchemaColumnSource, SchemaDynamicColumnFamily, + SchemaFinding, detect_schema_changes, ) from sqlbuild.adapter.contract.models import ColumnInfo from sqlbuild.compiler.compile.models import InferredColumn -from sqlbuild.compiler.planner.models import SchemaFinding from tests.unit.src.sqlbuild.compiler.planner._helpers.changes._test_types import ( DetectSchemaChangesTestCase, ) diff --git a/tests/unit/src/sqlbuild/compiler/planner/_helpers/helpers.py b/tests/unit/src/sqlbuild/compiler/planner/_helpers/helpers.py index 5d9a86562b..027529c245 100644 --- a/tests/unit/src/sqlbuild/compiler/planner/_helpers/helpers.py +++ b/tests/unit/src/sqlbuild/compiler/planner/_helpers/helpers.py @@ -13,6 +13,7 @@ import sqlbuild._native as _native from sqlbuild._native import ( + BackfillAction, BackfillResult, ChangeDetectionResult, CompiledDirectLogicSqlTestPayload, @@ -32,7 +33,10 @@ Fingerprint, FunctionArgument, PlannerChangeResults, + PlanOutput, + PlanReason, ScenarioArtifactIdentity, + ScenarioRelationMap, SchemaColumn, SchemaModelEntry, SchemaSeedEntry, @@ -81,14 +85,6 @@ DiscoveredSqlTestFile, ) from sqlbuild.compiler.pipeline.models import CompilePipelineResult -from sqlbuild.compiler.planner.models import ( - PlanOutput, - ScenarioRelationMap, -) -from sqlbuild.compiler.planner.types import ( - BackfillAction, - PlanReason, -) from sqlbuild.compiler.references.types import SqlReferenceKind from tests.unit.src.sqlbuild.compiler.planner._helpers._test_types import ( BuildModelWarningsTestCase, diff --git a/tests/unit/src/sqlbuild/compiler/planner/_helpers/identity/_test_types.py b/tests/unit/src/sqlbuild/compiler/planner/_helpers/identity/_test_types.py index fbb009e40f..7991d7f842 100644 --- a/tests/unit/src/sqlbuild/compiler/planner/_helpers/identity/_test_types.py +++ b/tests/unit/src/sqlbuild/compiler/planner/_helpers/identity/_test_types.py @@ -3,10 +3,8 @@ from dataclasses import dataclass from sqlbuild._native import ( - ChangeKind, -) -from sqlbuild.compiler.planner.types import ( BackfillAction, + ChangeKind, ) diff --git a/tests/unit/src/sqlbuild/compiler/planner/_helpers/identity/helpers.py b/tests/unit/src/sqlbuild/compiler/planner/_helpers/identity/helpers.py index 3ac9251cf6..a7df66545e 100644 --- a/tests/unit/src/sqlbuild/compiler/planner/_helpers/identity/helpers.py +++ b/tests/unit/src/sqlbuild/compiler/planner/_helpers/identity/helpers.py @@ -3,15 +3,13 @@ from __future__ import annotations from sqlbuild._native import ( + BackfillAction, BackfillResult, ChangeDetectionResult, ChangeKind, PlannerResolvedActions, ResolvedModelAction, ) -from sqlbuild.compiler.planner.types import ( - BackfillAction, -) MERGE_MODEL_NAME: str = "orders" MERGE_RESOLVED_HASH: str = "resolved-hash" diff --git a/tests/unit/src/sqlbuild/compiler/planner/_helpers/identity/test_definition_hashes.py b/tests/unit/src/sqlbuild/compiler/planner/_helpers/identity/test_definition_hashes.py index 8eba85f9f6..dd6e604a02 100644 --- a/tests/unit/src/sqlbuild/compiler/planner/_helpers/identity/test_definition_hashes.py +++ b/tests/unit/src/sqlbuild/compiler/planner/_helpers/identity/test_definition_hashes.py @@ -6,8 +6,7 @@ import pytest -from sqlbuild._native import build_function_local_hashes, model_definition_hash -from sqlbuild.compiler.planner.exceptions import PlannerInputError +from sqlbuild._native import PlannerInputError, build_function_local_hashes, model_definition_hash from tests.unit.src.sqlbuild.compiler.planner._helpers.helpers import build_compiled_function from tests.unit.src.sqlbuild.compiler.planner._helpers.identity._test_types import ( DefinitionHashFailureTestCase, diff --git a/tests/unit/src/sqlbuild/compiler/planner/_helpers/identity/test_merge_recomputed_model_changes.py b/tests/unit/src/sqlbuild/compiler/planner/_helpers/identity/test_merge_recomputed_model_changes.py index 0918c63812..b4e7f84f74 100644 --- a/tests/unit/src/sqlbuild/compiler/planner/_helpers/identity/test_merge_recomputed_model_changes.py +++ b/tests/unit/src/sqlbuild/compiler/planner/_helpers/identity/test_merge_recomputed_model_changes.py @@ -3,6 +3,7 @@ import pytest from sqlbuild._native import ( + BackfillAction, BackfillResult, ChangeDetectionResult, ChangeKind, @@ -11,9 +12,6 @@ ResolvedModelAction, merge_recomputed_model_changes, ) -from sqlbuild.compiler.planner.types import ( - BackfillAction, -) from tests.unit.src.sqlbuild.compiler.planner._helpers.identity._test_types import ( MergeRecomputedModelChangesTestCase, ) diff --git a/tests/unit/src/sqlbuild/compiler/planner/_helpers/migrations/helpers.py b/tests/unit/src/sqlbuild/compiler/planner/_helpers/migrations/helpers.py index 5a1712b79f..80c999e576 100644 --- a/tests/unit/src/sqlbuild/compiler/planner/_helpers/migrations/helpers.py +++ b/tests/unit/src/sqlbuild/compiler/planner/_helpers/migrations/helpers.py @@ -8,14 +8,12 @@ from sqlbuild._native import ( OldNameViewAction, + OldNameViewPlanEntry, QueryShape, build_migration_fingerprint, parse_query_shape, ) from sqlbuild.compiler.compile.models import CompiledRelationLocation -from sqlbuild.compiler.planner.models import ( - OldNameViewPlanEntry, -) if TYPE_CHECKING: from sqlbuild._native import InputColumns diff --git a/tests/unit/src/sqlbuild/compiler/planner/_helpers/migrations/test_column_migration_candidates.py b/tests/unit/src/sqlbuild/compiler/planner/_helpers/migrations/test_column_migration_candidates.py index b2bdc06b7c..c0f9de84d6 100644 --- a/tests/unit/src/sqlbuild/compiler/planner/_helpers/migrations/test_column_migration_candidates.py +++ b/tests/unit/src/sqlbuild/compiler/planner/_helpers/migrations/test_column_migration_candidates.py @@ -12,6 +12,7 @@ CompileModelConfig, DeferralInputs, Fingerprint, + PlannerInputError, PlannerOverrides, PlannerRuntime, PlannerScope, @@ -27,7 +28,6 @@ CompiledRelationLocation, ) from sqlbuild.compiler.compile.types import CompiledModelView, CompiledResourceType -from sqlbuild.compiler.planner.exceptions import PlannerInputError from tests.unit.src.sqlbuild.compiler.planner._helpers.migrations._test_types import ( UnfingerprintableCandidateTestCase, ) diff --git a/tests/unit/src/sqlbuild/compiler/planner/_helpers/planning/_test_types.py b/tests/unit/src/sqlbuild/compiler/planner/_helpers/planning/_test_types.py index 3193713d2a..dbb864f001 100644 --- a/tests/unit/src/sqlbuild/compiler/planner/_helpers/planning/_test_types.py +++ b/tests/unit/src/sqlbuild/compiler/planner/_helpers/planning/_test_types.py @@ -4,11 +4,12 @@ from sqlbuild._native import ( ChangeDetectionResult, + RetentionDirection, + RetentionPlanPhase, TableType, TableTypeDowngradePolicy, ) from sqlbuild.adapter.contract.models import RetentionState -from sqlbuild.compiler.planner.types import RetentionDirection, RetentionPlanPhase @dataclass(frozen=True) diff --git a/tests/unit/src/sqlbuild/compiler/planner/_helpers/planning/test_full_refresh.py b/tests/unit/src/sqlbuild/compiler/planner/_helpers/planning/test_full_refresh.py index 56d3145fc6..d2dcff5f74 100644 --- a/tests/unit/src/sqlbuild/compiler/planner/_helpers/planning/test_full_refresh.py +++ b/tests/unit/src/sqlbuild/compiler/planner/_helpers/planning/test_full_refresh.py @@ -5,16 +5,14 @@ import pytest from sqlbuild._native import ( + BackfillAction, BackfillResult, ChangeDetectionResult, ChangeKind, + PlannerInputError, check_full_rebuild_allowed, describe_full_rebuild_cause, ) -from sqlbuild.compiler.planner.exceptions import PlannerInputError -from sqlbuild.compiler.planner.types import ( - BackfillAction, -) from tests.unit.src.sqlbuild.compiler.planner._helpers.planning._test_types import ( FullRebuildCheckTestCase, ) diff --git a/tests/unit/src/sqlbuild/compiler/planner/_helpers/planning/test_retention.py b/tests/unit/src/sqlbuild/compiler/planner/_helpers/planning/test_retention.py index d15cc6fbc7..02f7d206a5 100644 --- a/tests/unit/src/sqlbuild/compiler/planner/_helpers/planning/test_retention.py +++ b/tests/unit/src/sqlbuild/compiler/planner/_helpers/planning/test_retention.py @@ -9,13 +9,19 @@ from sqlbuild._native import ( Fingerprint, LocalConfig, + MaterializationType, + PlannerInputError, PlannerRuntime, PlannerScope, PlannerWarehouseState, ProjectConfig, ResolvedTableType, + RetentionDirection, + RetentionPlanEntry, + RetentionPlanPhase, TableType, TableTypeDowngradePolicy, + TableTypePlanEntry, TableTypeSource, TargetConfig, WarehouseFingerprints, @@ -33,13 +39,6 @@ RetentionScope, ) from sqlbuild.adapters.snowflake.classes.snowflake_adapter import SnowflakeAdapter -from sqlbuild.compiler.planner.exceptions import PlannerInputError -from sqlbuild.compiler.planner.models import RetentionPlanEntry, TableTypePlanEntry -from sqlbuild.compiler.planner.types import ( - MaterializationType, - RetentionDirection, - RetentionPlanPhase, -) from tests.unit.src.sqlbuild.compiler.planner._helpers.planning._test_types import ( ForeignSnapshotRelationTestCase, RetentionPlanningErrorTestCase, diff --git a/tests/unit/src/sqlbuild/compiler/planner/_helpers/resolve/_test_types.py b/tests/unit/src/sqlbuild/compiler/planner/_helpers/resolve/_test_types.py index 419550374e..5793219d9f 100644 --- a/tests/unit/src/sqlbuild/compiler/planner/_helpers/resolve/_test_types.py +++ b/tests/unit/src/sqlbuild/compiler/planner/_helpers/resolve/_test_types.py @@ -5,9 +5,8 @@ from dataclasses import dataclass, field from datetime import date, datetime -from sqlbuild._native import SourceEntry +from sqlbuild._native import CursorBounds, CursorOverrides, ModelCursorSnapshot, SourceEntry from sqlbuild.adapter.contract.models import ColumnInfo -from sqlbuild.compiler.planner.models import CursorBounds, CursorOverrides, ModelCursorSnapshot @dataclass(frozen=True) diff --git a/tests/unit/src/sqlbuild/compiler/planner/_helpers/resolve/test_cursor.py b/tests/unit/src/sqlbuild/compiler/planner/_helpers/resolve/test_cursor.py index 790c2aba2f..cd72bfbf39 100644 --- a/tests/unit/src/sqlbuild/compiler/planner/_helpers/resolve/test_cursor.py +++ b/tests/unit/src/sqlbuild/compiler/planner/_helpers/resolve/test_cursor.py @@ -9,8 +9,12 @@ from sqlbuild._native import ( AlignedInterval, + CursorBounds, + CursorGrain, + CursorType, CursorWatermarkMode, DateValue, + ModelCursorSnapshot, TimestampValue, apply_typed_cursor_replay_policy, compute_cursor_bounds, @@ -22,11 +26,6 @@ MICROBATCH_END_SENTINEL, MICROBATCH_START_SENTINEL, ) -from sqlbuild.compiler.planner.models import CursorBounds, ModelCursorSnapshot -from sqlbuild.compiler.planner.types import ( - CursorGrain, - CursorType, -) from sqlbuild.cursor_algebra.types import CursorScalar from tests.unit.src.sqlbuild.compiler.planner._helpers.resolve._test_types import ( CursorBoundsTestCase, diff --git a/tests/unit/src/sqlbuild/compiler/planner/_helpers/resolve/test_cursor_intrinsics.py b/tests/unit/src/sqlbuild/compiler/planner/_helpers/resolve/test_cursor_intrinsics.py index 30324d6023..5ab6298183 100644 --- a/tests/unit/src/sqlbuild/compiler/planner/_helpers/resolve/test_cursor_intrinsics.py +++ b/tests/unit/src/sqlbuild/compiler/planner/_helpers/resolve/test_cursor_intrinsics.py @@ -3,18 +3,16 @@ import pytest from sqlbuild._native import ( + BackfillAction, BackfillResult, CursorOverridePair, + ModelCursorSnapshot, + PlannerInputError, WarehouseSnapshot, resolve_model_sql, ) from sqlbuild.adapters.duckdb.classes.duckdb_adapter import DuckDbAdapter from sqlbuild.compiler.compile.types import CompiledModelView -from sqlbuild.compiler.planner.exceptions import PlannerInputError -from sqlbuild.compiler.planner.models import ( - ModelCursorSnapshot, -) -from sqlbuild.compiler.planner.types import BackfillAction from tests.unit.src.sqlbuild.compiler.planner._helpers.resolve._test_types import ( CursorIntrinsicResolutionTestCase, ) diff --git a/tests/unit/src/sqlbuild/compiler/planner/_helpers/resolve/test_cursor_override_types.py b/tests/unit/src/sqlbuild/compiler/planner/_helpers/resolve/test_cursor_override_types.py index 19522990df..3c5a26a036 100644 --- a/tests/unit/src/sqlbuild/compiler/planner/_helpers/resolve/test_cursor_override_types.py +++ b/tests/unit/src/sqlbuild/compiler/planner/_helpers/resolve/test_cursor_override_types.py @@ -4,9 +4,7 @@ import pytest -from sqlbuild._native import type_cursor_overrides -from sqlbuild.compiler.planner.exceptions import PlannerInputError -from sqlbuild.compiler.planner.models import CursorOverrides +from sqlbuild._native import CursorOverrides, PlannerInputError, type_cursor_overrides from tests.unit.src.sqlbuild.compiler.planner._helpers.resolve._test_types import ( UntypedCursorOverrideErrorTestCase, UntypedCursorOverrideTestCase, diff --git a/tests/unit/src/sqlbuild/compiler/planner/_helpers/resolve/test_refs.py b/tests/unit/src/sqlbuild/compiler/planner/_helpers/resolve/test_refs.py index 9a83879c95..e47fe652dc 100644 --- a/tests/unit/src/sqlbuild/compiler/planner/_helpers/resolve/test_refs.py +++ b/tests/unit/src/sqlbuild/compiler/planner/_helpers/resolve/test_refs.py @@ -5,6 +5,7 @@ import pytest from sqlbuild._native import ( + CursorBounds, DbtCompileReferenceResolver, DbtManifestIndex, apply_deferred_locations, @@ -21,7 +22,6 @@ CompiledRelationLocation, ) from sqlbuild.compiler.compile.types import CompiledResourceType -from sqlbuild.compiler.planner.models import CursorBounds from tests.unit.src.sqlbuild.compiler.planner._helpers.resolve._test_types import ( ApplyDeferredTargetsTestCase, RefResolutionTestCase, diff --git a/tests/unit/src/sqlbuild/compiler/planner/_helpers/resolve/test_sources.py b/tests/unit/src/sqlbuild/compiler/planner/_helpers/resolve/test_sources.py index af90946cb8..5a8031c023 100644 --- a/tests/unit/src/sqlbuild/compiler/planner/_helpers/resolve/test_sources.py +++ b/tests/unit/src/sqlbuild/compiler/planner/_helpers/resolve/test_sources.py @@ -4,7 +4,7 @@ import pytest -from sqlbuild._native import SourceColumnEntry, SourceEntry, resolve_source_references +from sqlbuild._native import CursorBounds, SourceColumnEntry, SourceEntry, resolve_source_references from sqlbuild.adapter.contract.models import ColumnInfo from sqlbuild.adapter.contract.types import CursorKind from sqlbuild.adapters.bigquery.classes.bigquery_adapter import BigQueryAdapter @@ -12,7 +12,6 @@ from sqlbuild.adapters.postgres.classes.postgres_adapter import PostgresAdapter from sqlbuild.adapters.snowflake.classes.snowflake_adapter import SnowflakeAdapter from sqlbuild.adapters.sqlserver.classes.sqlserver_adapter import SqlServerAdapter -from sqlbuild.compiler.planner.models import CursorBounds from tests.unit.src.sqlbuild.compiler.planner._helpers.resolve._test_types import ( AdapterSourceResolutionTestCase, ReservedSourceColumnTestCase, diff --git a/tests/unit/src/sqlbuild/compiler/planner/_helpers/selection/test_unit_test_selectors.py b/tests/unit/src/sqlbuild/compiler/planner/_helpers/selection/test_unit_test_selectors.py index bff9a49c97..0bfda38ba4 100644 --- a/tests/unit/src/sqlbuild/compiler/planner/_helpers/selection/test_unit_test_selectors.py +++ b/tests/unit/src/sqlbuild/compiler/planner/_helpers/selection/test_unit_test_selectors.py @@ -5,12 +5,12 @@ import pytest from sqlbuild._native import ( + PlannerInputError, SqlTestSelection, UnitTestSelectorSplit, split_project_unit_test_selectors, ) from sqlbuild.compiler.discovery.models import DiscoveredProjectInputs -from sqlbuild.compiler.planner.exceptions import PlannerInputError from tests.unit.src.sqlbuild.compiler.planner._helpers.selection._test_types import ( UnitTestSelectorErrorTestCase, UnitTestSelectorSplitTestCase, diff --git a/tests/unit/src/sqlbuild/compiler/planner/_helpers/sql_test_assembly/helpers.py b/tests/unit/src/sqlbuild/compiler/planner/_helpers/sql_test_assembly/helpers.py index b4c972e37f..0cb86900ee 100644 --- a/tests/unit/src/sqlbuild/compiler/planner/_helpers/sql_test_assembly/helpers.py +++ b/tests/unit/src/sqlbuild/compiler/planner/_helpers/sql_test_assembly/helpers.py @@ -16,6 +16,7 @@ NativeSqlTestPlan, PlanWarning, SqlLexicalSyntax, + SqlTestPlanEntry, SqlTestPlanResult, expand_sql_macros, extract_sql_references, @@ -44,9 +45,6 @@ CompiledResourceType, ) from sqlbuild.compiler.discovery.models import DiscoveredSqlTestFile -from sqlbuild.compiler.planner.models import ( - SqlTestPlanEntry, -) from sqlbuild.compiler.references.types import SqlReferenceKind from sqlbuild.compiler.sql_analysis.main.import_polyglot_sql import import_polyglot_sql from tests.unit.src.sqlbuild.compiler.planner._helpers.sql_test_assembly._test_types import ( diff --git a/tests/unit/src/sqlbuild/compiler/planner/_helpers/sql_test_assembly/test_plan_test.py b/tests/unit/src/sqlbuild/compiler/planner/_helpers/sql_test_assembly/test_plan_test.py index 8299d95df0..905098b82f 100644 --- a/tests/unit/src/sqlbuild/compiler/planner/_helpers/sql_test_assembly/test_plan_test.py +++ b/tests/unit/src/sqlbuild/compiler/planner/_helpers/sql_test_assembly/test_plan_test.py @@ -6,15 +6,19 @@ import pytest from sqlbuild._native import ( + ChainStep, CompiledDirectLogicSqlTestPayload, CompiledFunction, CompiledModelSqlTestPayload, CompileSqlTestCte, DiscoveredSqlTestBlock, NativeSqlTestPlan, + PlannerInputError, PlannerScope, PlannerSelection, + PlanOutput, PlanWarning, + SqlTestPlanEntry, SqlTestPlanResult, WarningSeverity, build_test_command_plan, @@ -37,8 +41,6 @@ ) from sqlbuild.compiler.compile.types import CompiledProjectView, CompiledResourceType, SqlTestMode from sqlbuild.compiler.discovery.models import DiscoveredSqlTestFile -from sqlbuild.compiler.planner.exceptions import PlannerInputError -from sqlbuild.compiler.planner.models import ChainStep, PlanOutput, SqlTestPlanEntry from tests.unit.src.sqlbuild.compiler.planner._helpers.sql_test_assembly._test_types import ( AssertionChainCteErrorTestCase, NativePlanningDifferentialTestCase, diff --git a/tests/unit/src/sqlbuild/compiler/planner/_helpers/test_audit_entry.py b/tests/unit/src/sqlbuild/compiler/planner/_helpers/test_audit_entry.py index d45e637c1f..7694e02859 100644 --- a/tests/unit/src/sqlbuild/compiler/planner/_helpers/test_audit_entry.py +++ b/tests/unit/src/sqlbuild/compiler/planner/_helpers/test_audit_entry.py @@ -2,7 +2,7 @@ import pytest -from sqlbuild._native import SourceEntry, plan_audit +from sqlbuild._native import AuditPlanEntry, SourceEntry, plan_audit from sqlbuild.compiler.auditing.types import AuditAttachmentKind, AuditRunScope from sqlbuild.compiler.compile.models import ( CompiledObjectKey, @@ -10,7 +10,6 @@ CompileSqlReference, ) from sqlbuild.compiler.compile.types import AttachedAuditTargetKind, CompiledAuditView -from sqlbuild.compiler.planner.models import AuditPlanEntry from sqlbuild.compiler.references.types import SqlReferenceKind from tests.unit.src.sqlbuild.compiler.planner._helpers._test_types import ( PlanAttachedAuditTestCase, diff --git a/tests/unit/src/sqlbuild/compiler/planner/_helpers/test_audit_scheduling.py b/tests/unit/src/sqlbuild/compiler/planner/_helpers/test_audit_scheduling.py index 4264d405d9..4c3a44ea67 100644 --- a/tests/unit/src/sqlbuild/compiler/planner/_helpers/test_audit_scheduling.py +++ b/tests/unit/src/sqlbuild/compiler/planner/_helpers/test_audit_scheduling.py @@ -5,6 +5,7 @@ import pytest from sqlbuild._native import ( + MaterializationType, reads_outside_target_lineage, resolve_attachment_kind, resolve_effective_run_scope, @@ -21,7 +22,6 @@ AttachedAuditTargetKind, CompiledAuditView, ) -from sqlbuild.compiler.planner.types import MaterializationType from sqlbuild.compiler.references.types import SqlReferenceKind from tests.unit.src.sqlbuild.compiler.planner._helpers._test_types import ( ResolveAttachmentErrorTestCase, diff --git a/tests/unit/src/sqlbuild/compiler/planner/_helpers/test_cursor_override_resolution.py b/tests/unit/src/sqlbuild/compiler/planner/_helpers/test_cursor_override_resolution.py index 7ad94b1446..f17b1c62db 100644 --- a/tests/unit/src/sqlbuild/compiler/planner/_helpers/test_cursor_override_resolution.py +++ b/tests/unit/src/sqlbuild/compiler/planner/_helpers/test_cursor_override_resolution.py @@ -4,9 +4,8 @@ import pytest -from sqlbuild._native import resolve_cursor_overrides +from sqlbuild._native import CursorOverrides, resolve_cursor_overrides from sqlbuild.compiler.compile.types import CompiledModelView -from sqlbuild.compiler.planner.models import CursorOverrides from tests.unit.src.sqlbuild.compiler.planner._helpers._test_types import ( CursorOverrideResolutionTestCase, ) diff --git a/tests/unit/src/sqlbuild/compiler/planner/_helpers/test_cursor_overrides_validation.py b/tests/unit/src/sqlbuild/compiler/planner/_helpers/test_cursor_overrides_validation.py index 96b5261e91..360696a9c8 100644 --- a/tests/unit/src/sqlbuild/compiler/planner/_helpers/test_cursor_overrides_validation.py +++ b/tests/unit/src/sqlbuild/compiler/planner/_helpers/test_cursor_overrides_validation.py @@ -4,7 +4,7 @@ import pytest -from sqlbuild.compiler.planner.models import CursorOverrides +from sqlbuild._native import CursorOverrides from tests.unit.src.sqlbuild.compiler.planner._helpers._test_types import ( CursorOverridesValidationTestCase, ) diff --git a/tests/unit/src/sqlbuild/compiler/planner/_helpers/test_cursor_start.py b/tests/unit/src/sqlbuild/compiler/planner/_helpers/test_cursor_start.py index 351f03d67d..f2b8d5ca95 100644 --- a/tests/unit/src/sqlbuild/compiler/planner/_helpers/test_cursor_start.py +++ b/tests/unit/src/sqlbuild/compiler/planner/_helpers/test_cursor_start.py @@ -4,9 +4,8 @@ import pytest -from sqlbuild._native import compute_cursor_bounds +from sqlbuild._native import CursorBounds, ModelCursorSnapshot, compute_cursor_bounds from sqlbuild._native import cursor_sentinel_to_token as sentinel_to_token -from sqlbuild.compiler.planner.models import CursorBounds, ModelCursorSnapshot from tests.unit.src.sqlbuild.compiler.planner._helpers._test_types import ( CursorStartBoundsTestCase, ) diff --git a/tests/unit/src/sqlbuild/compiler/planner/_helpers/test_fixture_planning_work.py b/tests/unit/src/sqlbuild/compiler/planner/_helpers/test_fixture_planning_work.py index d0e79d1349..1aa809c355 100644 --- a/tests/unit/src/sqlbuild/compiler/planner/_helpers/test_fixture_planning_work.py +++ b/tests/unit/src/sqlbuild/compiler/planner/_helpers/test_fixture_planning_work.py @@ -14,6 +14,7 @@ RelationFixtureCompletion, RelationFixturePlanningContext, SettingsConfig, + SqlTestPlanEntry, SqlTestSelection, build_selected_test_entries, compiled_model_record, @@ -29,7 +30,6 @@ CompiledProjectView, CompiledResourceType, ) -from sqlbuild.compiler.planner.models import SqlTestPlanEntry from sqlbuild.compiler.references.types import SqlReferenceKind from tests.unit.src.sqlbuild.compiler.planner._helpers._test_types import ( CompleteFixturePlanningTestCase, diff --git a/tests/unit/src/sqlbuild/compiler/planner/_helpers/test_function_fingerprints.py b/tests/unit/src/sqlbuild/compiler/planner/_helpers/test_function_fingerprints.py index e46a99578a..d2d620d768 100644 --- a/tests/unit/src/sqlbuild/compiler/planner/_helpers/test_function_fingerprints.py +++ b/tests/unit/src/sqlbuild/compiler/planner/_helpers/test_function_fingerprints.py @@ -4,8 +4,8 @@ import pytest +from sqlbuild._native import PlanReason from sqlbuild.compiler.compile.types import CompiledFunctionView -from sqlbuild.compiler.planner.types import PlanReason from tests.unit.src.sqlbuild.compiler.planner._helpers._test_types import ( DetectFunctionChangeTestCase, ) diff --git a/tests/unit/src/sqlbuild/compiler/planner/_helpers/test_microbatch_limit.py b/tests/unit/src/sqlbuild/compiler/planner/_helpers/test_microbatch_limit.py index 8692e584a6..522b35b6e0 100644 --- a/tests/unit/src/sqlbuild/compiler/planner/_helpers/test_microbatch_limit.py +++ b/tests/unit/src/sqlbuild/compiler/planner/_helpers/test_microbatch_limit.py @@ -5,8 +5,15 @@ import pytest from sqlbuild._native import ( + CursorBounds, + CursorInputRelation, + IncrementalMode, + MaterializationType, MicrobatchLimitAction, + ModelPlanEntry, + PlanAction, PlanEntryBuildInputs, + PlanReason, PlanWarning, apply_configured_microbatch_limits, apply_microbatch_limit, @@ -18,17 +25,6 @@ CompiledRelationLocation, ) from sqlbuild.compiler.compile.types import CompiledResourceType -from sqlbuild.compiler.planner.models import ( - CursorBounds, - CursorInputRelation, - ModelPlanEntry, -) -from sqlbuild.compiler.planner.types import ( - IncrementalMode, - MaterializationType, - PlanAction, - PlanReason, -) from tests.unit.src.sqlbuild.compiler.planner._helpers._test_types import ( ConfiguredMicrobatchPolicyTestCase, EffectiveMicrobatchLimitPlanningTestCase, diff --git a/tests/unit/src/sqlbuild/compiler/planner/_helpers/test_microbatch_lookback.py b/tests/unit/src/sqlbuild/compiler/planner/_helpers/test_microbatch_lookback.py index 1a38e56bda..23d4d51949 100644 --- a/tests/unit/src/sqlbuild/compiler/planner/_helpers/test_microbatch_lookback.py +++ b/tests/unit/src/sqlbuild/compiler/planner/_helpers/test_microbatch_lookback.py @@ -4,9 +4,8 @@ import pytest -from sqlbuild._native import resolve_microbatch_lookback +from sqlbuild._native import IncrementalStrategy, resolve_microbatch_lookback from sqlbuild.compiler.compile.types import CompiledModelView -from sqlbuild.compiler.planner.types import IncrementalStrategy from tests.unit.src.sqlbuild.compiler.planner._helpers._test_types import ( MicrobatchLookbackTestCase, ) diff --git a/tests/unit/src/sqlbuild/compiler/planner/_helpers/test_plan_entry_cursor_overrides.py b/tests/unit/src/sqlbuild/compiler/planner/_helpers/test_plan_entry_cursor_overrides.py index 3d3f1430d8..c433a0e83a 100644 --- a/tests/unit/src/sqlbuild/compiler/planner/_helpers/test_plan_entry_cursor_overrides.py +++ b/tests/unit/src/sqlbuild/compiler/planner/_helpers/test_plan_entry_cursor_overrides.py @@ -8,8 +8,12 @@ import pytest from sqlbuild._native import ( + BackfillAction, BackfillResult, CompileModelConfig, + CursorBounds, + CursorInputRelation, + ModelCursorSnapshot, WarehouseSnapshot, compute_microbatch_range, compute_plan_cursor_bounds, @@ -22,12 +26,6 @@ ) from sqlbuild.compiler.compile.types import CompiledModelView, CompiledResourceType from sqlbuild.compiler.planner.constants import MICROBATCH_END_SENTINEL, MICROBATCH_START_SENTINEL -from sqlbuild.compiler.planner.models import ( - CursorBounds, - CursorInputRelation, - ModelCursorSnapshot, -) -from sqlbuild.compiler.planner.types import BackfillAction from tests.unit.src.sqlbuild.compiler.planner._helpers._test_types import ( AuthoritativeCursorOverrideTestCase, MicrobatchCursorEndPlanTestCase, diff --git a/tests/unit/src/sqlbuild/compiler/planner/_helpers/test_scenario_artifacts.py b/tests/unit/src/sqlbuild/compiler/planner/_helpers/test_scenario_artifacts.py index f874eeed7d..ab899d223f 100644 --- a/tests/unit/src/sqlbuild/compiler/planner/_helpers/test_scenario_artifacts.py +++ b/tests/unit/src/sqlbuild/compiler/planner/_helpers/test_scenario_artifacts.py @@ -4,8 +4,10 @@ from sqlbuild._native import ( ParsedScenarioArtifactName, + PlannerInputError, ScenarioArtifactIdentity, ScenarioArtifactName, + ScenarioRelationMap, build_scenario_artifact_name, build_scenario_hash_index, build_scenario_relation_map, @@ -13,10 +15,6 @@ is_scenario_artifact_physical_name, parse_scenario_artifact_physical_name, ) -from sqlbuild.compiler.planner.exceptions import PlannerInputError -from sqlbuild.compiler.planner.models import ( - ScenarioRelationMap, -) from tests.unit.src.sqlbuild.compiler.planner._helpers._test_types import ( ScenarioArtifactNameRecognitionTestCase, ScenarioArtifactNameTestCase, diff --git a/tests/unit/src/sqlbuild/compiler/planner/_helpers/test_scenario_cli.py b/tests/unit/src/sqlbuild/compiler/planner/_helpers/test_scenario_cli.py index 09c98105d2..5ba18dc8dd 100644 --- a/tests/unit/src/sqlbuild/compiler/planner/_helpers/test_scenario_cli.py +++ b/tests/unit/src/sqlbuild/compiler/planner/_helpers/test_scenario_cli.py @@ -7,14 +7,12 @@ from sqlbuild._native import ( ScenarioArtifactName, + ScenarioExecutionPlan, SqlLexicalSyntax, build_cli_scenario_plan, ) from sqlbuild.compiler.compile.models import CompiledSqlScenario from sqlbuild.compiler.pipeline.models import CompilePipelineResult -from sqlbuild.compiler.planner.models import ( - ScenarioExecutionPlan, -) from tests.unit.src.sqlbuild.compiler.planner._helpers._test_types import ( ScenarioCliPlanIdentifierLimitTestCase, ) diff --git a/tests/unit/src/sqlbuild/compiler/planner/_helpers/test_scenario_relations.py b/tests/unit/src/sqlbuild/compiler/planner/_helpers/test_scenario_relations.py index 2d645c67f2..fa88a184cb 100644 --- a/tests/unit/src/sqlbuild/compiler/planner/_helpers/test_scenario_relations.py +++ b/tests/unit/src/sqlbuild/compiler/planner/_helpers/test_scenario_relations.py @@ -8,7 +8,11 @@ from sqlbuild._native import ( CompileSqlScenarioCte, + PlannerInputError, + ScenarioExecutionPlan, + ScenarioFixturePlan, ScenarioGraphPlan, + ScenarioRelationPlan, SqlLexicalSyntax, build_scenario_execution_plan, build_scenario_fixture_plans, @@ -25,12 +29,6 @@ from sqlbuild.compiler.compile.models import CompiledSqlScenario from sqlbuild.compiler.compile.types import CompiledProjectView from sqlbuild.compiler.discovery.models import DiscoveredHookFunction -from sqlbuild.compiler.planner.exceptions import PlannerInputError -from sqlbuild.compiler.planner.models import ( - ScenarioExecutionPlan, - ScenarioFixturePlan, - ScenarioRelationPlan, -) from tests.unit.src.sqlbuild.compiler.planner._helpers._test_types import ( ScenarioCheckSqlResolutionErrorTestCase, ScenarioCheckSqlResolutionTestCase, diff --git a/tests/unit/src/sqlbuild/compiler/planner/_helpers/test_source_columns.py b/tests/unit/src/sqlbuild/compiler/planner/_helpers/test_source_columns.py index 15f88861cb..34e55fdd08 100644 --- a/tests/unit/src/sqlbuild/compiler/planner/_helpers/test_source_columns.py +++ b/tests/unit/src/sqlbuild/compiler/planner/_helpers/test_source_columns.py @@ -7,6 +7,8 @@ import pytest from sqlbuild._native import ( + CursorInputRelation, + PlannerInputError, PlannerRelationsContext, PlannerScope, SourceEntry, @@ -29,8 +31,6 @@ CompiledProjectView, CompiledResourceType, ) -from sqlbuild.compiler.planner.exceptions import PlannerInputError -from sqlbuild.compiler.planner.models import CursorInputRelation from sqlbuild.compiler.references.types import SqlReferenceKind from tests.unit.src.sqlbuild.compiler.planner._helpers._test_types import ( CursorRuntimeOwnershipTestCase, diff --git a/tests/unit/src/sqlbuild/compiler/planner/_helpers/test_source_load_nodes.py b/tests/unit/src/sqlbuild/compiler/planner/_helpers/test_source_load_nodes.py index b367074ae4..f2d27e4e57 100644 --- a/tests/unit/src/sqlbuild/compiler/planner/_helpers/test_source_load_nodes.py +++ b/tests/unit/src/sqlbuild/compiler/planner/_helpers/test_source_load_nodes.py @@ -10,6 +10,7 @@ from sqlbuild._native import ( CompileModelConfig, SourceEntry, + SourceLoadPlanEntry, SourceWriteStrategy, build_intermediate_source_map, build_source_load_entries, @@ -31,7 +32,6 @@ CompiledResourceType, ) from sqlbuild.compiler.discovery.models import DiscoveredLoaderFunction -from sqlbuild.compiler.planner.models import SourceLoadPlanEntry from sqlbuild.runtime.contracts.types import ExecutionResourceKind from tests.unit.src.sqlbuild.compiler.planner._helpers._test_types import ( LoaderDagExpansionTestCase, diff --git a/tests/unit/src/sqlbuild/compiler/planner/_helpers/test_strategy_action.py b/tests/unit/src/sqlbuild/compiler/planner/_helpers/test_strategy_action.py index 63b5df9306..f22f3e2ec2 100644 --- a/tests/unit/src/sqlbuild/compiler/planner/_helpers/test_strategy_action.py +++ b/tests/unit/src/sqlbuild/compiler/planner/_helpers/test_strategy_action.py @@ -3,21 +3,17 @@ import pytest from sqlbuild._native import ( + BackfillAction, ChangeDetectionResult, ChangeKind, + PlanAction, + PlanReason, SchemaChangeKind, SchemaColumnSource, + SchemaFinding, resolve_model_plan_action, ) from sqlbuild.compiler.compile.types import CompiledModelView -from sqlbuild.compiler.planner.models import ( - SchemaFinding, -) -from sqlbuild.compiler.planner.types import ( - BackfillAction, - PlanAction, - PlanReason, -) from tests.unit.src.sqlbuild.compiler.planner._helpers._test_types import ( IncrementalStrategyErrorTestCase, ResolveModelPlanActionTestCase, diff --git a/tests/unit/src/sqlbuild/compiler/planner/_helpers/test_strategy_schema_actions.py b/tests/unit/src/sqlbuild/compiler/planner/_helpers/test_strategy_schema_actions.py index 296125ebf7..c1c7835558 100644 --- a/tests/unit/src/sqlbuild/compiler/planner/_helpers/test_strategy_schema_actions.py +++ b/tests/unit/src/sqlbuild/compiler/planner/_helpers/test_strategy_schema_actions.py @@ -3,17 +3,13 @@ import pytest from sqlbuild._native import ( + OnSchemaChange, SchemaAction, SchemaActionKind, SchemaChangeKind, SchemaColumnSource, - resolve_schema_actions, -) -from sqlbuild.compiler.planner.models import ( SchemaFinding, -) -from sqlbuild.compiler.planner.types import ( - OnSchemaChange, + resolve_schema_actions, ) from tests.unit.src.sqlbuild.compiler.planner._helpers._test_types import ( ResolveSchemaActionsTestCase, diff --git a/tests/unit/src/sqlbuild/compiler/planner/_helpers/test_strategy_warnings.py b/tests/unit/src/sqlbuild/compiler/planner/_helpers/test_strategy_warnings.py index 16f393b55d..6a9e995c3b 100644 --- a/tests/unit/src/sqlbuild/compiler/planner/_helpers/test_strategy_warnings.py +++ b/tests/unit/src/sqlbuild/compiler/planner/_helpers/test_strategy_warnings.py @@ -3,24 +3,20 @@ import pytest from sqlbuild._native import ( + BackfillAction, ChangeDetectionResult, ChangeKind, + MaterializationType, + OnSchemaChange, PlanWarning, SchemaAction, SchemaActionKind, SchemaChangeKind, SchemaColumnSource, + SchemaFinding, WarningSeverity, build_model_warnings, ) -from sqlbuild.compiler.planner.models import ( - SchemaFinding, -) -from sqlbuild.compiler.planner.types import ( - BackfillAction, - MaterializationType, - OnSchemaChange, -) from tests.unit.src.sqlbuild.compiler.planner._helpers._test_types import ( BuildModelWarningsTestCase, ) diff --git a/tests/unit/src/sqlbuild/compiler/planner/_helpers/warehouse/test_source_tables.py b/tests/unit/src/sqlbuild/compiler/planner/_helpers/warehouse/test_source_tables.py index 31af47d331..f2918437ce 100644 --- a/tests/unit/src/sqlbuild/compiler/planner/_helpers/warehouse/test_source_tables.py +++ b/tests/unit/src/sqlbuild/compiler/planner/_helpers/warehouse/test_source_tables.py @@ -2,9 +2,8 @@ import pytest -from sqlbuild._native import SourceEntry +from sqlbuild._native import PlannerInputError, SourceEntry from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter -from sqlbuild.compiler.planner.exceptions import PlannerInputError from tests.unit.src.sqlbuild.adapter.contract.main.helpers import ( ADAPTERS_BY_NAME, BinderException, diff --git a/tests/unit/src/sqlbuild/compiler/planner/classes/test_sql_test_planning.py b/tests/unit/src/sqlbuild/compiler/planner/classes/test_sql_test_planning.py index fbe40f6ebf..a90e7a64a9 100644 --- a/tests/unit/src/sqlbuild/compiler/planner/classes/test_sql_test_planning.py +++ b/tests/unit/src/sqlbuild/compiler/planner/classes/test_sql_test_planning.py @@ -8,6 +8,7 @@ import sqlbuild._native as native_module from sqlbuild._native import ( PlannedSqlTests, + PlannerInputError, PlanWarning, SqlTestSelection, WarningSeverity, @@ -15,7 +16,6 @@ ) from sqlbuild.compiler.compile.models import CompiledObjectKey from sqlbuild.compiler.compile.types import CompiledResourceType -from sqlbuild.compiler.planner.exceptions import PlannerInputError from tests.unit.src.sqlbuild.compiler.planner.classes._test_types import ( BackgroundSqlTestPlanningTestCase, ) diff --git a/tests/unit/src/sqlbuild/compiler/planner/main/_test_types.py b/tests/unit/src/sqlbuild/compiler/planner/main/_test_types.py index 7741b24229..502519ebf2 100644 --- a/tests/unit/src/sqlbuild/compiler/planner/main/_test_types.py +++ b/tests/unit/src/sqlbuild/compiler/planner/main/_test_types.py @@ -2,13 +2,11 @@ from datetime import datetime from sqlbuild._native import ( + CursorBounds, FutureCursorsConfig, SelectionStalenessGraph, SelectionStalenessWarning, ) -from sqlbuild.compiler.planner.models import ( - CursorBounds, -) @dataclass(frozen=True) diff --git a/tests/unit/src/sqlbuild/compiler/planner/main/execution/_test_types.py b/tests/unit/src/sqlbuild/compiler/planner/main/execution/_test_types.py index af8416541e..926910e148 100644 --- a/tests/unit/src/sqlbuild/compiler/planner/main/execution/_test_types.py +++ b/tests/unit/src/sqlbuild/compiler/planner/main/execution/_test_types.py @@ -2,7 +2,7 @@ from dataclasses import dataclass -from sqlbuild.compiler.planner.types import PlanAction, PlanReason +from sqlbuild._native import PlanAction, PlanReason @dataclass(frozen=True) diff --git a/tests/unit/src/sqlbuild/compiler/planner/main/execution/test_display_plan.py b/tests/unit/src/sqlbuild/compiler/planner/main/execution/test_display_plan.py index 43420693fe..3bfe502d29 100644 --- a/tests/unit/src/sqlbuild/compiler/planner/main/execution/test_display_plan.py +++ b/tests/unit/src/sqlbuild/compiler/planner/main/execution/test_display_plan.py @@ -6,7 +6,15 @@ import pytest -from sqlbuild._native import CompileModelConfig, build_display_only_sqlbuild_plan, format_plan +from sqlbuild._native import ( + CompileModelConfig, + ModelPlanEntry, + PlanAction, + PlanOutput, + PlanReason, + build_display_only_sqlbuild_plan, + format_plan, +) from sqlbuild.compiler.compile.models import ( CompiledModel, CompiledObjectKey, @@ -14,8 +22,6 @@ CompiledRelationLocation, ) from sqlbuild.compiler.compile.types import CompiledModelView, CompiledResourceType -from sqlbuild.compiler.planner.models import ModelPlanEntry, PlanOutput -from sqlbuild.compiler.planner.types import PlanAction, PlanReason from tests.unit.src.sqlbuild.compiler.planner.main.execution._test_types import ( DisplayFullRefreshTestCase, ) diff --git a/tests/unit/src/sqlbuild/compiler/planner/main/helpers.py b/tests/unit/src/sqlbuild/compiler/planner/main/helpers.py index cb819512bb..f158c2aec8 100644 --- a/tests/unit/src/sqlbuild/compiler/planner/main/helpers.py +++ b/tests/unit/src/sqlbuild/compiler/planner/main/helpers.py @@ -12,6 +12,7 @@ PlannerOverrides, PlannerPolicies, PlannerSelection, + PlanOutput, SchemaColumn, SchemaModelEntry, SelectionStalenessGraph, @@ -33,7 +34,6 @@ CompiledProjectView, CompiledResourceType, ) -from sqlbuild.compiler.planner.models import PlanOutput def model_definition_hash(project: CompiledProjectView, name: str) -> str: diff --git a/tests/unit/src/sqlbuild/compiler/planner/main/pre_build/_test_types.py b/tests/unit/src/sqlbuild/compiler/planner/main/pre_build/_test_types.py index e4e1d8f073..8c1a0a302c 100644 --- a/tests/unit/src/sqlbuild/compiler/planner/main/pre_build/_test_types.py +++ b/tests/unit/src/sqlbuild/compiler/planner/main/pre_build/_test_types.py @@ -1,7 +1,11 @@ from dataclasses import dataclass -from sqlbuild._native import MigrationCompatibility, MigrationDiscovery -from sqlbuild.compiler.migrations.types import ColumnMigrationDecision, MigrationDecision +from sqlbuild._native import ( + ColumnMigrationDecision, + MigrationCompatibility, + MigrationDecision, + MigrationDiscovery, +) @dataclass(frozen=True) diff --git a/tests/unit/src/sqlbuild/compiler/planner/main/pre_build/helpers.py b/tests/unit/src/sqlbuild/compiler/planner/main/pre_build/helpers.py index dcf6d63594..45a0cfa718 100644 --- a/tests/unit/src/sqlbuild/compiler/planner/main/pre_build/helpers.py +++ b/tests/unit/src/sqlbuild/compiler/planner/main/pre_build/helpers.py @@ -3,23 +3,19 @@ from __future__ import annotations from sqlbuild._native import ( + ColumnMigrationDecision, + ColumnMigrationPlanEntry, DisplayOptions, MigrationCompatibility, - MigrationDiscovery, - format_pre_build_work, -) -from sqlbuild.adapter.contract.types import MigrationTransfer -from sqlbuild.compiler.compile.models import CompiledRelationLocation -from sqlbuild.compiler.migrations.types import ( - ColumnMigrationDecision, MigrationDecision, + MigrationDiscovery, MigrationPromotion, -) -from sqlbuild.compiler.planner.models import ( - ColumnMigrationPlanEntry, ModelMigrationPlanEntry, PlanOutput, + format_pre_build_work, ) +from sqlbuild.adapter.contract.types import MigrationTransfer +from sqlbuild.compiler.compile.models import CompiledRelationLocation def migration_entry( diff --git a/tests/unit/src/sqlbuild/compiler/planner/main/pre_build/test_pre_build_work_display.py b/tests/unit/src/sqlbuild/compiler/planner/main/pre_build/test_pre_build_work_display.py index 45345c47cb..f3c1786018 100644 --- a/tests/unit/src/sqlbuild/compiler/planner/main/pre_build/test_pre_build_work_display.py +++ b/tests/unit/src/sqlbuild/compiler/planner/main/pre_build/test_pre_build_work_display.py @@ -5,14 +5,15 @@ import pytest from sqlbuild._native import ( + ColumnMigrationDecision, DisplayOptions, MigrationCompatibility, + MigrationDecision, MigrationDiscovery, + PlanOutput, format_plan, format_pre_build_work, ) -from sqlbuild.compiler.migrations.types import ColumnMigrationDecision, MigrationDecision -from sqlbuild.compiler.planner.models import PlanOutput from tests.unit.src.sqlbuild.compiler.planner.main.pre_build._test_types import ( ColumnMigrationPlainTextTestCase, ColumnMigrationStyleTestCase, diff --git a/tests/unit/src/sqlbuild/compiler/planner/main/test_advance_cursor_end.py b/tests/unit/src/sqlbuild/compiler/planner/main/test_advance_cursor_end.py index e2c255b594..e341ef01d5 100644 --- a/tests/unit/src/sqlbuild/compiler/planner/main/test_advance_cursor_end.py +++ b/tests/unit/src/sqlbuild/compiler/planner/main/test_advance_cursor_end.py @@ -2,11 +2,11 @@ import pytest +from sqlbuild._native import CursorGrain, CursorType from sqlbuild._native import cursor_exclusive_to_inclusive as exclusive_to_inclusive from sqlbuild._native import cursor_inclusive_to_exclusive as inclusive_to_exclusive from sqlbuild._native import cursor_parse as parse from sqlbuild._native import cursor_render as render -from sqlbuild.compiler.planner.types import CursorGrain, CursorType from sqlbuild.cursor_algebra.types import CursorScalar from tests.unit.src.sqlbuild.compiler.planner.main._test_types import ( AdvanceCursorEndTestCase, diff --git a/tests/unit/src/sqlbuild/compiler/planner/main/test_cursor_bound_display.py b/tests/unit/src/sqlbuild/compiler/planner/main/test_cursor_bound_display.py index a21c19ef1b..93eccb9dee 100644 --- a/tests/unit/src/sqlbuild/compiler/planner/main/test_cursor_bound_display.py +++ b/tests/unit/src/sqlbuild/compiler/planner/main/test_cursor_bound_display.py @@ -2,8 +2,7 @@ import pytest -from sqlbuild._native import cursor_bound_display -from sqlbuild.compiler.planner.types import CursorGrain, CursorType +from sqlbuild._native import CursorGrain, CursorType, cursor_bound_display from tests.unit.src.sqlbuild.compiler.planner.main._test_types import ( CursorBoundDisplayTestCase, ) diff --git a/tests/unit/src/sqlbuild/compiler/planner/main/test_effective_microbatch_batch_size.py b/tests/unit/src/sqlbuild/compiler/planner/main/test_effective_microbatch_batch_size.py index f2bb357ec8..e5b58e756d 100644 --- a/tests/unit/src/sqlbuild/compiler/planner/main/test_effective_microbatch_batch_size.py +++ b/tests/unit/src/sqlbuild/compiler/planner/main/test_effective_microbatch_batch_size.py @@ -2,8 +2,7 @@ import pytest -from sqlbuild._native import resolve_effective_microbatch_batch_size -from sqlbuild.compiler.planner.types import CursorGrain +from sqlbuild._native import CursorGrain, resolve_effective_microbatch_batch_size from tests.unit.src.sqlbuild.compiler.planner.main._test_types import ( EffectiveMicrobatchBatchSizeTestCase, ) diff --git a/tests/unit/src/sqlbuild/compiler/planner/main/test_execution.py b/tests/unit/src/sqlbuild/compiler/planner/main/test_execution.py index 890873ae51..5d5ac90cc1 100644 --- a/tests/unit/src/sqlbuild/compiler/planner/main/test_execution.py +++ b/tests/unit/src/sqlbuild/compiler/planner/main/test_execution.py @@ -6,21 +6,17 @@ import pytest from sqlbuild._native import ( + ModelPlanEntry, + PlanAction, + PlannerInputError, + PlanOutput, + PlanReason, project_with, ) from sqlbuild.adapters.duckdb.classes.duckdb_adapter import DuckDbAdapter from sqlbuild.compiler.compile.models import CompiledProject from sqlbuild.compiler.compile.types import CompiledProjectView from sqlbuild.compiler.discovery.models import DiscoveredHookFunction -from sqlbuild.compiler.planner.exceptions import PlannerInputError -from sqlbuild.compiler.planner.models import ( - ModelPlanEntry, - PlanOutput, -) -from sqlbuild.compiler.planner.types import ( - PlanAction, - PlanReason, -) from tests.unit.src.sqlbuild.compiler.planner.main._test_types import ( DirectSourceFreshnessPlanOutputTestCase, ExternalBlockedPlanOutputTestCase, diff --git a/tests/unit/src/sqlbuild/compiler/planner/main/test_future_cursor_safety.py b/tests/unit/src/sqlbuild/compiler/planner/main/test_future_cursor_safety.py index c795785666..69a5bbe4fb 100644 --- a/tests/unit/src/sqlbuild/compiler/planner/main/test_future_cursor_safety.py +++ b/tests/unit/src/sqlbuild/compiler/planner/main/test_future_cursor_safety.py @@ -4,12 +4,17 @@ import pytest -from sqlbuild._native import FutureCursorAction, FutureCursorsConfig, future_cursor_cap_warning +from sqlbuild._native import ( + CursorBounds, + CursorGrain, + CursorType, + FutureCursorAction, + FutureCursorSafetyError, + FutureCursorsConfig, + future_cursor_cap_warning, +) from sqlbuild._native import cursor_sentinel_to_token as sentinel_to_token from sqlbuild._native import resolve_future_cursor_safety as apply_future_cursor_safety -from sqlbuild.compiler.planner.exceptions import FutureCursorSafetyError -from sqlbuild.compiler.planner.models import CursorBounds -from sqlbuild.compiler.planner.types import CursorGrain, CursorType from tests.unit.src.sqlbuild.compiler.planner.main._test_types import FutureCursorSafetyTestCase diff --git a/tests/unit/src/sqlbuild/compiler/planner/main/test_inclusive_cursor_end.py b/tests/unit/src/sqlbuild/compiler/planner/main/test_inclusive_cursor_end.py index 279f4c809e..f8dfc16cb0 100644 --- a/tests/unit/src/sqlbuild/compiler/planner/main/test_inclusive_cursor_end.py +++ b/tests/unit/src/sqlbuild/compiler/planner/main/test_inclusive_cursor_end.py @@ -2,8 +2,7 @@ import pytest -from sqlbuild._native import inclusive_cursor_end -from sqlbuild.compiler.planner.types import CursorGrain, CursorType +from sqlbuild._native import CursorGrain, CursorType, inclusive_cursor_end from tests.unit.src.sqlbuild.compiler.planner.main._test_types import InclusiveCursorEndTestCase diff --git a/tests/unit/src/sqlbuild/compiler/planner/main/test_maximum_start_safety.py b/tests/unit/src/sqlbuild/compiler/planner/main/test_maximum_start_safety.py index 592cbea885..4585925d8f 100644 --- a/tests/unit/src/sqlbuild/compiler/planner/main/test_maximum_start_safety.py +++ b/tests/unit/src/sqlbuild/compiler/planner/main/test_maximum_start_safety.py @@ -6,19 +6,17 @@ import pytest from sqlbuild._native import ( + CursorBounds, FutureCursorAction, FutureCursorsConfig, + MaximumAutomaticStartError, + MaximumStartPolicyInputs, + ModelCursorSnapshot, StartCursorsConfig, compute_cursor_bounds, ) from sqlbuild._native import cursor_sentinel_to_token as sentinel_to_token from sqlbuild._native import resolve_future_cursor_safety as apply_future_cursor_safety -from sqlbuild.compiler.planner.exceptions import MaximumAutomaticStartError -from sqlbuild.compiler.planner.models import ( - CursorBounds, - MaximumStartPolicyInputs, - ModelCursorSnapshot, -) from tests.unit.src.sqlbuild.compiler.planner.main._test_types import ( MaximumStartCapTestCase, MaximumStartErrorTestCase, diff --git a/tests/unit/src/sqlbuild/compiler/planner/main/test_static_command_planning.py b/tests/unit/src/sqlbuild/compiler/planner/main/test_static_command_planning.py index 74aae170d8..cd9d9916e4 100644 --- a/tests/unit/src/sqlbuild/compiler/planner/main/test_static_command_planning.py +++ b/tests/unit/src/sqlbuild/compiler/planner/main/test_static_command_planning.py @@ -16,6 +16,9 @@ PlannerRelationsContext, PlannerScope, PlannerSelection, + PlanOutput, + PlanReason, + SeedPlanEntry, build_audit_command_plan, build_seed_command_plan, build_test_command_plan, @@ -26,8 +29,6 @@ resolve_static_relation_context, ) from sqlbuild.compiler.compile.types import CompiledProjectView, CompiledSeedView -from sqlbuild.compiler.planner.models import PlanOutput, SeedPlanEntry -from sqlbuild.compiler.planner.types import PlanReason from tests.unit.src.sqlbuild.compiler.planner._helpers.helpers import ( PlannerTestAdapter, build_test_project, diff --git a/tests/unit/src/sqlbuild/compiler/planner/test_models.py b/tests/unit/src/sqlbuild/compiler/planner/test_models.py index 11959af4a7..c763b0560c 100644 --- a/tests/unit/src/sqlbuild/compiler/planner/test_models.py +++ b/tests/unit/src/sqlbuild/compiler/planner/test_models.py @@ -6,9 +6,11 @@ import pytest -from sqlbuild._native import MissingMigrationOriginPolicy -from sqlbuild.compiler.migrations.types import MigrationDecision -from sqlbuild.compiler.planner.models import ModelMigrationPlanEntry +from sqlbuild._native import ( + MigrationDecision, + MissingMigrationOriginPolicy, + ModelMigrationPlanEntry, +) from tests.unit.src.sqlbuild.compiler.planner._test_types import MissingOriginBlockTestCase from tests.unit.src.sqlbuild.compiler.planner.main.pre_build.helpers import migration_entry diff --git a/tests/unit/src/sqlbuild/cursor_algebra/main/_test_types.py b/tests/unit/src/sqlbuild/cursor_algebra/main/_test_types.py index d16e5d8aa8..1180e853b4 100644 --- a/tests/unit/src/sqlbuild/cursor_algebra/main/_test_types.py +++ b/tests/unit/src/sqlbuild/cursor_algebra/main/_test_types.py @@ -3,7 +3,7 @@ from dataclasses import dataclass from datetime import datetime -from sqlbuild.compiler.planner.types import CursorGrain +from sqlbuild._native import CursorGrain @dataclass(frozen=True, kw_only=True) diff --git a/tests/unit/src/sqlbuild/cursor_algebra/main/test_cursor_algebra.py b/tests/unit/src/sqlbuild/cursor_algebra/main/test_cursor_algebra.py index da1df0f0cf..382015ae9e 100644 --- a/tests/unit/src/sqlbuild/cursor_algebra/main/test_cursor_algebra.py +++ b/tests/unit/src/sqlbuild/cursor_algebra/main/test_cursor_algebra.py @@ -4,7 +4,7 @@ import pytest -from sqlbuild._native import AlignedInterval, IntegerValue, TimestampValue +from sqlbuild._native import AlignedInterval, CursorGrain, CursorType, IntegerValue, TimestampValue from sqlbuild._native import cursor_cap_from_end as cap_from_end from sqlbuild._native import cursor_cap_from_start as cap_from_start from sqlbuild._native import cursor_clamp as clamp @@ -25,7 +25,6 @@ from sqlbuild._native import cursor_sentinel_from_token as sentinel_from_token from sqlbuild._native import cursor_sentinel_to_token as sentinel_to_token from sqlbuild._native import cursor_split_into_batches as split_into_batches -from sqlbuild.compiler.planner.types import CursorGrain, CursorType from sqlbuild.cursor_algebra.types import BoundSentinel, CursorScalar from tests.unit.src.sqlbuild.cursor_algebra.main._test_types import ( CursorAlgebraMatrixCase, diff --git a/tests/unit/src/sqlbuild/executor/auditing/main/helpers.py b/tests/unit/src/sqlbuild/executor/auditing/main/helpers.py index a453b512c3..1ec5540435 100644 --- a/tests/unit/src/sqlbuild/executor/auditing/main/helpers.py +++ b/tests/unit/src/sqlbuild/executor/auditing/main/helpers.py @@ -6,7 +6,7 @@ from types import SimpleNamespace from typing import Any, cast -from sqlbuild._native import MeasurementThresholdBound, MeasurementThresholds +from sqlbuild._native import AuditPlanEntry, MeasurementThresholdBound, MeasurementThresholds from sqlbuild.compiler.auditing.types import ( AuditAttachmentKind, AuditEvaluationMode, @@ -17,7 +17,6 @@ ) from sqlbuild.compiler.compile.models import CompiledObjectKey from sqlbuild.compiler.compile.types import CompiledResourceType -from sqlbuild.compiler.planner.models import AuditPlanEntry from sqlbuild.executor.auditing.main._execute import execute_audit from sqlbuild.executor.auditing.main._execute_reporting_errors import ( execute_audit_reporting_errors, diff --git a/tests/unit/src/sqlbuild/executor/auditing/main/test_execute_reporting_errors.py b/tests/unit/src/sqlbuild/executor/auditing/main/test_execute_reporting_errors.py index fe2b5fb92e..ea7d76af3d 100644 --- a/tests/unit/src/sqlbuild/executor/auditing/main/test_execute_reporting_errors.py +++ b/tests/unit/src/sqlbuild/executor/auditing/main/test_execute_reporting_errors.py @@ -6,8 +6,8 @@ import pytest +from sqlbuild._native import AuditPlanEntry from sqlbuild.compiler.auditing.types import AuditOutcome, AuditSeverity -from sqlbuild.compiler.planner.models import AuditPlanEntry from sqlbuild.executor.auditing.models import AuditExecutionResult from tests.unit.src.sqlbuild.executor.auditing.main._test_types import ( AuditInterruptCase, diff --git a/tests/unit/src/sqlbuild/executor/auditing/main/test_project_results.py b/tests/unit/src/sqlbuild/executor/auditing/main/test_project_results.py index 304ed6fc8f..0e256cf66b 100644 --- a/tests/unit/src/sqlbuild/executor/auditing/main/test_project_results.py +++ b/tests/unit/src/sqlbuild/executor/auditing/main/test_project_results.py @@ -9,9 +9,9 @@ import pytest import sqlbuild.executor.auditing._helpers.result_projection as projection_module +from sqlbuild._native import PlanOutput from sqlbuild.compiler.auditing.types import AuditOutcome from sqlbuild.compiler.compile.models import CompiledRelationLocation -from sqlbuild.compiler.planner.models import PlanOutput from sqlbuild.executor.audit_results.exceptions import AuditResultStorageError from sqlbuild.executor.audit_results.models import AuditResultRecord from sqlbuild.executor.auditing.main._project_results import project_audit_result_batch diff --git a/tests/unit/src/sqlbuild/executor/build/_helpers/_test_types.py b/tests/unit/src/sqlbuild/executor/build/_helpers/_test_types.py index 7507f93334..9e942d49a5 100644 --- a/tests/unit/src/sqlbuild/executor/build/_helpers/_test_types.py +++ b/tests/unit/src/sqlbuild/executor/build/_helpers/_test_types.py @@ -1,15 +1,10 @@ from collections.abc import Callable from dataclasses import dataclass +from sqlbuild._native import MaterializationType, PlanAction, RetentionDirection, RetentionPlanPhase from sqlbuild.adapter.contract.models import RelationInfo from sqlbuild.adapter.contract.types import RetentionChangePhase from sqlbuild.compiler.discovery.models import DiscoveredLoaderFunction -from sqlbuild.compiler.planner.types import ( - MaterializationType, - PlanAction, - RetentionDirection, - RetentionPlanPhase, -) from sqlbuild.executor.build.types import BuildStatus from sqlbuild.executor.scheduling.types import ExecutionStatus from sqlbuild.runtime.contracts.types import ExecutionResourceKind diff --git a/tests/unit/src/sqlbuild/executor/build/_helpers/helpers.py b/tests/unit/src/sqlbuild/executor/build/_helpers/helpers.py index e97b231303..d46eaefea6 100644 --- a/tests/unit/src/sqlbuild/executor/build/_helpers/helpers.py +++ b/tests/unit/src/sqlbuild/executor/build/_helpers/helpers.py @@ -6,7 +6,19 @@ from typing import Any from unittest.mock import Mock -from sqlbuild._native import SeedCsvSettings, SourceColumnEntry, SourceEntry, SourceWriteStrategy +from sqlbuild._native import ( + MaterializationType, + ModelPlanEntry, + PlanAction, + PlanOutput, + PlanReason, + SeedCsvSettings, + SeedPlanEntry, + SourceColumnEntry, + SourceEntry, + SourceLoadPlanEntry, + SourceWriteStrategy, +) from sqlbuild.adapter.contract.classes.retention_adapter import RetentionAdapterMixin from sqlbuild.compiler.auditing.types import ( AuditAttachmentKind, @@ -20,17 +32,6 @@ ) from sqlbuild.compiler.compile.types import CompiledResourceType from sqlbuild.compiler.discovery.models import DiscoveredLoaderFunction -from sqlbuild.compiler.planner.models import ( - ModelPlanEntry, - PlanOutput, - SeedPlanEntry, - SourceLoadPlanEntry, -) -from sqlbuild.compiler.planner.types import ( - MaterializationType, - PlanAction, - PlanReason, -) from sqlbuild.executor.auditing.models import AuditExecutionResult diff --git a/tests/unit/src/sqlbuild/executor/build/_helpers/test_external_source_loads.py b/tests/unit/src/sqlbuild/executor/build/_helpers/test_external_source_loads.py index 1a0cc81d66..13e24c96f7 100644 --- a/tests/unit/src/sqlbuild/executor/build/_helpers/test_external_source_loads.py +++ b/tests/unit/src/sqlbuild/executor/build/_helpers/test_external_source_loads.py @@ -6,12 +6,11 @@ import pytest -from sqlbuild._native import SourceEntry +from sqlbuild._native import PlanOutput, SourceEntry, SourceLoadPlanEntry from sqlbuild.compiler.compile.models import CompiledObjectKey from sqlbuild.compiler.compile.types import CompiledResourceType from sqlbuild.compiler.discovery.models import DiscoveredLoaderFunction from sqlbuild.compiler.discovery.types import LoaderConnectionMode -from sqlbuild.compiler.planner.models import PlanOutput, SourceLoadPlanEntry from sqlbuild.executor.build._helpers.external_source_loads import ( run_external_source_loads_before_connections, ) diff --git a/tests/unit/src/sqlbuild/executor/build/_helpers/test_indexes.py b/tests/unit/src/sqlbuild/executor/build/_helpers/test_indexes.py index aaba5c6571..41c85adcf4 100644 --- a/tests/unit/src/sqlbuild/executor/build/_helpers/test_indexes.py +++ b/tests/unit/src/sqlbuild/executor/build/_helpers/test_indexes.py @@ -2,6 +2,7 @@ import pytest +from sqlbuild._native import AuditPlanEntry, PlanOutput from sqlbuild.compiler.auditing.types import ( AuditAttachmentKind, AuditRunScope, @@ -9,7 +10,6 @@ ) from sqlbuild.compiler.compile.models import CompiledObjectKey from sqlbuild.compiler.compile.types import AttachedAuditTargetKind, CompiledResourceType -from sqlbuild.compiler.planner.models import AuditPlanEntry, PlanOutput from sqlbuild.executor.build._helpers.indexes import build_execution_indexes from sqlbuild.executor.build.models import BuildIndexes from tests.unit.src.sqlbuild.executor.build._helpers._test_types import ( diff --git a/tests/unit/src/sqlbuild/executor/build/_helpers/test_retention.py b/tests/unit/src/sqlbuild/executor/build/_helpers/test_retention.py index 5c153f528b..e8d7554c65 100644 --- a/tests/unit/src/sqlbuild/executor/build/_helpers/test_retention.py +++ b/tests/unit/src/sqlbuild/executor/build/_helpers/test_retention.py @@ -5,6 +5,17 @@ import pytest +from sqlbuild._native import ( + IncrementalMode, + MaterializationType, + ModelPlanEntry, + PlanAction, + PlanOutput, + RetentionDirection, + RetentionPlanEntry, + RetentionPlanPhase, + TableTypePlanEntry, +) from sqlbuild.adapter.contract.models import ( RelationInfo, RenderedRetentionChange, @@ -13,19 +24,6 @@ ) from sqlbuild.adapter.contract.types import RetentionChangePhase, RetentionScope from sqlbuild.compiler.compile.models import CompiledRelationLocation -from sqlbuild.compiler.planner.models import ( - ModelPlanEntry, - PlanOutput, - RetentionPlanEntry, - TableTypePlanEntry, -) -from sqlbuild.compiler.planner.types import ( - IncrementalMode, - MaterializationType, - PlanAction, - RetentionDirection, - RetentionPlanPhase, -) from sqlbuild.errors.contracts.exceptions import ExecutorInputError from sqlbuild.executor.build._helpers.retention import ( apply_retention_phase, diff --git a/tests/unit/src/sqlbuild/executor/build/_helpers/test_scheduler.py b/tests/unit/src/sqlbuild/executor/build/_helpers/test_scheduler.py index 5e0371dc78..dd87ba00c6 100644 --- a/tests/unit/src/sqlbuild/executor/build/_helpers/test_scheduler.py +++ b/tests/unit/src/sqlbuild/executor/build/_helpers/test_scheduler.py @@ -9,7 +9,16 @@ import pytest -from sqlbuild._native import NativeProgressProjector, SourceEntry, SourceWriteStrategy +from sqlbuild._native import ( + ModelPlanEntry, + NativeProgressProjector, + PlanAction, + PlanOutput, + PlanReason, + SourceEntry, + SourceLoadPlanEntry, + SourceWriteStrategy, +) from sqlbuild.adapter.contract.types import TablePromotionMode from sqlbuild.adapters.duckdb.classes.duckdb_adapter import DuckDbAdapter from sqlbuild.compiler.compile.models import CompiledObjectKey @@ -19,8 +28,6 @@ DiscoveredLoaderFunction, PythonHookEntry, ) -from sqlbuild.compiler.planner.models import ModelPlanEntry, PlanOutput, SourceLoadPlanEntry -from sqlbuild.compiler.planner.types import PlanAction, PlanReason from sqlbuild.executor.build.main._execute import execute_build_plan from sqlbuild.executor.build.models import ( BuildCallbacks, diff --git a/tests/unit/src/sqlbuild/executor/build/_helpers/test_source_node.py b/tests/unit/src/sqlbuild/executor/build/_helpers/test_source_node.py index 192e3161cf..89774f9c08 100644 --- a/tests/unit/src/sqlbuild/executor/build/_helpers/test_source_node.py +++ b/tests/unit/src/sqlbuild/executor/build/_helpers/test_source_node.py @@ -6,10 +6,10 @@ import pytest +from sqlbuild._native import PlanOutput from sqlbuild.adapters.duckdb.classes.duckdb_adapter import DuckDbAdapter from sqlbuild.compiler.compile.models import CompiledObjectKey from sqlbuild.compiler.compile.types import CompiledResourceType -from sqlbuild.compiler.planner.models import PlanOutput from sqlbuild.executor.build._helpers.source_node import execute_build_source_node from sqlbuild.executor.build.models import BuildCallbacks, BuildRuntimeParams from sqlbuild.executor.load.models import LoadExecutionResult diff --git a/tests/unit/src/sqlbuild/executor/build/classes/test_build_scheduler_test_order.py b/tests/unit/src/sqlbuild/executor/build/classes/test_build_scheduler_test_order.py index 909aa90453..079f3980e8 100644 --- a/tests/unit/src/sqlbuild/executor/build/classes/test_build_scheduler_test_order.py +++ b/tests/unit/src/sqlbuild/executor/build/classes/test_build_scheduler_test_order.py @@ -4,9 +4,9 @@ import pytest +from sqlbuild._native import PlanOutput, SqlTestPlanEntry from sqlbuild.compiler.compile.models import CompiledObjectKey from sqlbuild.compiler.compile.types import CompiledResourceType -from sqlbuild.compiler.planner.models import PlanOutput, SqlTestPlanEntry from sqlbuild.executor.build.classes.build_scheduler import _test_result_authored_order from sqlbuild.executor.testing.models import SqlTestExecutionResult from sqlbuild.executor.testing.types import SqlTestOutcome diff --git a/tests/unit/src/sqlbuild/executor/clone/_helpers/_test_types.py b/tests/unit/src/sqlbuild/executor/clone/_helpers/_test_types.py index 7312f01590..963278a303 100644 --- a/tests/unit/src/sqlbuild/executor/clone/_helpers/_test_types.py +++ b/tests/unit/src/sqlbuild/executor/clone/_helpers/_test_types.py @@ -1,6 +1,6 @@ from dataclasses import dataclass -from sqlbuild.compiler.planner.types import RetentionPlanPhase +from sqlbuild._native import RetentionPlanPhase from sqlbuild.executor.clone.types import CloneAction, CloneStatus diff --git a/tests/unit/src/sqlbuild/executor/clone/_helpers/fingerprinting/helpers.py b/tests/unit/src/sqlbuild/executor/clone/_helpers/fingerprinting/helpers.py index 40b63bbbea..90a4f6e6ab 100644 --- a/tests/unit/src/sqlbuild/executor/clone/_helpers/fingerprinting/helpers.py +++ b/tests/unit/src/sqlbuild/executor/clone/_helpers/fingerprinting/helpers.py @@ -5,13 +5,20 @@ import pytest -from sqlbuild._native import Fingerprint, FingerprintSet, SeedCsvSettings +from sqlbuild._native import ( + Fingerprint, + FingerprintSet, + MaterializationType, + ModelPlanEntry, + PlanAction, + PlanReason, + SeedCsvSettings, + SeedPlanEntry, +) from sqlbuild.adapter.contract.models import RelationInfo from sqlbuild.compiler.compile.models import CompiledObjectKey, CompiledRelationLocation from sqlbuild.compiler.compile.types import CompiledResourceType from sqlbuild.compiler.fingerprints.constants import FINGERPRINT_TABLE_NAME -from sqlbuild.compiler.planner.models import ModelPlanEntry, SeedPlanEntry -from sqlbuild.compiler.planner.types import MaterializationType, PlanAction, PlanReason class CloneFingerprintAdapter: diff --git a/tests/unit/src/sqlbuild/executor/clone/_helpers/fingerprinting/test_fingerprinting.py b/tests/unit/src/sqlbuild/executor/clone/_helpers/fingerprinting/test_fingerprinting.py index f6eaf6b7cc..4e9a25ed87 100644 --- a/tests/unit/src/sqlbuild/executor/clone/_helpers/fingerprinting/test_fingerprinting.py +++ b/tests/unit/src/sqlbuild/executor/clone/_helpers/fingerprinting/test_fingerprinting.py @@ -5,11 +5,15 @@ import pytest -from sqlbuild._native import Fingerprint, FingerprintSet +from sqlbuild._native import ( + Fingerprint, + FingerprintSet, + MaterializationType, + ModelPlanEntry, + SeedPlanEntry, +) from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter from sqlbuild.compiler.fingerprints.constants import NODE_TYPE_MODEL, NODE_TYPE_SEED -from sqlbuild.compiler.planner.models import ModelPlanEntry, SeedPlanEntry -from sqlbuild.compiler.planner.types import MaterializationType from sqlbuild.executor.clone._helpers.fingerprinting import copy_clone_fingerprints from sqlbuild.executor.clone.models import CloneExecutionResult, CloneItemResult from sqlbuild.executor.clone.types import CloneAction, CloneStatus diff --git a/tests/unit/src/sqlbuild/executor/clone/_helpers/helpers.py b/tests/unit/src/sqlbuild/executor/clone/_helpers/helpers.py index cc48d4ce0a..f56d257012 100644 --- a/tests/unit/src/sqlbuild/executor/clone/_helpers/helpers.py +++ b/tests/unit/src/sqlbuild/executor/clone/_helpers/helpers.py @@ -5,6 +5,11 @@ from sqlbuild._native import ( CompileModelConfig, + FunctionPlanEntry, + MaterializationType, + ModelPlanEntry, + PlanAction, + PlanReason, ResolvedTimeTravelRetention, TimeTravelRetentionSource, ) @@ -21,8 +26,6 @@ CompiledProjectView, CompiledResourceType, ) -from sqlbuild.compiler.planner.models import FunctionPlanEntry, ModelPlanEntry -from sqlbuild.compiler.planner.types import MaterializationType, PlanAction, PlanReason class FakeCloneAdapter(BaseAdapter): diff --git a/tests/unit/src/sqlbuild/executor/clone/_helpers/test_operations.py b/tests/unit/src/sqlbuild/executor/clone/_helpers/test_operations.py index f15a4301e8..775fa6e004 100644 --- a/tests/unit/src/sqlbuild/executor/clone/_helpers/test_operations.py +++ b/tests/unit/src/sqlbuild/executor/clone/_helpers/test_operations.py @@ -2,10 +2,10 @@ import pytest +from sqlbuild._native import ModelPlanEntry from sqlbuild.adapter.contract.models import RelationLookup from sqlbuild.adapter.contract.types import LifeCycleEventKind from sqlbuild.adapter.relations.main.relation_lookup import build_relation_lookup -from sqlbuild.compiler.planner.models import ModelPlanEntry from sqlbuild.executor.clone._helpers.operations import clone_relation from sqlbuild.executor.clone.models import CloneItemResult from sqlbuild.executor.clone.types import CloneAction, CloneStatus diff --git a/tests/unit/src/sqlbuild/executor/clone/_helpers/test_retention.py b/tests/unit/src/sqlbuild/executor/clone/_helpers/test_retention.py index b6d9f37732..cfbdde3ade 100644 --- a/tests/unit/src/sqlbuild/executor/clone/_helpers/test_retention.py +++ b/tests/unit/src/sqlbuild/executor/clone/_helpers/test_retention.py @@ -4,13 +4,13 @@ import pytest +from sqlbuild._native import RetentionPlanPhase from sqlbuild.adapter.contract.models import ( RenderedRetentionChange, RetentionRequest, RetentionState, ) from sqlbuild.adapter.contract.types import BuiltinAdapter, RetentionChangePhase, RetentionScope -from sqlbuild.compiler.planner.types import RetentionPlanPhase from sqlbuild.errors.contracts.exceptions import ExecutorInputError from sqlbuild.executor.clone._helpers.retention import ( apply_clone_namespace_retention_phase, diff --git a/tests/unit/src/sqlbuild/executor/clone/main/test_execute.py b/tests/unit/src/sqlbuild/executor/clone/main/test_execute.py index fb15f5b492..6ed72ccf14 100644 --- a/tests/unit/src/sqlbuild/executor/clone/main/test_execute.py +++ b/tests/unit/src/sqlbuild/executor/clone/main/test_execute.py @@ -4,8 +4,7 @@ import pytest -from sqlbuild.compiler.planner.models import FunctionPlanEntry, ModelPlanEntry -from sqlbuild.compiler.planner.types import MaterializationType +from sqlbuild._native import FunctionPlanEntry, MaterializationType, ModelPlanEntry from sqlbuild.executor.clone.main.execute import execute_clone from sqlbuild.executor.clone.models import ( CloneExecutionInput, diff --git a/tests/unit/src/sqlbuild/executor/migrations/_helpers/helpers.py b/tests/unit/src/sqlbuild/executor/migrations/_helpers/helpers.py index 8188fa6641..d670813f52 100644 --- a/tests/unit/src/sqlbuild/executor/migrations/_helpers/helpers.py +++ b/tests/unit/src/sqlbuild/executor/migrations/_helpers/helpers.py @@ -9,16 +9,18 @@ import duckdb from sqlbuild._native import ( + ColumnMigrationDecision, + ColumnMigrationEvent, + ColumnMigrationPlanEntry, MigrationDiscovery, + MigrationRelation, OldNameViewAction, + OldNameViewPlanEntry, deterministic_column_migration_event_id, ) from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter from sqlbuild.adapters.duckdb.classes.duckdb_adapter import DuckDbAdapter from sqlbuild.compiler.compile.models import CompiledRelationLocation -from sqlbuild.compiler.migrations.models import ColumnMigrationEvent, MigrationRelation -from sqlbuild.compiler.migrations.types import ColumnMigrationDecision -from sqlbuild.compiler.planner.models import ColumnMigrationPlanEntry, OldNameViewPlanEntry RELATION: str = "main.fct_orders" diff --git a/tests/unit/src/sqlbuild/executor/migrations/_helpers/test_columns.py b/tests/unit/src/sqlbuild/executor/migrations/_helpers/test_columns.py index 5df4303783..f67bdd91a1 100644 --- a/tests/unit/src/sqlbuild/executor/migrations/_helpers/test_columns.py +++ b/tests/unit/src/sqlbuild/executor/migrations/_helpers/test_columns.py @@ -5,9 +5,8 @@ import duckdb import pytest -from sqlbuild._native import read_migration_state_events +from sqlbuild._native import ColumnMigrationEvent, read_migration_state_events from sqlbuild.adapters.duckdb.classes.duckdb_adapter import DuckDbAdapter -from sqlbuild.compiler.migrations.models import ColumnMigrationEvent from sqlbuild.executor.migrations._helpers.columns import apply_model_column_renames from tests.unit.src.sqlbuild.executor.migrations._helpers._test_types import ( OverlappingColumnRenameTestCase, diff --git a/tests/unit/src/sqlbuild/executor/migrations/_helpers/test_old_name_views.py b/tests/unit/src/sqlbuild/executor/migrations/_helpers/test_old_name_views.py index f385f71556..3f8ead34bd 100644 --- a/tests/unit/src/sqlbuild/executor/migrations/_helpers/test_old_name_views.py +++ b/tests/unit/src/sqlbuild/executor/migrations/_helpers/test_old_name_views.py @@ -6,6 +6,7 @@ import pytest +from sqlbuild._native import ColumnMigrationEvent from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter from sqlbuild.adapter.contract.models import ColumnInfo, RelationGrant from sqlbuild.adapters.bigquery.classes.bigquery_adapter import BigQueryAdapter @@ -13,7 +14,6 @@ from sqlbuild.adapters.postgres.classes.postgres_adapter import PostgresAdapter from sqlbuild.adapters.snowflake.classes.snowflake_adapter import SnowflakeAdapter from sqlbuild.adapters.sqlserver.classes.sqlserver_adapter import SqlServerAdapter -from sqlbuild.compiler.migrations.models import ColumnMigrationEvent from sqlbuild.executor.migrations._helpers.old_name_views import ( compose_column_aliases, reader_access_warning, diff --git a/tests/unit/src/sqlbuild/executor/pipeline/_helpers/helpers.py b/tests/unit/src/sqlbuild/executor/pipeline/_helpers/helpers.py index 3ce1e7ef7a..33de311c7d 100644 --- a/tests/unit/src/sqlbuild/executor/pipeline/_helpers/helpers.py +++ b/tests/unit/src/sqlbuild/executor/pipeline/_helpers/helpers.py @@ -9,8 +9,14 @@ from unittest.mock import MagicMock from sqlbuild._native import ( + AuditPlanEntry, + PlanOutput, + ScenarioExecutionPlan, ScenarioGraphPlan, + ScenarioRelationMap, + ScenarioRelationPlan, SeedCsvSettings, + SeedPlanEntry, SqlLexicalSyntax, ) from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter @@ -31,14 +37,6 @@ from sqlbuild.compiler.compile.types import CompiledResourceType from sqlbuild.compiler.discovery.models import DiscoveredSqlScenarioFile from sqlbuild.compiler.pipeline.models import CompilePipelineResult -from sqlbuild.compiler.planner.models import ( - AuditPlanEntry, - PlanOutput, - ScenarioExecutionPlan, - ScenarioRelationMap, - ScenarioRelationPlan, - SeedPlanEntry, -) from sqlbuild.errors.contracts.exceptions import ExecutorInputError from sqlbuild.executor.auditing.models import AuditExecutionResult from sqlbuild.executor.scenario.constants import SCENARIO_LOCAL_JSONL_INVALID diff --git a/tests/unit/src/sqlbuild/executor/pipeline/_helpers/test_auditing.py b/tests/unit/src/sqlbuild/executor/pipeline/_helpers/test_auditing.py index c7aefb0ff7..bc19af1caa 100644 --- a/tests/unit/src/sqlbuild/executor/pipeline/_helpers/test_auditing.py +++ b/tests/unit/src/sqlbuild/executor/pipeline/_helpers/test_auditing.py @@ -19,8 +19,10 @@ import pytest from sqlbuild._native import ( + AuditPlanEntry, AuditProgressReporter, CostContext, + PlanOutput, StatementLedgerEntry, audit_resource_id, read_statement_ledger, @@ -36,7 +38,6 @@ ) from sqlbuild.compiler.compile.models import CompiledObjectKey from sqlbuild.compiler.compile.types import AttachedAuditTargetKind, CompiledResourceType -from sqlbuild.compiler.planner.models import AuditPlanEntry, PlanOutput from sqlbuild.cost.models import CostResourceContext from sqlbuild.executor.auditing.models import AuditExecutionResult from sqlbuild.executor.pipeline._helpers import auditing diff --git a/tests/unit/src/sqlbuild/executor/pipeline/_helpers/test_connections.py b/tests/unit/src/sqlbuild/executor/pipeline/_helpers/test_connections.py index 94721e1ba7..efb84cb39d 100644 --- a/tests/unit/src/sqlbuild/executor/pipeline/_helpers/test_connections.py +++ b/tests/unit/src/sqlbuild/executor/pipeline/_helpers/test_connections.py @@ -6,8 +6,8 @@ import pytest +from sqlbuild._native import PlanOutput from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter -from sqlbuild.compiler.planner.models import PlanOutput from sqlbuild.executor.build.models import BuildCallbacks from sqlbuild.executor.pipeline._helpers import connections as connections_module from sqlbuild.executor.pipeline._helpers.connections import ( diff --git a/tests/unit/src/sqlbuild/executor/pipeline/_helpers/test_scenario.py b/tests/unit/src/sqlbuild/executor/pipeline/_helpers/test_scenario.py index abd9c47adc..3aa9db6548 100644 --- a/tests/unit/src/sqlbuild/executor/pipeline/_helpers/test_scenario.py +++ b/tests/unit/src/sqlbuild/executor/pipeline/_helpers/test_scenario.py @@ -7,10 +7,9 @@ import pytest -from sqlbuild._native import SqlLexicalSyntax +from sqlbuild._native import ScenarioExecutionPlan, SqlLexicalSyntax from sqlbuild.compiler.compile.models import CompiledSqlScenario from sqlbuild.compiler.pipeline.models import CompilePipelineResult -from sqlbuild.compiler.planner.models import ScenarioExecutionPlan from sqlbuild.executor.pipeline._helpers import scenario as scenario_pipeline from sqlbuild.executor.scenario._helpers.local import execution as local_scenario from sqlbuild.executor.scenario.constants import ( diff --git a/tests/unit/src/sqlbuild/executor/pipeline/_helpers/test_seeding.py b/tests/unit/src/sqlbuild/executor/pipeline/_helpers/test_seeding.py index a7d77ae537..28c6ee74a1 100644 --- a/tests/unit/src/sqlbuild/executor/pipeline/_helpers/test_seeding.py +++ b/tests/unit/src/sqlbuild/executor/pipeline/_helpers/test_seeding.py @@ -8,8 +8,7 @@ import pytest -from sqlbuild._native import TerminalEventIndex, format_seed_execution_json -from sqlbuild.compiler.planner.models import PlanOutput +from sqlbuild._native import PlanOutput, TerminalEventIndex, format_seed_execution_json from sqlbuild.executor.build.models import SeedExecutionResult from sqlbuild.executor.pipeline._helpers.seeding import run_seed_pipeline from sqlbuild.observability import ( diff --git a/tests/unit/src/sqlbuild/executor/pipeline/_helpers/test_testing.py b/tests/unit/src/sqlbuild/executor/pipeline/_helpers/test_testing.py index 6401a444f4..e8b2ca05ff 100644 --- a/tests/unit/src/sqlbuild/executor/pipeline/_helpers/test_testing.py +++ b/tests/unit/src/sqlbuild/executor/pipeline/_helpers/test_testing.py @@ -7,13 +7,13 @@ import pytest +from sqlbuild._native import ChainStep, PlanOutput, SqlTestPlanEntry from sqlbuild.adapters.duckdb.classes.duckdb_adapter import DuckDbAdapter from sqlbuild.compiler.compile.models import ( CompiledObjectKey, CompiledRelationLocation, ) from sqlbuild.compiler.compile.types import CompiledResourceType -from sqlbuild.compiler.planner.models import ChainStep, PlanOutput, SqlTestPlanEntry from sqlbuild.executor.pipeline._helpers import testing as testing_pipeline from sqlbuild.executor.pipeline._helpers.testing import run_test_pipeline from sqlbuild.executor.pipeline.models import TestPipelineCallbacks as PipelineCallbacks diff --git a/tests/unit/src/sqlbuild/executor/pipeline/main/helpers.py b/tests/unit/src/sqlbuild/executor/pipeline/main/helpers.py index 5e619402d2..05e3e432d6 100644 --- a/tests/unit/src/sqlbuild/executor/pipeline/main/helpers.py +++ b/tests/unit/src/sqlbuild/executor/pipeline/main/helpers.py @@ -3,20 +3,22 @@ from pathlib import Path from typing import ClassVar -from sqlbuild._native import SourceEntry -from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter -from sqlbuild.adapter.contract.classes.statement_recorder import StatementRecorder -from sqlbuild.adapter.contract.models import ColumnInfo -from sqlbuild.compiler.compile.models import CompiledObjectKey, CompiledRelationLocation -from sqlbuild.compiler.compile.types import CompiledResourceType, FunctionLanguage -from sqlbuild.compiler.planner.models import ( +from sqlbuild._native import ( FunctionPlanEntry, + MaterializationType, ModelPlanEntry, + PlanAction, PlanOutput, + PlanReason, SeedPlanEntry, + SourceEntry, SourceLoadPlanEntry, ) -from sqlbuild.compiler.planner.types import MaterializationType, PlanAction, PlanReason +from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter +from sqlbuild.adapter.contract.classes.statement_recorder import StatementRecorder +from sqlbuild.adapter.contract.models import ColumnInfo +from sqlbuild.compiler.compile.models import CompiledObjectKey, CompiledRelationLocation +from sqlbuild.compiler.compile.types import CompiledResourceType, FunctionLanguage from sqlbuild.runtime.contracts.types import ExecutionResourceKind from sqlbuild.spec.contracts.constants import DEFAULT_SEED_CSV_SETTINGS diff --git a/tests/unit/src/sqlbuild/executor/pipeline/main/test_identity.py b/tests/unit/src/sqlbuild/executor/pipeline/main/test_identity.py index 3c1789e893..21afaeaa3c 100644 --- a/tests/unit/src/sqlbuild/executor/pipeline/main/test_identity.py +++ b/tests/unit/src/sqlbuild/executor/pipeline/main/test_identity.py @@ -7,9 +7,8 @@ import pytest import sqlbuild.executor.pipeline.main.run as run_module -from sqlbuild._native import CostContext, SettingsConfig +from sqlbuild._native import CostContext, PlanOutput, SettingsConfig from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter -from sqlbuild.compiler.planner.models import PlanOutput from sqlbuild.cost.models import CostResourceContext from sqlbuild.executor.build.models import BuildExecutionResult, BuildRuntimeParams from sqlbuild.executor.build.types import BuildStatus diff --git a/tests/unit/src/sqlbuild/executor/pipeline/main/test_schema_preflight.py b/tests/unit/src/sqlbuild/executor/pipeline/main/test_schema_preflight.py index 4d1a4dfa06..dc3e4c57b4 100644 --- a/tests/unit/src/sqlbuild/executor/pipeline/main/test_schema_preflight.py +++ b/tests/unit/src/sqlbuild/executor/pipeline/main/test_schema_preflight.py @@ -4,10 +4,9 @@ import pytest +from sqlbuild._native import MaterializationType, ModelPlanEntry, PlanAction, PlanOutput, PlanReason from sqlbuild.compiler.compile.models import CompiledObjectKey, CompiledRelationLocation from sqlbuild.compiler.compile.types import CompiledResourceType -from sqlbuild.compiler.planner.models import ModelPlanEntry, PlanOutput -from sqlbuild.compiler.planner.types import MaterializationType, PlanAction, PlanReason from sqlbuild.executor.pipeline._helpers.graph_width import runnable_graph_width from sqlbuild.executor.pipeline._helpers.schema_preparation import prepare_build_schemas from tests.unit.src.sqlbuild.executor.pipeline.main._test_types import ( diff --git a/tests/unit/src/sqlbuild/executor/run/_helpers/_test_types.py b/tests/unit/src/sqlbuild/executor/run/_helpers/_test_types.py index 6e604826fb..481ec6c633 100644 --- a/tests/unit/src/sqlbuild/executor/run/_helpers/_test_types.py +++ b/tests/unit/src/sqlbuild/executor/run/_helpers/_test_types.py @@ -1,8 +1,7 @@ from dataclasses import dataclass, field -from sqlbuild._native import SchemaDynamicColumnFamily +from sqlbuild._native import PlanReason, SchemaDynamicColumnFamily from sqlbuild.adapter.contract.models import ColumnInfo, LifeCycleEvent -from sqlbuild.compiler.planner.types import PlanReason from sqlbuild.executor.run.types import ( AuditGateReuseReason, AuditGateStatus, diff --git a/tests/unit/src/sqlbuild/executor/run/_helpers/helpers.py b/tests/unit/src/sqlbuild/executor/run/_helpers/helpers.py index 5e5fa60084..c21ee40fad 100644 --- a/tests/unit/src/sqlbuild/executor/run/_helpers/helpers.py +++ b/tests/unit/src/sqlbuild/executor/run/_helpers/helpers.py @@ -2,7 +2,15 @@ from pathlib import Path from typing import Any -from sqlbuild._native import SchemaDynamicColumnFamily, SourceEntry +from sqlbuild._native import ( + AuditPlanEntry, + MaterializationType, + ModelPlanEntry, + PlanAction, + PlanReason, + SchemaDynamicColumnFamily, + SourceEntry, +) from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter from sqlbuild.adapter.contract.classes.statement_recorder import StatementRecorder from sqlbuild.adapter.contract.models import ColumnInfo @@ -22,8 +30,6 @@ ) from sqlbuild.compiler.compile.types import CompiledResourceType from sqlbuild.compiler.discovery.models import DiscoveredHookFunction, PythonHookEntry -from sqlbuild.compiler.planner.models import AuditPlanEntry, ModelPlanEntry -from sqlbuild.compiler.planner.types import MaterializationType, PlanAction, PlanReason from sqlbuild.executor.auditing.models import AuditExecutionResult from sqlbuild.executor.run._helpers.execution.hooks import execute_hooks from sqlbuild.executor.run.models import HookContext, HookRelationLookup, HookRunContext diff --git a/tests/unit/src/sqlbuild/executor/run/_helpers/test_contracts.py b/tests/unit/src/sqlbuild/executor/run/_helpers/test_contracts.py index 01b98ffe99..32e1292e35 100644 --- a/tests/unit/src/sqlbuild/executor/run/_helpers/test_contracts.py +++ b/tests/unit/src/sqlbuild/executor/run/_helpers/test_contracts.py @@ -2,9 +2,8 @@ import pytest -from sqlbuild._native import SchemaDynamicColumnFamily +from sqlbuild._native import ModelPlanEntry, SchemaDynamicColumnFamily from sqlbuild.adapter.contract.models import ColumnInfo -from sqlbuild.compiler.planner.models import ModelPlanEntry from sqlbuild.errors.contracts.exceptions import ExecutorInputError from sqlbuild.executor.run._helpers.validation.contracts import validate_runtime_contract from tests.unit.src.sqlbuild.executor.run._helpers._test_types import ( diff --git a/tests/unit/src/sqlbuild/executor/run/_helpers/test_cursor_bounds.py b/tests/unit/src/sqlbuild/executor/run/_helpers/test_cursor_bounds.py index 571316f70f..a97ececb65 100644 --- a/tests/unit/src/sqlbuild/executor/run/_helpers/test_cursor_bounds.py +++ b/tests/unit/src/sqlbuild/executor/run/_helpers/test_cursor_bounds.py @@ -9,9 +9,19 @@ import pytest from sqlbuild._native import ( + CursorBounds, + CursorGrain, + CursorInputRelation, + CursorType, CursorWatermarkMode, FutureCursorAction, + FutureCursorSafetyError, FutureCursorsConfig, + MaterializationType, + ModelCursorSnapshot, + ModelPlanEntry, + PlanAction, + PlanReason, StartCursorsConfig, normalize_cursor_snapshot_grain, ) @@ -21,20 +31,6 @@ from sqlbuild.adapters.duckdb.classes.duckdb_adapter import DuckDbAdapter from sqlbuild.compiler.compile.models import CompiledObjectKey, CompiledRelationLocation from sqlbuild.compiler.compile.types import CompiledResourceType -from sqlbuild.compiler.planner.exceptions import FutureCursorSafetyError -from sqlbuild.compiler.planner.models import ( - CursorBounds, - CursorInputRelation, - ModelCursorSnapshot, - ModelPlanEntry, -) -from sqlbuild.compiler.planner.types import ( - CursorGrain, - CursorType, - MaterializationType, - PlanAction, - PlanReason, -) from sqlbuild.errors.contracts.exceptions import ExecutorInputError from sqlbuild.executor.run._helpers.validation.cursor_bounds import ( build_runtime_cursor_spec, diff --git a/tests/unit/src/sqlbuild/executor/run/_helpers/test_custom_query_change.py b/tests/unit/src/sqlbuild/executor/run/_helpers/test_custom_query_change.py index 2942843573..36a5418f99 100644 --- a/tests/unit/src/sqlbuild/executor/run/_helpers/test_custom_query_change.py +++ b/tests/unit/src/sqlbuild/executor/run/_helpers/test_custom_query_change.py @@ -4,7 +4,7 @@ import pytest -from sqlbuild.compiler.planner.types import PlanReason +from sqlbuild._native import PlanReason from sqlbuild.executor.run._helpers.materializations.custom import reports_query_change from tests.unit.src.sqlbuild.executor.run._helpers._test_types import CustomQueryChangeTestCase diff --git a/tests/unit/src/sqlbuild/executor/run/_helpers/test_fingerprint_metadata.py b/tests/unit/src/sqlbuild/executor/run/_helpers/test_fingerprint_metadata.py index d6a13fb914..df86220afe 100644 --- a/tests/unit/src/sqlbuild/executor/run/_helpers/test_fingerprint_metadata.py +++ b/tests/unit/src/sqlbuild/executor/run/_helpers/test_fingerprint_metadata.py @@ -9,10 +9,9 @@ import pytest -from sqlbuild._native import Fingerprint +from sqlbuild._native import AuditPlanEntry, Fingerprint, ModelPlanEntry from sqlbuild.compiler.auditing.types import AuditOutcome, AuditSeverity from sqlbuild.compiler.fingerprints.constants import AUDIT_GATE_METADATA_KEY -from sqlbuild.compiler.planner.models import AuditPlanEntry, ModelPlanEntry from sqlbuild.executor.auditing.models import AuditExecutionResult from sqlbuild.executor.run._helpers.reuse import fingerprinting from sqlbuild.executor.run._helpers.reuse.fingerprint_metadata import ( diff --git a/tests/unit/src/sqlbuild/executor/run/_helpers/test_hooks.py b/tests/unit/src/sqlbuild/executor/run/_helpers/test_hooks.py index 934ce4aedc..1775657443 100644 --- a/tests/unit/src/sqlbuild/executor/run/_helpers/test_hooks.py +++ b/tests/unit/src/sqlbuild/executor/run/_helpers/test_hooks.py @@ -7,12 +7,11 @@ import pytest -from sqlbuild._native import CostContext +from sqlbuild._native import CostContext, ModelPlanEntry from sqlbuild.adapter.contract.classes.statement_recorder import StatementRecorder from sqlbuild.adapters.duckdb.classes.duckdb_adapter import DuckDbAdapter from sqlbuild.compiler.compile.models import CompiledRelationLocation from sqlbuild.compiler.discovery.models import DiscoveredHookFunction, PythonHookEntry, SqlHookEntry -from sqlbuild.compiler.planner.models import ModelPlanEntry from sqlbuild.cost.models import CostResourceContext from sqlbuild.errors.contracts.exceptions import ExecutorInputError from sqlbuild.executor.run._helpers.execution.hooks import execute_hooks, render_hooks diff --git a/tests/unit/src/sqlbuild/executor/run/_helpers/test_inclusive_end_batch_windows.py b/tests/unit/src/sqlbuild/executor/run/_helpers/test_inclusive_end_batch_windows.py index 4d865e97ca..a997e8f6ce 100644 --- a/tests/unit/src/sqlbuild/executor/run/_helpers/test_inclusive_end_batch_windows.py +++ b/tests/unit/src/sqlbuild/executor/run/_helpers/test_inclusive_end_batch_windows.py @@ -9,11 +9,11 @@ import pytest +from sqlbuild._native import CursorGrain, CursorType from sqlbuild._native import cursor_compare as compare from sqlbuild._native import cursor_inclusive_to_exclusive as inclusive_to_exclusive from sqlbuild._native import cursor_parse as parse from sqlbuild._native import cursor_render as render -from sqlbuild.compiler.planner.types import CursorGrain, CursorType from sqlbuild.cursor_algebra.types import CursorScalar from sqlbuild.executor.run._helpers.materializations.microbatch import compute_batch_windows from sqlbuild.executor.run.models import BatchWindow diff --git a/tests/unit/src/sqlbuild/executor/run/_helpers/test_lifecycle_progress.py b/tests/unit/src/sqlbuild/executor/run/_helpers/test_lifecycle_progress.py index 0bf29e1032..de21a651c1 100644 --- a/tests/unit/src/sqlbuild/executor/run/_helpers/test_lifecycle_progress.py +++ b/tests/unit/src/sqlbuild/executor/run/_helpers/test_lifecycle_progress.py @@ -4,10 +4,10 @@ import pytest +from sqlbuild._native import OnSchemaChange from sqlbuild.adapter.contract.classes.statement_recorder import StatementRecorder from sqlbuild.adapter.contract.models import ColumnInfo from sqlbuild.adapter.contract.types import PromotionStrategy -from sqlbuild.compiler.planner.types import OnSchemaChange from sqlbuild.errors.contracts.exceptions import ExecutorInputError from sqlbuild.executor.run._helpers.execution.promotion import promote_relation_to_destination from sqlbuild.executor.run._helpers.execution.staging import create_staging_relation diff --git a/tests/unit/src/sqlbuild/executor/run/_helpers/test_reuse_audit.py b/tests/unit/src/sqlbuild/executor/run/_helpers/test_reuse_audit.py index d5553fd303..1988180e60 100644 --- a/tests/unit/src/sqlbuild/executor/run/_helpers/test_reuse_audit.py +++ b/tests/unit/src/sqlbuild/executor/run/_helpers/test_reuse_audit.py @@ -6,7 +6,7 @@ import pytest -from sqlbuild.compiler.planner.models import AuditPlanEntry +from sqlbuild._native import AuditPlanEntry from sqlbuild.executor.auditing.models import AuditExecutionResult from sqlbuild.executor.run._helpers.reuse.audit import reused_final_audit_results_by_binding_key from sqlbuild.executor.run._helpers.reuse.fingerprint_metadata import ( diff --git a/tests/unit/src/sqlbuild/executor/run/_helpers/test_snapshot_adapter_boundary.py b/tests/unit/src/sqlbuild/executor/run/_helpers/test_snapshot_adapter_boundary.py index 165c3aa037..b17ded6151 100644 --- a/tests/unit/src/sqlbuild/executor/run/_helpers/test_snapshot_adapter_boundary.py +++ b/tests/unit/src/sqlbuild/executor/run/_helpers/test_snapshot_adapter_boundary.py @@ -8,16 +8,16 @@ import pytest -from sqlbuild.adapters.duckdb.classes.duckdb_adapter import DuckDbAdapter -from sqlbuild.compiler.compile.models import CompiledObjectKey, CompiledRelationLocation -from sqlbuild.compiler.compile.types import CompiledResourceType -from sqlbuild.compiler.planner.models import ModelPlanEntry -from sqlbuild.compiler.planner.types import ( +from sqlbuild._native import ( HistoricalInput, MaterializationType, + ModelPlanEntry, PlanAction, PlanReason, ) +from sqlbuild.adapters.duckdb.classes.duckdb_adapter import DuckDbAdapter +from sqlbuild.compiler.compile.models import CompiledObjectKey, CompiledRelationLocation +from sqlbuild.compiler.compile.types import CompiledResourceType from sqlbuild.executor.run._helpers.materializations.snapshot import execute_snapshot_entry from sqlbuild.executor.run.models import ModelExecutionResult, ModelMaterializationContext from sqlbuild.executor.scheduling.types import ExecutionStatus diff --git a/tests/unit/src/sqlbuild/executor/run/_helpers/test_snapshot_lifecycle.py b/tests/unit/src/sqlbuild/executor/run/_helpers/test_snapshot_lifecycle.py index 3675bd3b9b..e12176d150 100644 --- a/tests/unit/src/sqlbuild/executor/run/_helpers/test_snapshot_lifecycle.py +++ b/tests/unit/src/sqlbuild/executor/run/_helpers/test_snapshot_lifecycle.py @@ -7,14 +7,13 @@ import pytest -from sqlbuild._native import SnapshotsConfig +from sqlbuild._native import ModelPlanEntry, SnapshotsConfig from sqlbuild.adapter.contract.classes.statement_recorder import StatementRecorder from sqlbuild.adapter.contract.models import ColumnInfo from sqlbuild.adapters.duckdb.classes.duckdb_adapter import DuckDbAdapter from sqlbuild.adapters.snowflake.classes.snowflake_adapter import SnowflakeAdapter from sqlbuild.compiler.discovery.models import DiscoveredHookFunction, PythonHookEntry, SqlHookEntry from sqlbuild.compiler.fingerprints.constants import FINGERPRINT_TABLE_NAME, NODE_TYPE_MODEL -from sqlbuild.compiler.planner.models import ModelPlanEntry from sqlbuild.errors.contracts.exceptions import ExecutorInputError from sqlbuild.executor.run._helpers.materializations.snapshot import ( _apply_snapshot_schema_change, diff --git a/tests/unit/src/sqlbuild/executor/scenario/_helpers/_test_types.py b/tests/unit/src/sqlbuild/executor/scenario/_helpers/_test_types.py index 1abc8b0deb..63e0cc8ae0 100644 --- a/tests/unit/src/sqlbuild/executor/scenario/_helpers/_test_types.py +++ b/tests/unit/src/sqlbuild/executor/scenario/_helpers/_test_types.py @@ -1,8 +1,7 @@ from dataclasses import dataclass from pathlib import Path -from sqlbuild.compiler.planner.models import ScenarioExecutionPlan -from sqlbuild.compiler.planner.types import ScenarioArtifactKind +from sqlbuild._native import ScenarioArtifactKind, ScenarioExecutionPlan from sqlbuild.executor.scenario.models import ( ScenarioSnapshotCapturePlan, ScenarioSnapshotFileStats, diff --git a/tests/unit/src/sqlbuild/executor/scenario/_helpers/helpers.py b/tests/unit/src/sqlbuild/executor/scenario/_helpers/helpers.py index c9ef94e5bb..2c74f5b417 100644 --- a/tests/unit/src/sqlbuild/executor/scenario/_helpers/helpers.py +++ b/tests/unit/src/sqlbuild/executor/scenario/_helpers/helpers.py @@ -5,29 +5,25 @@ from typing import cast from sqlbuild._native import ( - ScenarioGraphPlan, -) -from sqlbuild.compiler.compile.models import ( - CompiledObjectKey, - CompiledRelationLocation, -) -from sqlbuild.compiler.compile.types import CompiledResourceType -from sqlbuild.compiler.planner.models import ( + MaterializationType, ModelPlanEntry, + PlanAction, + PlanReason, + ScenarioArtifactKind, ScenarioAssertionExpectationPlan, ScenarioExecutionPlan, ScenarioExpectedExpectationPlan, ScenarioFixturePlan, + ScenarioGraphPlan, ScenarioRelationMap, ScenarioRelationPlan, SeedPlanEntry, ) -from sqlbuild.compiler.planner.types import ( - MaterializationType, - PlanAction, - PlanReason, - ScenarioArtifactKind, +from sqlbuild.compiler.compile.models import ( + CompiledObjectKey, + CompiledRelationLocation, ) +from sqlbuild.compiler.compile.types import CompiledResourceType from sqlbuild.executor.scenario.models import ( ScenarioSnapshotCaptureRunResult, ScenarioSnapshotColumn, diff --git a/tests/unit/src/sqlbuild/executor/scenario/_helpers/test_capture.py b/tests/unit/src/sqlbuild/executor/scenario/_helpers/test_capture.py index 352c0adcb5..ef2f111d58 100644 --- a/tests/unit/src/sqlbuild/executor/scenario/_helpers/test_capture.py +++ b/tests/unit/src/sqlbuild/executor/scenario/_helpers/test_capture.py @@ -5,8 +5,7 @@ import pytest -from sqlbuild.compiler.planner.models import ScenarioExecutionPlan, SeedPlanEntry -from sqlbuild.compiler.planner.types import ScenarioArtifactKind +from sqlbuild._native import ScenarioArtifactKind, ScenarioExecutionPlan, SeedPlanEntry from sqlbuild.executor.scenario._helpers.capture import core as capture_core from sqlbuild.executor.scenario._helpers.capture.core import execute_scenario_snapshot_capture_steps from sqlbuild.executor.scenario.models import ( diff --git a/tests/unit/src/sqlbuild/executor/scenario/_helpers/test_failures.py b/tests/unit/src/sqlbuild/executor/scenario/_helpers/test_failures.py index 540d94ae32..a5e3afef91 100644 --- a/tests/unit/src/sqlbuild/executor/scenario/_helpers/test_failures.py +++ b/tests/unit/src/sqlbuild/executor/scenario/_helpers/test_failures.py @@ -4,7 +4,7 @@ import pytest -from sqlbuild.compiler.planner.types import ScenarioArtifactKind +from sqlbuild._native import ScenarioArtifactKind from sqlbuild.executor.scenario._helpers.lifecycle.failures import first_failure_details from sqlbuild.executor.scenario.models import ( ScenarioFailureDetails, diff --git a/tests/unit/src/sqlbuild/executor/scenario/_helpers/test_snapshots.py b/tests/unit/src/sqlbuild/executor/scenario/_helpers/test_snapshots.py index a6b6cb685e..175f1f0695 100644 --- a/tests/unit/src/sqlbuild/executor/scenario/_helpers/test_snapshots.py +++ b/tests/unit/src/sqlbuild/executor/scenario/_helpers/test_snapshots.py @@ -6,8 +6,7 @@ import pytest -from sqlbuild.compiler.planner.models import ScenarioExecutionPlan -from sqlbuild.compiler.planner.types import ScenarioArtifactKind +from sqlbuild._native import ScenarioArtifactKind, ScenarioExecutionPlan from sqlbuild.executor.scenario._helpers.snapshots.core import ( build_scenario_snapshot_capture_plan, build_scenario_snapshot_input_fingerprint, diff --git a/tests/unit/src/sqlbuild/executor/scenario/main/helpers.py b/tests/unit/src/sqlbuild/executor/scenario/main/helpers.py index 312696eee7..07d230a5f5 100644 --- a/tests/unit/src/sqlbuild/executor/scenario/main/helpers.py +++ b/tests/unit/src/sqlbuild/executor/scenario/main/helpers.py @@ -5,8 +5,18 @@ from typing import Any, ClassVar from sqlbuild._native import ( + MaterializationType, + ModelPlanEntry, + PlanAction, + PlanReason, + ScenarioArtifactKind, + ScenarioExecutionPlan, + ScenarioFixturePlan, ScenarioGraphPlan, + ScenarioRelationMap, + ScenarioRelationPlan, SeedCsvSettings, + SeedPlanEntry, ) from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter from sqlbuild.adapter.contract.classes.statement_recorder import StatementRecorder @@ -16,20 +26,6 @@ CompiledRelationLocation, ) from sqlbuild.compiler.compile.types import CompiledResourceType -from sqlbuild.compiler.planner.models import ( - ModelPlanEntry, - ScenarioExecutionPlan, - ScenarioFixturePlan, - ScenarioRelationMap, - ScenarioRelationPlan, - SeedPlanEntry, -) -from sqlbuild.compiler.planner.types import ( - MaterializationType, - PlanAction, - PlanReason, - ScenarioArtifactKind, -) from sqlbuild.spec.contracts.constants import DEFAULT_SEED_CSV_SETTINGS diff --git a/tests/unit/src/sqlbuild/executor/scenario/main/test_capture.py b/tests/unit/src/sqlbuild/executor/scenario/main/test_capture.py index 591bf67964..da2be93081 100644 --- a/tests/unit/src/sqlbuild/executor/scenario/main/test_capture.py +++ b/tests/unit/src/sqlbuild/executor/scenario/main/test_capture.py @@ -5,10 +5,9 @@ import pytest +from sqlbuild._native import ScenarioArtifactKind, ScenarioExecutionPlan from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter from sqlbuild.adapter.contract.models import ColumnInfo, QueryResult -from sqlbuild.compiler.planner.models import ScenarioExecutionPlan -from sqlbuild.compiler.planner.types import ScenarioArtifactKind from sqlbuild.executor.scenario._helpers.snapshots.core import ( build_scenario_snapshot_capture_plan, build_scenario_snapshot_manifest_shell, diff --git a/tests/unit/src/sqlbuild/executor/scenario/main/test_cleanup.py b/tests/unit/src/sqlbuild/executor/scenario/main/test_cleanup.py index 3503c7dcf8..805788c1eb 100644 --- a/tests/unit/src/sqlbuild/executor/scenario/main/test_cleanup.py +++ b/tests/unit/src/sqlbuild/executor/scenario/main/test_cleanup.py @@ -2,8 +2,8 @@ import pytest +from sqlbuild._native import MaterializationType from sqlbuild.adapter.contract.types import TablePromotionMode -from sqlbuild.compiler.planner.types import MaterializationType from sqlbuild.executor.scenario.main._cleanup import execute_scenario_cleanup from sqlbuild.executor.scenario.main._run import execute_scenario_run from sqlbuild.executor.scenario.models import ScenarioCleanupExecutionResult, ScenarioRunOptions diff --git a/tests/unit/src/sqlbuild/executor/scenario/main/test_fixtures.py b/tests/unit/src/sqlbuild/executor/scenario/main/test_fixtures.py index 17aaa5106a..bc9d6bc935 100644 --- a/tests/unit/src/sqlbuild/executor/scenario/main/test_fixtures.py +++ b/tests/unit/src/sqlbuild/executor/scenario/main/test_fixtures.py @@ -2,9 +2,8 @@ import pytest +from sqlbuild._native import ScenarioArtifactKind, ScenarioFixturePlan from sqlbuild.adapter.contract.types import LifeCycleEventKind -from sqlbuild.compiler.planner.models import ScenarioFixturePlan -from sqlbuild.compiler.planner.types import ScenarioArtifactKind from sqlbuild.executor.scenario._helpers.lifecycle.fixtures import execute_scenario_fixtures from sqlbuild.executor.scenario.main._fixtures import execute_scenario_fixture from sqlbuild.executor.scenario.models import ScenarioFixtureExecutionResult diff --git a/tests/unit/src/sqlbuild/executor/scenario/main/test_models.py b/tests/unit/src/sqlbuild/executor/scenario/main/test_models.py index 1f46373255..082f42e4a7 100644 --- a/tests/unit/src/sqlbuild/executor/scenario/main/test_models.py +++ b/tests/unit/src/sqlbuild/executor/scenario/main/test_models.py @@ -2,9 +2,8 @@ import pytest +from sqlbuild._native import MaterializationType, ModelPlanEntry, PlanAction, ScenarioExecutionPlan from sqlbuild.adapter.contract.types import TablePromotionMode -from sqlbuild.compiler.planner.models import ModelPlanEntry, ScenarioExecutionPlan -from sqlbuild.compiler.planner.types import MaterializationType, PlanAction from sqlbuild.executor.run.models import ModelExecutionResult from sqlbuild.executor.scenario._helpers.execution.model_execution import execute_scenario_models from sqlbuild.executor.scenario.main._execute import execute_scenario_model diff --git a/tests/unit/src/sqlbuild/executor/seed/main/helpers.py b/tests/unit/src/sqlbuild/executor/seed/main/helpers.py index 840da74ff4..2804a83c52 100644 --- a/tests/unit/src/sqlbuild/executor/seed/main/helpers.py +++ b/tests/unit/src/sqlbuild/executor/seed/main/helpers.py @@ -4,11 +4,10 @@ from pathlib import Path -from sqlbuild._native import SeedCsvSettings +from sqlbuild._native import SeedCsvSettings, SeedPlanEntry from sqlbuild.adapter.contract.models import ColumnInfo from sqlbuild.compiler.compile.models import CompiledObjectKey, CompiledRelationLocation from sqlbuild.compiler.compile.types import CompiledResourceType -from sqlbuild.compiler.planner.models import SeedPlanEntry def build_seed_plan_entry(*, seed_name: str, file_path: Path) -> SeedPlanEntry: diff --git a/tests/unit/src/sqlbuild/executor/testing/main/helpers.py b/tests/unit/src/sqlbuild/executor/testing/main/helpers.py index 85e5494de1..6d6495a4d8 100644 --- a/tests/unit/src/sqlbuild/executor/testing/main/helpers.py +++ b/tests/unit/src/sqlbuild/executor/testing/main/helpers.py @@ -1,4 +1,4 @@ -from sqlbuild._native import SqlTestAssertionStep +from sqlbuild._native import ChainStep, SqlTestAssertionStep, SqlTestPlanEntry from sqlbuild.adapters.bigquery.classes.bigquery_adapter import BigQueryAdapter from sqlbuild.adapters.databricks.classes.databricks_adapter import DatabricksAdapter from sqlbuild.adapters.duckdb.classes.duckdb_adapter import DuckDbAdapter @@ -6,7 +6,6 @@ from sqlbuild.adapters.sqlserver.classes.sqlserver_adapter import SqlServerAdapter from sqlbuild.compiler.compile.models import CompiledObjectKey from sqlbuild.compiler.compile.types import CompiledResourceType -from sqlbuild.compiler.planner.models import ChainStep, SqlTestPlanEntry def build_comparison_test_entry(*, sql_analysis_enabled: bool = True) -> SqlTestPlanEntry: diff --git a/tests/unit/src/sqlbuild/executor/testing/main/test_comparison_sql.py b/tests/unit/src/sqlbuild/executor/testing/main/test_comparison_sql.py index 6f7a19a455..8b85a2b55f 100644 --- a/tests/unit/src/sqlbuild/executor/testing/main/test_comparison_sql.py +++ b/tests/unit/src/sqlbuild/executor/testing/main/test_comparison_sql.py @@ -4,9 +4,8 @@ import pytest -from sqlbuild._native import render_sql_test_comparison_sql +from sqlbuild._native import ChainStep, SqlTestPlanEntry, render_sql_test_comparison_sql from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter -from sqlbuild.compiler.planner.models import ChainStep, SqlTestPlanEntry from tests.unit.src.sqlbuild.executor.testing.main._test_types import ( BuildComparisonSqlTestCase, ExpectedBooleanTestCase, diff --git a/tests/unit/src/sqlbuild/integrations/dagster/test_assets.py b/tests/unit/src/sqlbuild/integrations/dagster/test_assets.py index eb881eb96c..01b3f07982 100644 --- a/tests/unit/src/sqlbuild/integrations/dagster/test_assets.py +++ b/tests/unit/src/sqlbuild/integrations/dagster/test_assets.py @@ -6,7 +6,7 @@ import pytest -from sqlbuild.compiler.planner.exceptions import PlannerInputError +from sqlbuild._native import PlannerInputError from sqlbuild.integrations.dagster import ( SqlBuildDagsterTranslator, build_sqlbuild_asset_selection, diff --git a/tests/unit/src/sqlbuild/integrations/dagster/test_resource.py b/tests/unit/src/sqlbuild/integrations/dagster/test_resource.py index 4b2195d857..492cdcaba0 100644 --- a/tests/unit/src/sqlbuild/integrations/dagster/test_resource.py +++ b/tests/unit/src/sqlbuild/integrations/dagster/test_resource.py @@ -15,10 +15,10 @@ from sqlbuild._native import ( FutureCursorAction, IntegrationResultEnvelope, + MaximumStartSafetyEvidence, MicrobatchLimitAction, serialize_maximum_start_safety, ) -from sqlbuild.compiler.planner.models import MaximumStartSafetyEvidence from sqlbuild.integrations.dagster import ( SqlBuildCliResource, SqlBuildDagsterTranslator, diff --git a/tests/unit/src/sqlbuild/integrations/dbt/helpers.py b/tests/unit/src/sqlbuild/integrations/dbt/helpers.py index 25ed4886b8..7eef80fe3e 100644 --- a/tests/unit/src/sqlbuild/integrations/dbt/helpers.py +++ b/tests/unit/src/sqlbuild/integrations/dbt/helpers.py @@ -9,6 +9,7 @@ from typing import cast from sqlbuild._native import ( + BackfillAction, BackfillResult, CompileModelConfig, CompileProjectInputs, @@ -20,6 +21,10 @@ DbtEventStreamUpdate, DbtRunner, LocalConfig, + MaterializationType, + ModelPlanEntry, + PlanAction, + PlanReason, ProjectConfig, SqlLexicalSyntax, assemble_project, @@ -36,13 +41,6 @@ ) from sqlbuild.compiler.compile.types import CompiledProjectView, CompiledResourceType from sqlbuild.compiler.discovery.models import DiscoveredProjectInputs, DiscoveredSqlModelFile -from sqlbuild.compiler.planner.models import ModelPlanEntry -from sqlbuild.compiler.planner.types import ( - BackfillAction, - MaterializationType, - PlanAction, - PlanReason, -) _GENERIC_SQL_SYNTAX: SqlLexicalSyntax = SqlLexicalSyntax() diff --git a/tests/unit/src/sqlbuild/microbatches/test_sql.py b/tests/unit/src/sqlbuild/microbatches/test_sql.py index c0a21be303..07050ec121 100644 --- a/tests/unit/src/sqlbuild/microbatches/test_sql.py +++ b/tests/unit/src/sqlbuild/microbatches/test_sql.py @@ -6,6 +6,7 @@ import pytest +from sqlbuild._native import CursorType from sqlbuild._native import cursor_parse as parse from sqlbuild._native import cursor_render as render from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter @@ -15,7 +16,6 @@ from sqlbuild.adapters.postgres.classes.postgres_adapter import PostgresAdapter from sqlbuild.adapters.snowflake.classes.snowflake_adapter import SnowflakeAdapter from sqlbuild.adapters.sqlserver.classes.sqlserver_adapter import SqlServerAdapter -from sqlbuild.compiler.planner.types import CursorType from sqlbuild.microbatches._helpers.sql import ( build_create_table_sql, build_insert_many_sql, From 4376097870997243a27744a8334beb3c6b03112d Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Thu, 15 Oct 2026 16:49:31 +0100 Subject: [PATCH 04/48] refactor: host compile exceptions, rule engine, references, discovery and pipeline shapes natively --- .../src/hosted_shapes/constants.rs | 166 ++++- .../src/stage_capture/main/mod.rs | 1 + .../stage_capture/main/next_capture_number.rs | 21 + .../sqlbuild-cache/src/stage_capture/tests.rs | 2 + .../tests/test_next_capture_number.rs | 36 + .../src/stage_capture/tests/test_types.rs | 6 + .../_helpers/analysis/explicit_references.rs | 4 +- .../_helpers/analysis/reference_facts.rs | 2 +- .../_helpers/attachments/cursor_intrinsics.rs | 2 +- .../_helpers/attachments/reference_markers.rs | 2 +- .../src/bindings/_helpers/cli/entry.rs | 2 +- .../command_runtime/runtime_errors.rs | 2 +- .../compile_attachment/declaration_scope.rs | 2 +- .../compile_attachment/effective_config.rs | 2 +- .../compile_attachment/model_config.rs | 2 +- .../compile_attachment/model_inputs.rs | 12 +- .../compile_driver/assembly_sql_tests.rs | 2 +- .../compile_driver/diagnostic_collector.rs | 2 +- .../_helpers/compile_driver/phases.rs | 2 +- .../_helpers/compile_driver/pipeline.rs | 4 +- .../compile_driver/pipeline_entries.rs | 2 +- .../_helpers/compile_driver/plan_result.rs | 2 +- .../_helpers/compile_driver/project.rs | 12 +- .../_helpers/compile_records/constants.rs | 400 ++++++++++ .../_helpers/compiled_project/attachment.rs | 2 +- .../_helpers/compiled_project/loop_classes.rs | 6 +- .../_helpers/compiler_enums/constants.rs | 25 + .../_helpers/compiler_errors/constants.rs | 99 +++ .../_helpers/compiler_records/constants.rs | 33 +- .../_helpers/dbt_command/auto_init.rs | 9 +- .../bindings/_helpers/dbt_interop/manifest.rs | 2 +- .../dbt_interop/reference_resolver.rs | 2 +- .../_helpers/discovery/configuration.rs | 2 +- .../_helpers/discovery/declaration_records.rs | 2 +- .../_helpers/discovery/description_inputs.rs | 2 +- .../_helpers/discovery/header_declarations.rs | 2 +- .../_helpers/discovery/hook_entries.rs | 2 +- .../_helpers/discovery/scope_snapshot.rs | 2 +- .../_helpers/discovery/sql_test_blocks.rs | 2 +- .../_helpers/discovery/sql_test_records.rs | 13 +- .../_helpers/discovery/yaml_records.rs | 2 +- .../_helpers/format_command/selection.rs | 2 +- .../_helpers/frontier/stage_capture.rs | 32 +- .../_helpers/graph/execution_gates.rs | 2 +- .../src/bindings/_helpers/graph/views.rs | 2 +- .../src/bindings/_helpers/manifest/rows.rs | 2 +- .../_helpers/model_config/config_errors.rs | 2 +- .../_helpers/model_config/template_data.rs | 2 +- .../model_config/unicode_environment.rs | 2 +- .../_helpers/planner_changes/renames.rs | 2 +- .../_helpers/planner_entries/values.rs | 4 +- .../planner_resolution/cursor_ranges.rs | 2 +- .../planner_resolution/orchestration.rs | 2 +- .../_helpers/planner_strategy/values.rs | 2 +- .../_helpers/planner_warehouse/constants.rs | 3 +- .../project_assembly/native_resources.rs | 2 +- .../project_assembly/syntax_validation.rs | 4 +- .../_helpers/record_behaviours/constants.rs | 51 ++ .../_helpers/record_behaviours/discovery.rs | 73 ++ .../_helpers/record_behaviours/error_inits.rs | 174 +++++ .../_helpers/record_behaviours/mod.rs | 3 + .../_helpers/record_behaviours/references.rs | 59 ++ .../bindings/_helpers/records/constants.rs | 6 +- .../src/bindings/_helpers/records/fields.rs | 37 +- .../src/bindings/_helpers/records/models.rs | 22 +- .../bindings/_helpers/records/registration.rs | 19 +- .../_helpers/refactor/offline_compile.rs | 4 +- .../bindings/_helpers/render/authored_sql.rs | 2 +- .../_helpers/render/declaration_indexes.rs | 2 +- .../_helpers/render/declaration_values.rs | 4 +- .../bindings/_helpers/render/macro_calls.rs | 2 +- .../_helpers/render/macro_expansion_result.rs | 2 +- .../_helpers/rules/custom_evaluation.rs | 2 +- .../bindings/_helpers/rules/custom_hosts.rs | 2 +- .../_helpers/rules/custom_rule_cache.rs | 2 +- .../src/bindings/_helpers/rules/findings.rs | 2 +- .../bindings/_helpers/rules/fix_session.rs | 2 +- .../bindings/_helpers/rules/lint_expansion.rs | 2 +- .../_helpers/rules/lint_preparation.rs | 2 +- .../bindings/_helpers/rules/rules_config.rs | 2 +- .../src/bindings/_helpers/rules/selection.rs | 2 +- .../_helpers/scopes/relationship_names.rs | 2 +- .../_helpers/semantic_checks/resource_sql.rs | 2 +- .../_helpers/semantic_checks/stage_objects.rs | 2 +- .../_helpers/sql_test_glue/authored_ctes.rs | 2 +- .../_helpers/sql_test_glue/plan_errors.rs | 9 +- .../_helpers/sqltext/authored_values.rs | 2 +- .../bindings/_helpers/sqltext/code_scanner.rs | 2 +- .../_helpers/sqltext/sql_references.rs | 4 +- python-boundary.toml | 2 +- .../_helpers/corpus/emitted_codes.py | 8 +- .../_helpers/coverage/discovery.py | 5 +- src/sqlbuild/_native.pyi | 705 +++++++++++++++++- .../_helpers/diagnostics/python_resources.py | 8 +- .../_helpers/macro_bridge/call_store.py | 2 +- .../compile/_helpers/render/user_macros.py | 14 +- src/sqlbuild/compiler/compile/exceptions.py | 92 +-- src/sqlbuild/compiler/compile/models.py | 25 +- src/sqlbuild/compiler/compile/types.py | 50 +- .../filesystem/command_output_sinks.py | 6 + .../_helpers/validation/discovery.py | 2 +- src/sqlbuild/compiler/discovery/constants.py | 3 - src/sqlbuild/compiler/discovery/models.py | 409 ---------- src/sqlbuild/compiler/discovery/types.py | 18 - src/sqlbuild/compiler/frontier/constants.py | 3 - .../pipeline/_helpers/materializations.py | 3 +- src/sqlbuild/compiler/pipeline/models.py | 75 -- .../python_nodes/main/_hook_identities.py | 3 +- src/sqlbuild/compiler/references/types.py | 76 -- .../build/_helpers/external_source_loads.py | 10 +- .../executor/build/_helpers/source_node.py | 9 +- .../executor/build/classes/build_scheduler.py | 2 +- .../build/main/_external_source_loads.py | 3 +- src/sqlbuild/executor/build/models.py | 2 +- .../executor/load/_helpers/execution.py | 3 +- .../executor/load/_helpers/external.py | 3 +- .../load/_helpers/loader_invocation.py | 9 +- .../executor/load/_helpers/scheduling.py | 4 +- .../load/main/_build_execution_indexes.py | 3 +- src/sqlbuild/executor/load/main/_execute.py | 10 +- src/sqlbuild/executor/load/main/run.py | 3 +- src/sqlbuild/executor/load/models.py | 3 +- .../executor/pipeline/_helpers/scenario.py | 2 +- .../python_nodes/_helpers/execution.py | 3 +- .../_helpers/ingress_execution.py | 8 +- .../python_nodes/_helpers/python_checks.py | 9 +- .../python_nodes/_helpers/relation_guard.py | 2 +- .../classes/runtime_relation_guard.py | 2 +- .../executor/python_nodes/main/_checks.py | 3 +- .../executor/python_nodes/main/_ingress.py | 3 +- src/sqlbuild/executor/python_nodes/types.py | 3 +- .../executor/run/_helpers/execution/hooks.py | 17 +- .../run/_helpers/materializations/view.py | 3 +- src/sqlbuild/executor/run/models.py | 3 +- .../executor/testing/main/_execute.py | 3 +- .../executor/testing/main/_sql_length.py | 2 +- src/sqlbuild/executor/testing/models.py | 3 +- src/sqlbuild/presentation/constants.py | 5 - src/sqlbuild/presentation/types.py | 15 - src/sqlbuild/provider/_helpers/session.py | 2 +- src/sqlbuild/provider/main/session.py | 2 +- .../_helpers/engine/_custom_catalogue.py | 2 +- .../rule_engine/_helpers/engine/definition.py | 2 +- .../_helpers/engine/fact_replay.py | 2 +- .../_helpers/engine/hermeticity.py | 3 +- .../_helpers/engine/rule_harness.py | 5 +- .../_helpers/host/custom_evaluation.py | 6 +- .../rule_engine/_helpers/host/custom_host.py | 3 +- .../rule_engine/_helpers/host/module_state.py | 2 +- .../_helpers/host/tracked_facts.py | 3 +- .../rule_engine/classes/declaration_facts.py | 2 +- .../classes/module_state_monitor.py | 2 +- .../rule_engine/classes/project_tree.py | 2 +- .../rule_engine/classes/rule_context.py | 2 +- .../rule_engine/classes/runtime_guard.py | 2 +- .../rule_engine/classes/sql_document.py | 2 +- src/sqlbuild/rule_engine/classes/sql_facts.py | 2 +- src/sqlbuild/rule_engine/exceptions.py | 38 - .../cli/commands/main/test_load_command.py | 3 +- .../sqlbuild/cli/commands/main/test_rules.py | 3 +- .../main/test_startup_import_footprint.py | 1 - .../compiler/analysis_session/helpers.py | 2 +- .../test_native_query_columns.py | 8 +- .../sqlbuild/compiler/attachments/helpers.py | 2 +- .../_helpers/refs/test_native_references.py | 3 +- .../src/sqlbuild/compiler/compile/helpers.py | 4 +- .../compile/native_model_config/helpers.py | 4 +- .../compiler/compiled_project/helpers.py | 3 +- .../compiled_project/test_native_audits.py | 2 +- .../test_native_compiled_project.py | 2 +- .../compiled_project/test_native_functions.py | 2 +- .../test_native_parameter_boundaries.py | 2 +- .../compiled_project/test_native_sources.py | 2 +- .../compiler/discovery/_test_types.py | 2 +- .../sqlbuild/compiler/discovery/helpers.py | 12 +- .../test_native_declaration_files.py | 3 +- .../discovery/test_native_model_discovery.py | 2 +- .../test_native_model_file_record.py | 2 +- .../sqlbuild/compiler/model_config/helpers.py | 5 +- .../src/sqlbuild/compiler/pipeline/helpers.py | 4 +- .../pipeline/test_attached_audit_gates.py | 2 +- .../compiler/pipeline/test_diff_selectors.py | 2 +- .../test_macro_declaration_context.py | 11 +- .../pipeline/test_macro_load_count.py | 3 +- .../sqlbuild/compiler/pipeline/test_main.py | 6 +- .../pipeline/test_measurement_audits.py | 3 +- .../pipeline/test_python_source_reads.py | 3 +- .../pipeline/test_selection_lineage.py | 2 +- .../compiler/planner/_helpers/helpers.py | 4 +- .../planner/_helpers/resolve/helpers.py | 2 +- .../_helpers/resolve/test_orchestration.py | 2 +- .../_helpers/sql_test_assembly/helpers.py | 4 +- .../sql_test_assembly/test_execute_chain.py | 2 +- .../planner/_helpers/test_fixture_planning.py | 2 +- .../sqlbuild/compiler/planner/main/helpers.py | 7 +- .../main/test_snowflake_inspection_budget.py | 3 +- .../planner/main/test_source_table_probes.py | 2 +- .../test_native_project_assembly.py | 3 +- .../src/sqlbuild/compiler/scopes/helpers.py | 10 +- .../executor/build/concurrent/helpers.py | 4 +- .../concurrent/test_ordering_invariant.py | 4 +- .../sqlbuild/executor/build/custom/helpers.py | 6 +- .../custom/test_custom_materialization.py | 10 +- .../src/sqlbuild/executor/build/helpers.py | 4 +- .../build/snowflake_metadata/helpers.py | 10 +- .../test_capped_microbatch_dependencies.py | 4 +- .../executor/build/test_snapshot_execution.py | 4 +- .../src/sqlbuild/executor/run/helpers.py | 2 +- .../executor/run/incremental/helpers.py | 2 +- .../incremental/test_incremental_execution.py | 10 +- .../executor/run/microbatch/helpers.py | 2 +- .../microbatch/test_microbatch_execution.py | 4 +- .../executor/run/test_table_execution.py | 2 +- .../src/sqlbuild/executor/run/view/helpers.py | 2 +- .../executor/run/view/test_view_execution.py | 2 +- .../src/sqlbuild/executor/scenario/helpers.py | 2 +- .../scenario/test_fixture_lifecycle.py | 9 +- .../src/sqlbuild/executor/testing/helpers.py | 8 +- .../src/sqlbuild/integrations/dbt/helpers.py | 2 +- .../sqlbuild/integrations/dbt/test_graph.py | 2 +- .../integrations/dbt/test_manifest.py | 2 +- .../_helpers/coverage/_test_types.py | 2 +- .../_helpers/coverage/helpers.py | 7 +- .../_helpers/coverage/test_discovery.py | 8 +- .../_helpers/coverage/test_render.py | 2 +- .../base_adapter/test_statement_size.py | 2 +- .../commands/_helpers/check/test_planning.py | 2 +- .../_helpers/load_selection/helpers.py | 7 +- .../_helpers/load_selection/test_selection.py | 3 +- .../cli/commands/_helpers/runtime/helpers.py | 9 +- .../_helpers/runtime/test_connection.py | 4 +- .../_helpers/runtime/test_warehouses.py | 2 +- .../cli/commands/main/compile/helpers.py | 2 +- .../main/compile/test_compile_output.py | 4 +- .../cli/commands/main/entry/_test_types.py | 2 +- .../cli/commands/main/entry/test_main.py | 2 +- .../cli/commands/main/execution/helpers.py | 8 +- .../main/execution/test_scenario_command.py | 2 +- .../test_execution_result_document.py | 2 +- .../sqlbuild/cli/output/main/plan/helpers.py | 2 +- .../compiler/compile/_helpers/_test_types.py | 2 +- .../_helpers/config/test_table_type.py | 2 +- .../compiler/compile/_helpers/helpers.py | 6 +- .../_helpers/render/test_cursor_intrinsics.py | 2 +- .../compile/_helpers/test_assembly.py | 9 +- .../compile/_helpers/test_audit_factories.py | 2 +- .../_helpers/test_declaration_scopes.py | 10 +- .../compile/_helpers/test_declarations.py | 5 +- .../compile/_helpers/test_descriptions.py | 2 +- .../compiler/compile/_helpers/test_macros.py | 8 +- .../_helpers/test_microbatch_redesign.py | 2 +- .../_helpers/test_model_config_validation.py | 2 +- .../compile/_helpers/test_model_schemas.py | 4 +- .../_helpers/test_parameterized_sql_tests.py | 2 +- .../_helpers/test_rebind_declaration_scope.py | 2 +- .../compile/_helpers/test_required_columns.py | 3 +- .../_helpers/test_sql_analysis_columns.py | 2 +- .../_helpers/test_sql_analysis_validation.py | 2 +- .../compile/_helpers/test_sql_references.py | 10 +- .../compile/_helpers/test_sql_vars.py | 3 +- .../compile/_helpers/test_templating.py | 2 +- .../_helpers/test_typed_macro_references.py | 2 +- .../compiler/compile/_test_helpers.py | 2 +- .../compiler/compile/test_cursor_start.py | 3 +- .../compile/test_effective_runtime.py | 2 +- .../sqlbuild/compiler/compile/test_main.py | 8 +- .../compile/test_resolved_connection.py | 3 +- .../src/sqlbuild/compiler/dag/main/helpers.py | 12 +- .../sqlbuild/compiler/dag/main/test_build.py | 4 +- .../discovery/_helpers/_test_helpers.py | 2 +- .../compiler/discovery/_helpers/helpers.py | 3 +- .../_helpers/test_filesystem_hooks.py | 5 + .../_helpers/test_filesystem_macros.py | 2 +- .../test_filesystem_materializations.py | 4 + .../_helpers/test_filesystem_python_nodes.py | 5 + .../test_filesystem_scoped_declarations.py | 2 + .../_helpers/test_sql_declarations.py | 3 +- .../discovery/_helpers/test_sql_models.py | 4 +- .../discovery/_helpers/test_sql_scenarios.py | 3 +- .../discovery/_helpers/test_sql_tests.py | 3 +- .../_helpers/test_target_warehouses.py | 8 +- .../sqlbuild/compiler/discovery/helpers.py | 3 +- .../main/test_serialize_hook_entries.py | 3 +- .../sqlbuild/compiler/discovery/test_main.py | 2 +- .../test_runtime_event_exporter_config.py | 6 +- .../src/sqlbuild/compiler/graph/helpers.py | 3 +- .../compiler/macro_bridge/_helpers/helpers.py | 3 +- .../compiler/macro_bridge/main/helpers.py | 3 +- .../src/sqlbuild/compiler/manifest/helpers.py | 2 +- .../sqlbuild/compiler/manifest/test_main.py | 2 +- .../_helpers/target_validation/test_main.py | 3 +- .../compiler/pipeline/_helpers/test_clone.py | 2 +- .../compiler/pipeline/main/helpers.py | 2 +- .../main/test_compile_phase_timings.py | 2 +- .../compiler/planner/_helpers/_test_types.py | 2 +- .../planner/_helpers/changes/test_detect.py | 2 +- .../compiler/planner/_helpers/helpers.py | 14 +- .../planner/_helpers/resolve/test_hooks.py | 3 +- .../planner/_helpers/selection/helpers.py | 2 +- .../selection/test_unit_test_selectors.py | 2 +- .../_helpers/sql_test_assembly/helpers.py | 4 +- .../sql_test_assembly/test_plan_test.py | 5 +- .../planner/_helpers/test_audit_entry.py | 3 +- .../planner/_helpers/test_audit_scheduling.py | 2 +- .../_helpers/test_fixture_planning_work.py | 2 +- .../planner/_helpers/test_microbatch_limit.py | 2 +- .../planner/_helpers/test_scenario_cli.py | 2 +- .../_helpers/test_scenario_relations.py | 2 +- .../planner/_helpers/test_source_columns.py | 2 +- .../_helpers/test_source_load_nodes.py | 2 +- .../planner/_helpers/warehouse/helpers.py | 2 +- .../identity/test_version_identity_hooks.py | 2 +- .../compiler/planner/main/test_execution.py | 2 +- .../compiler/python_nodes/_helpers/helpers.py | 18 +- .../python_nodes/_helpers/test_inventory.py | 19 +- .../_helpers/test_run_selectors.py | 3 +- .../_helpers/test_unified_selectors.py | 8 +- .../compiler/references/_test_types.py | 2 +- .../compiler/references/test_types.py | 2 +- .../src/sqlbuild/compiler/scopes/helpers.py | 8 +- .../scopes/test_builder_and_visibility.py | 14 +- .../sql_analysis/_helpers/test_scanning.py | 3 +- .../executor/build/_helpers/_test_types.py | 9 +- .../executor/build/_helpers/helpers.py | 2 +- .../_helpers/test_external_source_loads.py | 10 +- .../executor/build/_helpers/test_scheduler.py | 8 +- .../src/sqlbuild/executor/load/_test_types.py | 3 +- .../sqlbuild/executor/load/test_execute.py | 10 +- .../src/sqlbuild/executor/load/test_run.py | 10 +- .../executor/pipeline/_helpers/helpers.py | 4 +- .../pipeline/_helpers/test_scenario.py | 3 +- .../executor/python_nodes/_helpers/helpers.py | 12 +- .../python_nodes/_helpers/test_execution.py | 3 +- .../_helpers/test_python_checks.py | 2 +- .../python_nodes/classes/_test_types.py | 2 +- .../classes/test_runtime_relation_guard.py | 2 +- .../sqlbuild/executor/run/_helpers/helpers.py | 3 +- .../executor/run/_helpers/test_hooks.py | 9 +- .../run/_helpers/test_snapshot_lifecycle.py | 9 +- .../src/sqlbuild/integrations/dbt/helpers.py | 3 +- .../sqlbuild/integrations/dbt/test_execute.py | 2 +- .../integrations/dbt/test_manifest.py | 2 +- .../lint/_helpers/test_project_files.py | 2 +- .../lint/test_adapter_lexical_syntax.py | 3 +- .../src/sqlbuild/lint/test_expanded_lint.py | 2 +- .../provider/_helpers/test_build_session.py | 2 +- .../rule_engine/_helpers/engine/helpers.py | 2 +- .../_helpers/engine/test_config.py | 3 +- .../engine/test_custom_rule_evidence.py | 2 +- .../_helpers/engine/test_custom_rules.py | 2 +- .../_helpers/engine/test_definition.py | 2 +- .../_helpers/engine/test_hermeticity.py | 2 +- .../_helpers/engine/test_native_facts.py | 4 +- .../_helpers/engine/test_typed_constants.py | 2 +- .../rule_engine/_helpers/host/helpers.py | 2 +- .../_helpers/host/test_custom_host_pool.py | 9 +- .../_helpers/run/test_native_rows.py | 3 +- .../classes/project_tree/test_read_text.py | 2 +- .../classes/shared_fact_encoder/helpers.py | 2 +- .../rule_engine/main/evaluate/helpers.py | 3 +- .../evaluate/test_cache_and_suppressions.py | 2 +- .../main/evaluate_rule/test_evaluate_rule.py | 3 +- .../main/run_rules/test_run_rules.py | 4 +- .../rule_engine/main/skills/test_skills.py | 3 +- .../event_exporting/classes/helpers.py | 1 + .../test_resolve_effective_scenario_config.py | 2 +- 366 files changed, 2655 insertions(+), 1374 deletions(-) create mode 100644 crates/sqlbuild-cache/src/stage_capture/main/next_capture_number.rs create mode 100644 crates/sqlbuild-cache/src/stage_capture/tests/test_next_capture_number.rs create mode 100644 crates/sqlbuild-python/src/bindings/_helpers/record_behaviours/discovery.rs create mode 100644 crates/sqlbuild-python/src/bindings/_helpers/record_behaviours/error_inits.rs create mode 100644 crates/sqlbuild-python/src/bindings/_helpers/record_behaviours/references.rs delete mode 100644 src/sqlbuild/compiler/discovery/models.py delete mode 100644 src/sqlbuild/compiler/discovery/types.py delete mode 100644 src/sqlbuild/compiler/pipeline/models.py delete mode 100644 src/sqlbuild/compiler/references/types.py delete mode 100644 src/sqlbuild/presentation/constants.py delete mode 100644 src/sqlbuild/presentation/types.py delete mode 100644 src/sqlbuild/rule_engine/exceptions.py diff --git a/crates/sqlbuild-cache/src/hosted_shapes/constants.rs b/crates/sqlbuild-cache/src/hosted_shapes/constants.rs index 451e84c100..40d6468874 100644 --- a/crates/sqlbuild-cache/src/hosted_shapes/constants.rs +++ b/crates/sqlbuild-cache/src/hosted_shapes/constants.rs @@ -3,7 +3,7 @@ /// The extension module hosting the shapes. pub const NATIVE_MODULE: &str = "sqlbuild._native"; /// The Python model type a natively hosted compiler shape is captured as. -pub const HOSTED_SHAPE_TYPES: [(&str, &str); 409] = [ +pub const HOSTED_SHAPE_TYPES: [(&str, &str); 450] = [ ( "sqlbuild._native:AlignedInterval", "sqlbuild.cursor_algebra.models:AlignedInterval", @@ -120,6 +120,10 @@ pub const HOSTED_SHAPE_TYPES: [(&str, &str); 409] = [ "sqlbuild._native:CompileFacts", "sqlbuild.lint.models:CompileFacts", ), + ( + "sqlbuild._native:CompileInputError", + "sqlbuild.compiler.compile.exceptions:CompileInputError", + ), ( "sqlbuild._native:CompileModelConfig", "sqlbuild.compiler.compile.models:CompileModelConfig", @@ -132,6 +136,10 @@ pub const HOSTED_SHAPE_TYPES: [(&str, &str); 409] = [ "sqlbuild._native:CompilePipelineOptions", "sqlbuild.compiler.pipeline.models:CompilePipelineOptions", ), + ( + "sqlbuild._native:CompilePipelineResult", + "sqlbuild.compiler.pipeline.models:CompilePipelineResult", + ), ( "sqlbuild._native:CompileProjectInputs", "sqlbuild.compiler.compile.models:CompileProjectInputs", @@ -196,6 +204,10 @@ pub const HOSTED_SHAPE_TYPES: [(&str, &str); 409] = [ "sqlbuild._native:ConfigValueTypeError", "sqlbuild.spec.contracts.exceptions:ConfigValueTypeError", ), + ( + "sqlbuild._native:ConstantDeclaration", + "sqlbuild.compiler.discovery.models:ConstantDeclaration", + ), ( "sqlbuild._native:ConstantMetadata", "sqlbuild.compiler.scopes.models:ConstantMetadata", @@ -404,14 +416,34 @@ pub const HOSTED_SHAPE_TYPES: [(&str, &str); 409] = [ "sqlbuild._native:DiscoveredAdapterFile", "sqlbuild.compiler.discovery.models:DiscoveredAdapterFile", ), + ( + "sqlbuild._native:DiscoveredAssetFunction", + "sqlbuild.compiler.discovery.models:DiscoveredAssetFunction", + ), ( "sqlbuild._native:DiscoveredAuditBlock", "sqlbuild.compiler.discovery.models:DiscoveredAuditBlock", ), + ( + "sqlbuild._native:DiscoveredAuditFactory", + "sqlbuild.compiler.discovery.models:DiscoveredAuditFactory", + ), ( "sqlbuild._native:DiscoveredAuditFile", "sqlbuild.compiler.discovery.models:DiscoveredAuditFile", ), + ( + "sqlbuild._native:DiscoveredCheckFunction", + "sqlbuild.compiler.discovery.models:DiscoveredCheckFunction", + ), + ( + "sqlbuild._native:DiscoveredCommandOutputSink", + "sqlbuild.compiler.discovery.models:DiscoveredCommandOutputSink", + ), + ( + "sqlbuild._native:DiscoveredCommandOutputSinkDeclaration", + "sqlbuild.compiler.discovery.models:DiscoveredCommandOutputSinkDeclaration", + ), ( "sqlbuild._native:DiscoveredConstantFile", "sqlbuild.compiler.discovery.models:DiscoveredConstantFile", @@ -420,10 +452,46 @@ pub const HOSTED_SHAPE_TYPES: [(&str, &str); 409] = [ "sqlbuild._native:DiscoveredEnumFile", "sqlbuild.compiler.discovery.models:DiscoveredEnumFile", ), + ( + "sqlbuild._native:DiscoveredEventExporter", + "sqlbuild.compiler.discovery.models:DiscoveredEventExporter", + ), + ( + "sqlbuild._native:DiscoveredEventExporterDeclaration", + "sqlbuild.compiler.discovery.models:DiscoveredEventExporterDeclaration", + ), + ( + "sqlbuild._native:DiscoveredHookFunction", + "sqlbuild.compiler.discovery.models:DiscoveredHookFunction", + ), + ( + "sqlbuild._native:DiscoveredLoaderFunction", + "sqlbuild.compiler.discovery.models:DiscoveredLoaderFunction", + ), + ( + "sqlbuild._native:DiscoveredMacroFile", + "sqlbuild.compiler.discovery.models:DiscoveredMacroFile", + ), + ( + "sqlbuild._native:DiscoveredMaterializationFile", + "sqlbuild.compiler.discovery.models:DiscoveredMaterializationFile", + ), ( "sqlbuild._native:DiscoveredModelSchemaFile", "sqlbuild.compiler.discovery.models:DiscoveredModelSchemaFile", ), + ( + "sqlbuild._native:DiscoveredProjectInputs", + "sqlbuild.compiler.discovery.models:DiscoveredProjectInputs", + ), + ( + "sqlbuild._native:DiscoveredProvider", + "sqlbuild.compiler.discovery.models:DiscoveredProvider", + ), + ( + "sqlbuild._native:DiscoveredProviderUsage", + "sqlbuild.compiler.discovery.models:DiscoveredProviderUsage", + ), ( "sqlbuild._native:DiscoveredPythonAssetMetadata", "sqlbuild.compiler.python_nodes.models:DiscoveredPythonAssetMetadata", @@ -444,10 +512,18 @@ pub const HOSTED_SHAPE_TYPES: [(&str, &str); 409] = [ "sqlbuild._native:DiscoveredPythonNode", "sqlbuild.compiler.python_nodes.models:DiscoveredPythonNode", ), + ( + "sqlbuild._native:DiscoveredPythonNodeFunctions", + "sqlbuild.compiler.discovery.models:DiscoveredPythonNodeFunctions", + ), ( "sqlbuild._native:DiscoveredPythonTaskMetadata", "sqlbuild.compiler.python_nodes.models:DiscoveredPythonTaskMetadata", ), + ( + "sqlbuild._native:DiscoveredRuntimeExtensions", + "sqlbuild.compiler.discovery.models:DiscoveredRuntimeExtensions", + ), ( "sqlbuild._native:DiscoveredSchemaFile", "sqlbuild.compiler.discovery.models:DiscoveredSchemaFile", @@ -456,6 +532,10 @@ pub const HOSTED_SHAPE_TYPES: [(&str, &str); 409] = [ "sqlbuild._native:DiscoveredSeedFile", "sqlbuild.compiler.discovery.models:DiscoveredSeedFile", ), + ( + "sqlbuild._native:DiscoveredSourceFile", + "sqlbuild.compiler.discovery.models:DiscoveredSourceFile", + ), ( "sqlbuild._native:DiscoveredSqlFunctionFile", "sqlbuild.compiler.discovery.models:DiscoveredSqlFunctionFile", @@ -464,6 +544,10 @@ pub const HOSTED_SHAPE_TYPES: [(&str, &str); 409] = [ "sqlbuild._native:DiscoveredSqlHookFile", "sqlbuild.compiler.discovery.models:DiscoveredSqlHookFile", ), + ( + "sqlbuild._native:DiscoveredSqlScenarioFile", + "sqlbuild.compiler.discovery.models:DiscoveredSqlScenarioFile", + ), ( "sqlbuild._native:DiscoveredSqlTestBlock", "sqlbuild.compiler.discovery.models:DiscoveredSqlTestBlock", @@ -472,6 +556,14 @@ pub const HOSTED_SHAPE_TYPES: [(&str, &str); 409] = [ "sqlbuild._native:DiscoveredSqlTestCase", "sqlbuild.compiler.discovery.models:DiscoveredSqlTestCase", ), + ( + "sqlbuild._native:DiscoveredSqlTestFile", + "sqlbuild.compiler.discovery.models:DiscoveredSqlTestFile", + ), + ( + "sqlbuild._native:DiscoveredTaskFunction", + "sqlbuild.compiler.discovery.models:DiscoveredTaskFunction", + ), ( "sqlbuild._native:DiscoveryConflictError", "sqlbuild.compiler.discovery.exceptions:DiscoveryConflictError", @@ -504,6 +596,10 @@ pub const HOSTED_SHAPE_TYPES: [(&str, &str); 409] = [ "sqlbuild._native:EmptyCursorInputDecision", "sqlbuild.compiler.planner.models:EmptyCursorInputDecision", ), + ( + "sqlbuild._native:EnumDeclaration", + "sqlbuild.compiler.discovery.models:EnumDeclaration", + ), ( "sqlbuild._native:EnumMember", "sqlbuild.compiler.discovery.models:EnumMember", @@ -536,6 +632,10 @@ pub const HOSTED_SHAPE_TYPES: [(&str, &str); 409] = [ "sqlbuild._native:ExpansionSpan", "sqlbuild.compiler.compile.models:ExpansionSpan", ), + ( + "sqlbuild._native:FactDigestError", + "sqlbuild.rule_engine.exceptions:FactDigestError", + ), ( "sqlbuild._native:Fingerprint", "sqlbuild.compiler.fingerprints.models:Fingerprint", @@ -600,6 +700,10 @@ pub const HOSTED_SHAPE_TYPES: [(&str, &str); 409] = [ "sqlbuild._native:HandBuiltAnalysis", "sqlbuild.compiler.compile.models:HandBuiltAnalysis", ), + ( + "sqlbuild._native:HardCodedRelationOwnerKind", + "sqlbuild.compiler.references.types:HardCodedRelationOwnerKind", + ), ( "sqlbuild._native:HeaderSpan", "sqlbuild.lint.models:HeaderSpan", @@ -616,6 +720,10 @@ pub const HOSTED_SHAPE_TYPES: [(&str, &str); 409] = [ "sqlbuild._native:HookReadType", "sqlbuild.compiler.graph.types:HookReadType", ), + ( + "sqlbuild._native:HostCancelledError", + "sqlbuild.rule_engine.exceptions:HostCancelledError", + ), ( "sqlbuild._native:InaccessibleReason", "sqlbuild.compiler.scopes.types:InaccessibleReason", @@ -710,6 +818,10 @@ pub const HOSTED_SHAPE_TYPES: [(&str, &str); 409] = [ "sqlbuild._native:LiteralSqlRelation", "sqlbuild.compiler.references.models:LiteralSqlRelation", ), + ( + "sqlbuild._native:LoaderConnectionMode", + "sqlbuild.compiler.discovery.types:LoaderConnectionMode", + ), ( "sqlbuild._native:LoaderDestinationParts", "sqlbuild.spec.contracts.models:LoaderDestinationParts", @@ -730,6 +842,10 @@ pub const HOSTED_SHAPE_TYPES: [(&str, &str); 409] = [ "sqlbuild._native:LocalTargetConfig", "sqlbuild.spec.contracts.models:LocalTargetConfig", ), + ( + "sqlbuild._native:MacroArgumentError", + "sqlbuild.compiler.compile.exceptions:MacroArgumentError", + ), ( "sqlbuild._native:MacroExpansionResult", "sqlbuild.compiler.compile.models:MacroExpansionResult", @@ -934,6 +1050,10 @@ pub const HOSTED_SHAPE_TYPES: [(&str, &str); 409] = [ "sqlbuild._native:NativeStageMismatchError", "sqlbuild.compiler.frontier.exceptions:NativeStageMismatchError", ), + ( + "sqlbuild._native:NonHermeticRuleError", + "sqlbuild.rule_engine.exceptions:NonHermeticRuleError", + ), ( "sqlbuild._native:NormalizedDbtProfileConnection", "sqlbuild.integrations.dbt.models:NormalizedDbtProfileConnection", @@ -974,6 +1094,10 @@ pub const HOSTED_SHAPE_TYPES: [(&str, &str); 409] = [ "sqlbuild._native:OnSchemaChange", "sqlbuild.compiler.planner.types:OnSchemaChange", ), + ( + "sqlbuild._native:OpaqueModuleStateError", + "sqlbuild.rule_engine.exceptions:OpaqueModuleStateError", + ), ( "sqlbuild._native:OwnershipRoot", "sqlbuild.compiler.scopes.models:OwnershipRoot", @@ -1062,6 +1186,10 @@ pub const HOSTED_SHAPE_TYPES: [(&str, &str); 409] = [ "sqlbuild._native:ProjectConfigError", "sqlbuild.compiler.discovery.exceptions:ProjectConfigError", ), + ( + "sqlbuild._native:ProjectGraph", + "sqlbuild.compiler.pipeline.models:ProjectGraph", + ), ( "sqlbuild._native:ProjectPathError", "sqlbuild.compiler.discovery.exceptions:ProjectPathError", @@ -1074,6 +1202,10 @@ pub const HOSTED_SHAPE_TYPES: [(&str, &str); 409] = [ "sqlbuild._native:ProviderDiscoveryError", "sqlbuild.compiler.discovery.exceptions:ProviderDiscoveryError", ), + ( + "sqlbuild._native:PythonHookEntry", + "sqlbuild.compiler.discovery.models:PythonHookEntry", + ), ( "sqlbuild._native:PythonIdentityDependency", "sqlbuild.compiler.python_nodes.models:PythonIdentityDependency", @@ -1194,6 +1326,10 @@ pub const HOSTED_SHAPE_TYPES: [(&str, &str); 409] = [ "sqlbuild._native:RetentionPlanPhase", "sqlbuild.compiler.planner.types:RetentionPlanPhase", ), + ( + "sqlbuild._native:RuleAssertionError", + "sqlbuild.rule_engine.exceptions:RuleAssertionError", + ), ( "sqlbuild._native:RuleExemption", "sqlbuild.rule_engine.models:RuleExemption", @@ -1214,10 +1350,22 @@ pub const HOSTED_SHAPE_TYPES: [(&str, &str); 409] = [ "sqlbuild._native:RuleRunPlan", "sqlbuild.rule_engine.models:RuleRunPlan", ), + ( + "sqlbuild._native:RuleUsageError", + "sqlbuild.rule_engine.exceptions:RuleUsageError", + ), ( "sqlbuild._native:RulesCacheConfig", "sqlbuild.rule_engine.models:RulesCacheConfig", ), + ( + "sqlbuild._native:RulesConfigError", + "sqlbuild.rule_engine.exceptions:RulesConfigError", + ), + ( + "sqlbuild._native:RulesError", + "sqlbuild.rule_engine.exceptions:RulesError", + ), ( "sqlbuild._native:RulesRunResult", "sqlbuild.rule_engine.models:RulesRunResult", @@ -1478,6 +1626,10 @@ pub const HOSTED_SHAPE_TYPES: [(&str, &str); 409] = [ "sqlbuild._native:SqlExpansionReuse", "sqlbuild.rule_engine.models:SqlExpansionReuse", ), + ( + "sqlbuild._native:SqlHookEntry", + "sqlbuild.compiler.discovery.models:SqlHookEntry", + ), ( "sqlbuild._native:SqlHookParseError", "sqlbuild.compiler.discovery.exceptions:SqlHookParseError", @@ -1490,6 +1642,10 @@ pub const HOSTED_SHAPE_TYPES: [(&str, &str); 409] = [ "sqlbuild._native:SqlLogicalType", "sqlbuild.sql_values.models:SqlLogicalType", ), + ( + "sqlbuild._native:SqlReferenceKind", + "sqlbuild.compiler.references.types:SqlReferenceKind", + ), ( "sqlbuild._native:SqlReferenceScan", "sqlbuild.compiler.compile.models:SqlReferenceScan", @@ -1510,6 +1666,10 @@ pub const HOSTED_SHAPE_TYPES: [(&str, &str); 409] = [ "sqlbuild._native:SqlTestAssertionStep", "sqlbuild.compiler.planner.models:SqlTestAssertionStep", ), + ( + "sqlbuild._native:SqlTestParameterDeclaration", + "sqlbuild.compiler.discovery.models:SqlTestParameterDeclaration", + ), ( "sqlbuild._native:SqlTestParseError", "sqlbuild.compiler.discovery.exceptions:SqlTestParseError", @@ -1518,6 +1678,10 @@ pub const HOSTED_SHAPE_TYPES: [(&str, &str); 409] = [ "sqlbuild._native:SqlTestPlanEntry", "sqlbuild.compiler.planner.models:SqlTestPlanEntry", ), + ( + "sqlbuild._native:SqlTestReferenceError", + "sqlbuild.compiler.compile.exceptions:SqlTestReferenceError", + ), ( "sqlbuild._native:SqlTestRulesConfig", "sqlbuild.rule_engine.models:SqlTestRulesConfig", diff --git a/crates/sqlbuild-cache/src/stage_capture/main/mod.rs b/crates/sqlbuild-cache/src/stage_capture/main/mod.rs index 96a12d57ab..3af58fac93 100644 --- a/crates/sqlbuild-cache/src/stage_capture/main/mod.rs +++ b/crates/sqlbuild-cache/src/stage_capture/main/mod.rs @@ -1,4 +1,5 @@ pub mod capture_sort_key; pub mod compact_capture_json; pub mod mask_capture_noise; +pub mod next_capture_number; pub mod shared_node_digest; diff --git a/crates/sqlbuild-cache/src/stage_capture/main/next_capture_number.rs b/crates/sqlbuild-cache/src/stage_capture/main/next_capture_number.rs new file mode 100644 index 0000000000..5de392957e --- /dev/null +++ b/crates/sqlbuild-cache/src/stage_capture/main/next_capture_number.rs @@ -0,0 +1,21 @@ +//! The sequence number of the next capture written into a capture directory. + +/// Whether `name` is a numbered capture such as `001-discovery.json`. +fn numbered_capture(name: &str) -> bool { + let Some((number, _)) = name.split_once('-') else { + return false; + }; + name.ends_with(".json") + && !number.is_empty() + && number.bytes().all(|byte| byte.is_ascii_digit()) +} + +/// One more than the numbered captures (`NNN-name.json`) among `file_names`. +#[must_use] +pub fn next_capture_number<'a>(file_names: impl IntoIterator) -> u64 { + let written: usize = file_names + .into_iter() + .filter(|name| numbered_capture(name)) + .count(); + u64::try_from(written).map_or(u64::MAX, |count| count.saturating_add(1)) +} diff --git a/crates/sqlbuild-cache/src/stage_capture/tests.rs b/crates/sqlbuild-cache/src/stage_capture/tests.rs index 80c6c333f9..b69b5e1b53 100644 --- a/crates/sqlbuild-cache/src/stage_capture/tests.rs +++ b/crates/sqlbuild-cache/src/stage_capture/tests.rs @@ -4,5 +4,7 @@ mod capture_document; mod helpers; #[path = "tests/test_mask_capture_noise.rs"] mod mask_capture_noise; +#[path = "tests/test_next_capture_number.rs"] +mod next_capture_number; #[path = "tests/test_types.rs"] mod test_types; diff --git a/crates/sqlbuild-cache/src/stage_capture/tests/test_next_capture_number.rs b/crates/sqlbuild-cache/src/stage_capture/tests/test_next_capture_number.rs new file mode 100644 index 0000000000..c4ed4d04ca --- /dev/null +++ b/crates/sqlbuild-cache/src/stage_capture/tests/test_next_capture_number.rs @@ -0,0 +1,36 @@ +use crate::stage_capture::main::next_capture_number::next_capture_number; +use crate::stage_capture::tests::test_types::CaptureNumberTestCase; + +#[test] +fn given_capture_directory_files_when_numbering_then_counts_numbered_captures() { + let test_cases = [ + CaptureNumberTestCase { + description: "an empty directory starts at one", + file_names: &[], + expected_number: 1, + }, + CaptureNumberTestCase { + description: "numbered captures advance the sequence", + file_names: &["001-discovery.json", "002-plan_output.json"], + expected_number: 3, + }, + CaptureNumberTestCase { + description: "other files are ignored", + file_names: &[ + "001-discovery.json", + "notes.txt", + "x-1.json", + ".001-a.json.tmp", + ], + expected_number: 2, + }, + ]; + for test_case in test_cases { + assert_eq!( + next_capture_number(test_case.file_names.iter().copied()), + test_case.expected_number, + "{}", + test_case.description + ); + } +} diff --git a/crates/sqlbuild-cache/src/stage_capture/tests/test_types.rs b/crates/sqlbuild-cache/src/stage_capture/tests/test_types.rs index 9962247a4d..7a1bacf6ba 100644 --- a/crates/sqlbuild-cache/src/stage_capture/tests/test_types.rs +++ b/crates/sqlbuild-cache/src/stage_capture/tests/test_types.rs @@ -18,3 +18,9 @@ pub(crate) struct SortKeyTestCase { pub(crate) item: JsonValue, pub(crate) expected_key: &'static str, } + +pub(crate) struct CaptureNumberTestCase { + pub(crate) description: &'static str, + pub(crate) file_names: &'static [&'static str], + pub(crate) expected_number: u64, +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/analysis/explicit_references.rs b/crates/sqlbuild-python/src/bindings/_helpers/analysis/explicit_references.rs index c9f5446a7e..a45a5041db 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/analysis/explicit_references.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/analysis/explicit_references.rs @@ -30,12 +30,12 @@ use crate::bindings::_helpers::compile_driver::python_calls::imported; use crate::bindings::types::CompilerDetach; const EXPLICIT_REFERENCES_CONTEXT: &str = "native explicit-reference checks"; -const COMPILE_ERRORS_MODULE: &str = "sqlbuild.compiler.compile.exceptions"; +const COMPILE_ERRORS_MODULE: &str = "sqlbuild._native"; const COMPILE_INPUT_ERROR: &str = "CompileInputError"; const COMPILE_MODELS_MODULE: &str = "sqlbuild.compiler.compile.models"; const REFERENCE_MODELS_MODULE: &str = "sqlbuild.compiler.references.models"; const CONTRACT_MODELS_MODULE: &str = "sqlbuild._native"; -const REFERENCE_TYPES_MODULE: &str = "sqlbuild.compiler.references.types"; +const REFERENCE_TYPES_MODULE: &str = "sqlbuild._native"; const LOADER_DEFINITION_MODULE: &str = "sqlbuild.python_nodes.main.read_loader_definition"; const NATIVE_MODULE: &str = "sqlbuild._native"; const BUILTINS_MODULE: &str = "builtins"; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/analysis/reference_facts.rs b/crates/sqlbuild-python/src/bindings/_helpers/analysis/reference_facts.rs index d03eb26660..81e054e539 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/analysis/reference_facts.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/analysis/reference_facts.rs @@ -8,7 +8,7 @@ use sqlbuild_analysis::explicit_references::models::{ }; use sqlbuild_analysis::explicit_references::types::ResourceKey; -const DISCOVERY_MODELS_MODULE: &str = "sqlbuild.compiler.discovery.models"; +const DISCOVERY_MODELS_MODULE: &str = "sqlbuild._native"; const PYTHON_HOOK_ENTRY: &str = "PythonHookEntry"; const SQL_HOOK_ENTRY: &str = "SqlHookEntry"; const MODEL_HOOK_KEYS: [&str; 2] = ["pre_hooks", "post_hooks"]; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/attachments/cursor_intrinsics.rs b/crates/sqlbuild-python/src/bindings/_helpers/attachments/cursor_intrinsics.rs index 332a25bccd..ad6064e7bc 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/attachments/cursor_intrinsics.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/attachments/cursor_intrinsics.rs @@ -10,7 +10,7 @@ use sqlbuild_attachments::cursor_intrinsics::models::IntrinsicModel; use crate::bindings::_helpers::boundary::panics::compiler_guard; use crate::bindings::_helpers::compile_driver::python_calls::imported; -const COMPILE_ERRORS_MODULE: &str = "sqlbuild.compiler.compile.exceptions"; +const COMPILE_ERRORS_MODULE: &str = "sqlbuild._native"; const INCREMENTAL_MATERIALIZATION: &str = "incremental"; const CURSOR_START: &str = "__cursor_start"; const CURSOR_START_CALL: &str = "__cursor_start()"; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/attachments/reference_markers.rs b/crates/sqlbuild-python/src/bindings/_helpers/attachments/reference_markers.rs index a71a07949d..849c6f09cf 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/attachments/reference_markers.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/attachments/reference_markers.rs @@ -9,7 +9,7 @@ use sqlbuild_attachments::sql_lexing::main::unclosed_message::unclosed_message; use crate::bindings::_helpers::boundary::panics::compiler_guard; -const COMPILE_ERRORS_MODULE: &str = "sqlbuild.compiler.compile.exceptions"; +const COMPILE_ERRORS_MODULE: &str = "sqlbuild._native"; /// The reference function and diagnostic code of each marker kind, in the scanner's order. const MARKER_KINDS: [(&str, &str); 5] = [ ("__ref", "R001"), diff --git a/crates/sqlbuild-python/src/bindings/_helpers/cli/entry.rs b/crates/sqlbuild-python/src/bindings/_helpers/cli/entry.rs index c9dffcda47..b6bec490be 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/cli/entry.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/cli/entry.rs @@ -35,7 +35,7 @@ const OUTPUT_CAPTURE_ERRORS_MODULE: &str = "sqlbuild.runtime.output_capture.exce const DIAGNOSTICS_MODULE: &str = "sqlbuild.diagnostics.main.diagnostics_context"; const TRANSIENT_LINE_MODULE: &str = "sqlbuild._native"; const SKILLS_MODULE: &str = "sqlbuild._native"; -const RULES_ERRORS_MODULE: &str = "sqlbuild.rule_engine.exceptions"; +const RULES_ERRORS_MODULE: &str = "sqlbuild._native"; const LINT_ERRORS_MODULE: &str = "sqlbuild._native"; const CONTRACT_MODELS_MODULE: &str = "sqlbuild._native"; const NATIVE_MODULE: &str = "sqlbuild._native"; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/command_runtime/runtime_errors.rs b/crates/sqlbuild-python/src/bindings/_helpers/command_runtime/runtime_errors.rs index 1d8619cffc..33da4de19c 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/command_runtime/runtime_errors.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/command_runtime/runtime_errors.rs @@ -8,7 +8,7 @@ use crate::bindings::_helpers::compile_driver::python_calls::imported; const CLI_ERRORS_MODULE: &str = "sqlbuild._native"; const DISCOVERY_ERRORS_MODULE: &str = "sqlbuild._native"; -const COMPILE_ERRORS_MODULE: &str = "sqlbuild.compiler.compile.exceptions"; +const COMPILE_ERRORS_MODULE: &str = "sqlbuild._native"; /// The `CliUserError` or `ProjectConfigError` for `error`. pub(crate) fn raised(py: Python<'_>, error: CommandConnectionError) -> PyErr { diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compile_attachment/declaration_scope.rs b/crates/sqlbuild-python/src/bindings/_helpers/compile_attachment/declaration_scope.rs index 00e47e1ad6..30b2b65cc1 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compile_attachment/declaration_scope.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compile_attachment/declaration_scope.rs @@ -15,7 +15,7 @@ const NATIVE_RECORDS_MODULE: &str = "sqlbuild._native"; const NATIVE_MODULE: &str = "sqlbuild._native"; const SCAN_CACHE_MODULE: &str = "sqlbuild._native"; -const COMPILE_ERRORS_MODULE: &str = "sqlbuild.compiler.compile.exceptions"; +const COMPILE_ERRORS_MODULE: &str = "sqlbuild._native"; const SCOPE_ERRORS_MODULE: &str = "sqlbuild._native"; fn compile_error(py: Python<'_>, message: Bound<'_, PyAny>) -> PyResult { diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compile_attachment/effective_config.rs b/crates/sqlbuild-python/src/bindings/_helpers/compile_attachment/effective_config.rs index c390e9eb03..0a524f17fa 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compile_attachment/effective_config.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compile_attachment/effective_config.rs @@ -15,7 +15,7 @@ use crate::bindings::_helpers::model_config::template_data::{ }; const NATIVE_MODULE: &str = "sqlbuild._native"; -const COMPILE_ERRORS_MODULE: &str = "sqlbuild.compiler.compile.exceptions"; +const COMPILE_ERRORS_MODULE: &str = "sqlbuild._native"; const CONTRACT_MODELS_MODULE: &str = "sqlbuild._native"; const CONNECTION_LABEL: &str = "effective connection"; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compile_attachment/model_config.rs b/crates/sqlbuild-python/src/bindings/_helpers/compile_attachment/model_config.rs index a53bf2781d..42bf143684 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compile_attachment/model_config.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compile_attachment/model_config.rs @@ -13,7 +13,7 @@ use crate::bindings::_helpers::model_config::unicode_environment::unicode_enviro const NATIVE_RECORDS_MODULE: &str = "sqlbuild._native"; const NATIVE_MODULE: &str = "sqlbuild._native"; -const DISCOVERY_MODELS_MODULE: &str = "sqlbuild.compiler.discovery.models"; +const DISCOVERY_MODELS_MODULE: &str = "sqlbuild._native"; const CONTRACT_MODELS_MODULE: &str = "sqlbuild._native"; /// Capture the project inputs every model's native config build and validation read. diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compile_attachment/model_inputs.rs b/crates/sqlbuild-python/src/bindings/_helpers/compile_attachment/model_inputs.rs index b5b0a06f01..bda1aed684 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compile_attachment/model_inputs.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compile_attachment/model_inputs.rs @@ -20,9 +20,9 @@ const NATIVE_RECORDS_MODULE: &str = "sqlbuild._native"; const NATIVE_MODULE: &str = "sqlbuild._native"; const COMPILE_MODELS_MODULE: &str = "sqlbuild.compiler.compile.models"; -const COMPILE_ERRORS_MODULE: &str = "sqlbuild.compiler.compile.exceptions"; -const DISCOVERY_MODELS_MODULE: &str = "sqlbuild.compiler.discovery.models"; -const DISCOVERY_CONSTANTS_MODULE: &str = "sqlbuild.compiler.discovery.constants"; +const COMPILE_ERRORS_MODULE: &str = "sqlbuild._native"; +const DISCOVERY_MODELS_MODULE: &str = "sqlbuild._native"; +const PROJECT_CONFIG_FILENAME: &str = "sqlbuild_project.toml"; const SCOPE_MODELS_MODULE: &str = "sqlbuild._native"; const SCOPE_TYPES_MODULE: &str = "sqlbuild._native"; const COLLECTOR_MODULE: &str = "sqlbuild._native"; @@ -211,11 +211,7 @@ pub(crate) fn build_model_inputs<'py>( } else { imported(py, "pathlib", "Path")?.call0()? }; - let project_config_path: Bound<'py, PyAny> = root.div(imported( - py, - DISCOVERY_CONSTANTS_MODULE, - "PROJECT_CONFIG_FILENAME", - )?)?; + let project_config_path: Bound<'py, PyAny> = root.div(PROJECT_CONFIG_FILENAME)?; let config_inputs: Bound<'py, PyAny> = call_with( &imported(py, NATIVE_RECORDS_MODULE, "NativeModelConfigInputs")?, &[ diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/assembly_sql_tests.rs b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/assembly_sql_tests.rs index 543432df89..e085317b33 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/assembly_sql_tests.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/assembly_sql_tests.rs @@ -10,7 +10,7 @@ use crate::bindings::_helpers::compile_driver::python_calls::{call_with, importe const NATIVE_RECORDS_MODULE: &str = "sqlbuild._native"; const NATIVE_MODULE: &str = "sqlbuild._native"; -const COMPILE_ERRORS_MODULE: &str = "sqlbuild.compiler.compile.exceptions"; +const COMPILE_ERRORS_MODULE: &str = "sqlbuild._native"; const COLLECTOR_MODULE: &str = "sqlbuild._native"; const DECIMAL_MODULE: &str = "decimal"; const DATACLASSES_MODULE: &str = "dataclasses"; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/diagnostic_collector.rs b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/diagnostic_collector.rs index 5b151c0955..f2b8f5325c 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/diagnostic_collector.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/diagnostic_collector.rs @@ -9,7 +9,7 @@ use pyo3::{PyErr, pyclass, pyfunction, pymethods, wrap_pyfunction}; use crate::bindings::_helpers::compile_driver::python_calls::{context_variable, imported}; -const COMPILE_ERRORS_MODULE: &str = "sqlbuild.compiler.compile.exceptions"; +const COMPILE_ERRORS_MODULE: &str = "sqlbuild._native"; const CONTEXT_VARIABLE_NAME: &str = "sqlbuild_collected_compile_diagnostics"; static ACTIVE_COLLECTOR: PyOnceLock> = PyOnceLock::new(); diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/phases.rs b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/phases.rs index 255780afe9..043957d495 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/phases.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/phases.rs @@ -18,7 +18,7 @@ const NATIVE_RECORDS_MODULE: &str = "sqlbuild._native"; const NATIVE_MODULE: &str = "sqlbuild._native"; const ADAPTER_NAME_MODULE: &str = "sqlbuild._native"; const OUTPUT_MODELS_MODULE: &str = "sqlbuild._native"; -const COMPILE_EXCEPTIONS_MODULE: &str = "sqlbuild.compiler.compile.exceptions"; +const COMPILE_EXCEPTIONS_MODULE: &str = "sqlbuild._native"; const PLANNER_EXCEPTIONS_MODULE: &str = "sqlbuild._native"; const COMPILE_MODELS_MODULE: &str = "sqlbuild.compiler.compile.models"; const EMPTY_DAG_PATH: &str = ""; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/pipeline.rs b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/pipeline.rs index 063eda4ee2..0b9ccb6019 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/pipeline.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/pipeline.rs @@ -22,11 +22,11 @@ const NATIVE_RECORDS_MODULE: &str = "sqlbuild._native"; const NATIVE_MODULE: &str = "sqlbuild._native"; const EFFECTIVE_CONFIG_MODULE: &str = "sqlbuild._native"; -const COMPILE_EXCEPTIONS_MODULE: &str = "sqlbuild.compiler.compile.exceptions"; +const COMPILE_EXCEPTIONS_MODULE: &str = "sqlbuild._native"; const COMPILE_TYPES_MODULE: &str = "sqlbuild.compiler.compile.types"; const ANALYSIS_SELECTION_MODULE: &str = "sqlbuild._native"; const TIMING_TRACKER_MODULE: &str = "sqlbuild._native"; -const RULE_EXCEPTIONS_MODULE: &str = "sqlbuild.rule_engine.exceptions"; +const RULE_EXCEPTIONS_MODULE: &str = "sqlbuild._native"; const RULE_THREADS_MODULE: &str = "sqlbuild._native"; const RULE_MODELS_MODULE: &str = "sqlbuild.rule_engine.models"; const RUNTIME_MODELS_MODULE: &str = "sqlbuild.runtime.contracts.models"; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/pipeline_entries.rs b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/pipeline_entries.rs index ce5ca3fe7d..5b964605ec 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/pipeline_entries.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/pipeline_entries.rs @@ -10,7 +10,7 @@ use crate::bindings::_helpers::compile_driver::python_calls::{ const NATIVE_RECORDS_MODULE: &str = "sqlbuild._native"; -const PIPELINE_MODELS_MODULE: &str = "sqlbuild.compiler.pipeline.models"; +const PIPELINE_MODELS_MODULE: &str = "sqlbuild._native"; const PLANNER_MODELS_MODULE: &str = "sqlbuild._native"; const NATIVE_MODULE: &str = "sqlbuild._native"; /// Discovered Python node functions whose selectors need the Python graph. diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/plan_result.rs b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/plan_result.rs index 7821a065e4..a2660a48e6 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/plan_result.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/plan_result.rs @@ -17,7 +17,7 @@ const NATIVE_RECORDS_MODULE: &str = "sqlbuild._native"; const NATIVE_MODULE: &str = "sqlbuild._native"; const EXECUTION_PLAN_MODULE: &str = "sqlbuild._native"; const MATERIALIZATIONS_MODULE: &str = "sqlbuild.compiler.pipeline._helpers.materializations"; -const PIPELINE_MODELS_MODULE: &str = "sqlbuild.compiler.pipeline.models"; +const PIPELINE_MODELS_MODULE: &str = "sqlbuild._native"; const SINGLE_CONNECTION: i64 = 1; /// The deferred relation locations and the warehouse relations behind them, when deferring. diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/project.rs b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/project.rs index 81b9bcee82..5e566a6b29 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/project.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/project.rs @@ -26,7 +26,8 @@ const BUILD_INPUTS_MODULE: &str = "sqlbuild._native"; const ASSEMBLE_MODULE: &str = "sqlbuild._native"; const PROJECT_WITH_MODULE: &str = "sqlbuild._native"; const PROMOTION_MODULE: &str = "sqlbuild._native"; -const DISCOVERY_CONSTANTS_MODULE: &str = "sqlbuild.compiler.discovery.constants"; +const PROJECT_CONFIG_FILENAME: &str = "sqlbuild_project.toml"; +const TABLE_PROMOTION_MODE_SETTING_KEY: &str = "table_promotion_mode"; const LOCAL_CONFIG_FILENAME: &str = "sqlbuild_local.toml"; const ANALYSIS_SELECTION_MODULE: &str = "sqlbuild._native"; const ADAPTER_NAME_MODULE: &str = "sqlbuild._native"; @@ -231,18 +232,15 @@ fn targeted_project<'py>( ], )?; project_targets(&adapter_name, &project)?; - let promotion_key: Bound<'py, PyAny> = imported( - py, - DISCOVERY_CONSTANTS_MODULE, - "TABLE_PROMOTION_MODE_SETTING_KEY", - )?; + let promotion_key: Bound<'py, PyAny> = + pyo3::types::PyString::new(py, TABLE_PROMOTION_MODE_SETTING_KEY).into_any(); let settings_file: Bound<'py, PyAny> = if local_config .getattr("setting_overrides")? .contains(promotion_key)? { pyo3::types::PyString::new(py, LOCAL_CONFIG_FILENAME).into_any() } else { - imported(py, DISCOVERY_CONSTANTS_MODULE, "PROJECT_CONFIG_FILENAME")? + pyo3::types::PyString::new(py, PROJECT_CONFIG_FILENAME).into_any() }; let conflicts: Bound<'py, PyAny> = call_with( &imported(py, PROMOTION_MODULE, "promotion_conflict_diagnostics")?, diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compile_records/constants.rs b/crates/sqlbuild-python/src/bindings/_helpers/compile_records/constants.rs index 5a0cf88c03..35b0657118 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compile_records/constants.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compile_records/constants.rs @@ -792,6 +792,381 @@ const R_TOLERANT_SCOPE_DISCOVERY: RecordSpec = RecordSpec { FieldSpec::new("config_faults", FieldDefault::EmptyTuple), ], }; +const R_SQL_HOOK_ENTRY: RecordSpec = RecordSpec { + name: "SqlHookEntry", + doc: "A model lifecycle hook that executes SQL.", + fields: &[ + FieldSpec::required("statement"), + FieldSpec::new("name", FieldDefault::None), + FieldSpec::new("relative_path", FieldDefault::None), + FieldSpec::new("definition_sql", FieldDefault::None), + FieldSpec::new("kwargs", FieldDefault::None), + FieldSpec::new("description", FieldDefault::None), + FieldSpec::new("reads", FieldDefault::EmptyTuple), + ], +}; +const R_PYTHON_HOOK_ENTRY: RecordSpec = RecordSpec { + name: "PythonHookEntry", + doc: "A model lifecycle hook that invokes a discovered Python hook.", + fields: &[FieldSpec::required("name"), FieldSpec::required("kwargs")], +}; +const R_ENUM_DECLARATION: RecordSpec = RecordSpec { + name: "EnumDeclaration", + doc: "One validated public or model-local enum declaration.", + fields: &[ + FieldSpec::required("name"), + FieldSpec::required("members"), + FieldSpec::required("scalar_type"), + FieldSpec::required("relative_path"), + FieldSpec::new("model_name", FieldDefault::None), + ], +}; +const R_CONSTANT_DECLARATION: RecordSpec = RecordSpec { + name: "ConstantDeclaration", + doc: "One validated public or model-local constant declaration.", + fields: &[ + FieldSpec::required("name"), + FieldSpec::required("value"), + FieldSpec::required("relative_path"), + FieldSpec::new("model_name", FieldDefault::None), + FieldSpec::new("render_as", FieldDefault::None), + ], +}; +const R_DISCOVERED_SOURCE_FILE: RecordSpec = RecordSpec { + name: "DiscoveredSourceFile", + doc: "A discovered source declaration file and its raw contents.", + fields: &[ + FieldSpec::required("file_path"), + FieldSpec::required("relative_path"), + FieldSpec::required("contents"), + FieldSpec::required("source_entries"), + ], +}; +const R_DISCOVERED_SQL_TEST_FILE: RecordSpec = RecordSpec { + name: "DiscoveredSqlTestFile", + doc: "A discovered SQL-native test file and its raw contents.", + fields: &[ + FieldSpec::required("file_path"), + FieldSpec::required("relative_path"), + FieldSpec::required("contents"), + FieldSpec::required("blocks"), + FieldSpec::new( + "ownership_root", + FieldDefault::CallText("pathlib", "Path", "tests/unit"), + ), + ], +}; +const R_SQL_TEST_PARAMETER_DECLARATION: RecordSpec = RecordSpec { + name: "SqlTestParameterDeclaration", + doc: "One typed parameter declared by a SQL-native test template.", + fields: &[ + FieldSpec::required("name"), + FieldSpec::required("value_type"), + FieldSpec::new("nullable", FieldDefault::False), + ], +}; +const R_DISCOVERED_SQL_SCENARIO_FILE: RecordSpec = RecordSpec { + name: "DiscoveredSqlScenarioFile", + doc: "A discovered SQL-native scenario file and its raw contents.", + fields: &[ + FieldSpec::required("file_path"), + FieldSpec::required("relative_path"), + FieldSpec::required("contents"), + FieldSpec::required("header_values"), + FieldSpec::required("sql_body"), + FieldSpec::required("name"), + FieldSpec::new( + "ownership_root", + FieldDefault::CallText("pathlib", "Path", "tests/scenarios"), + ), + FieldSpec::new("sql_body_span", FieldDefault::None), + ], +}; +const R_DISCOVERED_MACRO_FILE: RecordSpec = RecordSpec { + name: "DiscoveredMacroFile", + doc: "A discovered project macro file and its raw contents.", + fields: &[ + FieldSpec::required("file_path"), + FieldSpec::required("relative_path"), + FieldSpec::required("contents"), + FieldSpec::new( + "scope_kind", + FieldDefault::Member("sqlbuild._native", "ScopeKind", "GLOBAL"), + ), + FieldSpec::new("ownership_root", FieldDefault::None), + FieldSpec::new("owning_path", FieldDefault::None), + FieldSpec::new("declaration_root", FieldDefault::None), + ], +}; +const R_DISCOVERED_MATERIALIZATION_FILE: RecordSpec = RecordSpec { + name: "DiscoveredMaterializationFile", + doc: "A discovered custom materialization Python file.", + fields: &[ + FieldSpec::required("file_path"), + FieldSpec::required("relative_path"), + FieldSpec::required("name"), + FieldSpec::new("provider_usages", FieldDefault::EmptyTuple), + ], +}; +const R_DISCOVERED_PROVIDER_USAGE: RecordSpec = RecordSpec { + name: "DiscoveredProviderUsage", + doc: "Framework-owned metadata for provider usage by a Python callable surface.", + fields: &[ + FieldSpec::required("provider_name"), + FieldSpec::required("parameter_name"), + FieldSpec::new("annotation_class_name", FieldDefault::None), + FieldSpec::new("annotation_module", FieldDefault::None), + ], +}; +const R_DISCOVERED_LOADER_FUNCTION: RecordSpec = RecordSpec { + name: "DiscoveredLoaderFunction", + doc: "A discovered project source loader function.", + fields: &[ + FieldSpec::required("file_path"), + FieldSpec::required("relative_path"), + FieldSpec::required("name"), + FieldSpec::required("function"), + FieldSpec::new("depends_on", FieldDefault::EmptyTuple), + FieldSpec::new("destination", FieldDefault::None), + FieldSpec::new("write_strategy", FieldDefault::None), + FieldSpec::new("cursor_column", FieldDefault::None), + FieldSpec::new("unique_key", FieldDefault::EmptyTuple), + FieldSpec::new("columns", FieldDefault::EmptyTuple), + FieldSpec::new("contract", FieldDefault::None), + FieldSpec::new("description", FieldDefault::None), + FieldSpec::new( + "connection_mode", + FieldDefault::Member("sqlbuild._native", "LoaderConnectionMode", "SQLBUILD"), + ), + FieldSpec::new("provider_usages", FieldDefault::EmptyTuple), + ], +}; +const R_DISCOVERED_TASK_FUNCTION: RecordSpec = RecordSpec { + name: "DiscoveredTaskFunction", + doc: "A discovered project task function.", + fields: &[ + FieldSpec::required("file_path"), + FieldSpec::required("relative_path"), + FieldSpec::required("name"), + FieldSpec::required("function"), + FieldSpec::new("depends_on", FieldDefault::EmptyTuple), + FieldSpec::new("tags", FieldDefault::EmptyTuple), + FieldSpec::new("group", FieldDefault::None), + FieldSpec::new("description", FieldDefault::None), + FieldSpec::new("meta", FieldDefault::None), + FieldSpec::new("retry", FieldDefault::None), + FieldSpec::new("provider_usages", FieldDefault::EmptyTuple), + ], +}; +const R_DISCOVERED_ASSET_FUNCTION: RecordSpec = RecordSpec { + name: "DiscoveredAssetFunction", + doc: "A discovered project asset function.", + fields: &[ + FieldSpec::required("file_path"), + FieldSpec::required("relative_path"), + FieldSpec::required("name"), + FieldSpec::required("function"), + FieldSpec::new("depends_on", FieldDefault::EmptyTuple), + FieldSpec::new("tags", FieldDefault::EmptyTuple), + FieldSpec::new("group", FieldDefault::None), + FieldSpec::new("description", FieldDefault::None), + FieldSpec::new("meta", FieldDefault::None), + FieldSpec::new("columns", FieldDefault::EmptyTuple), + FieldSpec::new("column_lineage", FieldDefault::None), + FieldSpec::new("retry", FieldDefault::None), + FieldSpec::new("provider_usages", FieldDefault::EmptyTuple), + ], +}; +const R_DISCOVERED_CHECK_FUNCTION: RecordSpec = RecordSpec { + name: "DiscoveredCheckFunction", + doc: "A discovered project check function.", + fields: &[ + FieldSpec::required("file_path"), + FieldSpec::required("relative_path"), + FieldSpec::required("name"), + FieldSpec::required("function"), + FieldSpec::required("depends_on"), + FieldSpec::new( + "severity", + FieldDefault::Member( + "sqlbuild.python_nodes.types", + "PythonCheckSeverity", + "ERROR", + ), + ), + FieldSpec::new("tags", FieldDefault::EmptyTuple), + FieldSpec::new("group", FieldDefault::None), + FieldSpec::new("description", FieldDefault::None), + FieldSpec::new("meta", FieldDefault::None), + FieldSpec::new("provider_usages", FieldDefault::EmptyTuple), + ], +}; +const R_DISCOVERED_HOOK_FUNCTION: RecordSpec = RecordSpec { + name: "DiscoveredHookFunction", + doc: "A discovered project model lifecycle hook function.", + fields: &[ + FieldSpec::required("file_path"), + FieldSpec::required("relative_path"), + FieldSpec::required("name"), + FieldSpec::required("function"), + FieldSpec::new("description", FieldDefault::None), + FieldSpec::new("reads", FieldDefault::EmptyTuple), + FieldSpec::new("provider_usages", FieldDefault::EmptyTuple), + FieldSpec::new( + "scope_kind", + FieldDefault::Member("sqlbuild._native", "ScopeKind", "GLOBAL"), + ), + FieldSpec::new("ownership_root", FieldDefault::None), + FieldSpec::new("owning_path", FieldDefault::None), + FieldSpec::new("declaration_root", FieldDefault::None), + ], +}; +const R_DISCOVERED_AUDIT_FACTORY: RecordSpec = RecordSpec { + name: "DiscoveredAuditFactory", + doc: "A discovered audit factory and its normalized cases.", + fields: &[ + FieldSpec::required("name"), + FieldSpec::required("function"), + FieldSpec::required("file_path"), + FieldSpec::required("relative_path"), + FieldSpec::required("line"), + FieldSpec::new("cases", FieldDefault::EmptyTuple), + ], +}; +const R_DISCOVERED_EVENT_EXPORTER_DECLARATION: RecordSpec = RecordSpec { + name: "DiscoveredEventExporterDeclaration", + doc: "A discovered exporter declaration before provider binding.", + fields: &[ + FieldSpec::required("file_path"), + FieldSpec::required("relative_path"), + FieldSpec::required("name"), + FieldSpec::required("function"), + FieldSpec::required("event_kinds"), + FieldSpec::required("min_severity"), + ], +}; +const R_DISCOVERED_EVENT_EXPORTER: RecordSpec = RecordSpec { + name: "DiscoveredEventExporter", + doc: "A discovered canonical lifecycle event exporter function.", + fields: &[ + FieldSpec::required("file_path"), + FieldSpec::required("relative_path"), + FieldSpec::required("name"), + FieldSpec::required("function"), + FieldSpec::new( + "event_kinds", + FieldDefault::TextFrozenset(&[ + "audit", + "invocation", + "microbatch", + "operation", + "resource", + "retry", + "run", + "statement", + ]), + ), + FieldSpec::new( + "min_severity", + FieldDefault::Member("sqlbuild._native", "EventExportSeverity", "DEBUG"), + ), + FieldSpec::new("provider_usages", FieldDefault::EmptyTuple), + ], +}; +const R_DISCOVERED_COMMAND_OUTPUT_SINK_DECLARATION: RecordSpec = RecordSpec { + name: "DiscoveredCommandOutputSinkDeclaration", + doc: "A command-output sink declaration before provider binding.", + fields: &[ + FieldSpec::required("file_path"), + FieldSpec::required("relative_path"), + FieldSpec::required("name"), + FieldSpec::required("function"), + FieldSpec::required("streams"), + ], +}; +const R_DISCOVERED_COMMAND_OUTPUT_SINK: RecordSpec = RecordSpec { + name: "DiscoveredCommandOutputSink", + doc: "A discovered command-output sink function.", + fields: &[ + FieldSpec::required("file_path"), + FieldSpec::required("relative_path"), + FieldSpec::required("name"), + FieldSpec::required("function"), + FieldSpec::required("streams"), + FieldSpec::new("provider_usages", FieldDefault::EmptyTuple), + ], +}; +const R_DISCOVERED_PROVIDER: RecordSpec = RecordSpec { + name: "DiscoveredProvider", + doc: "A discovered project provider class and validated settings object.", + fields: &[ + FieldSpec::required("file_path"), + FieldSpec::required("relative_path"), + FieldSpec::required("name"), + FieldSpec::required("provider_class"), + FieldSpec::required("settings"), + ], +}; +const R_DISCOVERED_PYTHON_NODE_FUNCTIONS: RecordSpec = RecordSpec { + name: "DiscoveredPythonNodeFunctions", + doc: "Discovered project Python DAG node functions grouped by kind.", + fields: &[ + FieldSpec::new("loaders", FieldDefault::EmptyTuple), + FieldSpec::new("tasks", FieldDefault::EmptyTuple), + FieldSpec::new("assets", FieldDefault::EmptyTuple), + FieldSpec::new("checks", FieldDefault::EmptyTuple), + FieldSpec::new("audit_factories", FieldDefault::EmptyTuple), + ], +}; +const R_DISCOVERED_PROJECT_INPUTS: RecordSpec = RecordSpec { + name: "DiscoveredProjectInputs", + doc: "All raw project inputs discovered from disk before semantic resolution.", + fields: &[ + FieldSpec::required("project_config"), + FieldSpec::required("local_config"), + FieldSpec::new("project_dir", FieldDefault::None), + FieldSpec::new("model_files", FieldDefault::EmptyTuple), + FieldSpec::new("enum_files", FieldDefault::EmptyTuple), + FieldSpec::new("constant_files", FieldDefault::EmptyTuple), + FieldSpec::new("model_schema_files", FieldDefault::EmptyTuple), + FieldSpec::new("sql_function_files", FieldDefault::EmptyTuple), + FieldSpec::new("sql_hook_files", FieldDefault::EmptyTuple), + FieldSpec::new("python_function_files", FieldDefault::EmptyTuple), + FieldSpec::new("schema_files", FieldDefault::EmptyTuple), + FieldSpec::new("source_files", FieldDefault::EmptyTuple), + FieldSpec::new("seed_files", FieldDefault::EmptyTuple), + FieldSpec::new("test_files", FieldDefault::EmptyTuple), + FieldSpec::new("scenario_files", FieldDefault::EmptyTuple), + FieldSpec::new("audit_files", FieldDefault::EmptyTuple), + FieldSpec::new("macro_files", FieldDefault::EmptyTuple), + FieldSpec::new("materialization_files", FieldDefault::EmptyTuple), + FieldSpec::new("loader_functions", FieldDefault::EmptyTuple), + FieldSpec::new("task_functions", FieldDefault::EmptyTuple), + FieldSpec::new("asset_functions", FieldDefault::EmptyTuple), + FieldSpec::new("check_functions", FieldDefault::EmptyTuple), + FieldSpec::new("audit_factories", FieldDefault::EmptyTuple), + FieldSpec::new("hook_functions", FieldDefault::EmptyTuple), + FieldSpec::new("event_exporters", FieldDefault::EmptyTuple), + FieldSpec::new("command_output_sinks", FieldDefault::EmptyTuple), + FieldSpec::new("providers", FieldDefault::EmptyTuple), + FieldSpec::new("adapter_file", FieldDefault::None), + FieldSpec { + name: "native_session", + default: FieldDefault::None, + compared: false, + }, + ], +}; +const R_DISCOVERED_RUNTIME_EXTENSIONS: RecordSpec = RecordSpec { + name: "DiscoveredRuntimeExtensions", + doc: "Command-startup providers, typed sinks, and lifecycle delivery settings.", + fields: &[ + FieldSpec::new("providers", FieldDefault::EmptyTuple), + FieldSpec::new("event_exporters", FieldDefault::EmptyTuple), + FieldSpec::new("command_output_sinks", FieldDefault::EmptyTuple), + FieldSpec::new("lifecycle_shutdown_timeout_seconds", FieldDefault::None), + ], +}; /// The compile and discovery value records. pub(crate) const COMPILE_VALUE_RECORDS: &[RecordSpec] = &[ @@ -857,4 +1232,29 @@ pub(crate) const COMPILE_VALUE_RECORDS: &[RecordSpec] = &[ R_DISCOVERED_ADAPTER_FILE, R_DISCOVERY_FILE_FAULT, R_TOLERANT_SCOPE_DISCOVERY, + R_SQL_HOOK_ENTRY, + R_PYTHON_HOOK_ENTRY, + R_ENUM_DECLARATION, + R_CONSTANT_DECLARATION, + R_DISCOVERED_SOURCE_FILE, + R_DISCOVERED_SQL_TEST_FILE, + R_SQL_TEST_PARAMETER_DECLARATION, + R_DISCOVERED_SQL_SCENARIO_FILE, + R_DISCOVERED_MACRO_FILE, + R_DISCOVERED_MATERIALIZATION_FILE, + R_DISCOVERED_PROVIDER_USAGE, + R_DISCOVERED_LOADER_FUNCTION, + R_DISCOVERED_TASK_FUNCTION, + R_DISCOVERED_ASSET_FUNCTION, + R_DISCOVERED_CHECK_FUNCTION, + R_DISCOVERED_HOOK_FUNCTION, + R_DISCOVERED_AUDIT_FACTORY, + R_DISCOVERED_EVENT_EXPORTER_DECLARATION, + R_DISCOVERED_EVENT_EXPORTER, + R_DISCOVERED_COMMAND_OUTPUT_SINK_DECLARATION, + R_DISCOVERED_COMMAND_OUTPUT_SINK, + R_DISCOVERED_PROVIDER, + R_DISCOVERED_PYTHON_NODE_FUNCTIONS, + R_DISCOVERED_PROJECT_INPUTS, + R_DISCOVERED_RUNTIME_EXTENSIONS, ]; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/attachment.rs b/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/attachment.rs index de2f57c822..39daf1c150 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/attachment.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/attachment.rs @@ -28,7 +28,7 @@ use crate::bindings::_helpers::sql_test_glue::constants::SQL_TEST_CTE_SCAN_ALGOR const NATIVE_RECORDS_MODULE: &str = "sqlbuild._native"; const NATIVE_RECORDS: &str = "sqlbuild._native"; -const COMPILE_ERRORS_MODULE: &str = "sqlbuild.compiler.compile.exceptions"; +const COMPILE_ERRORS_MODULE: &str = "sqlbuild._native"; const COMPILE_MODELS_MODULE: &str = "sqlbuild.compiler.compile.models"; const COMPILE_TYPES_MODULE: &str = "sqlbuild.compiler.compile.types"; const SCOPE_TYPES_MODULE: &str = "sqlbuild._native"; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/loop_classes.rs b/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/loop_classes.rs index 6021852ebf..4759a5be83 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/loop_classes.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/loop_classes.rs @@ -10,12 +10,12 @@ const NATIVE_RECORDS_MODULE: &str = "sqlbuild._native"; const COMPILE_MODELS_MODULE: &str = "sqlbuild.compiler.compile.models"; const COMPILE_TYPES_MODULE: &str = "sqlbuild.compiler.compile.types"; -const COMPILE_ERRORS_MODULE: &str = "sqlbuild.compiler.compile.exceptions"; +const COMPILE_ERRORS_MODULE: &str = "sqlbuild._native"; const AUDITING_TYPES_MODULE: &str = "sqlbuild.compiler.auditing.types"; -const DISCOVERY_MODELS_MODULE: &str = "sqlbuild.compiler.discovery.models"; +const DISCOVERY_MODELS_MODULE: &str = "sqlbuild._native"; const LINEAGE_TYPES_MODULE: &str = "sqlbuild.compiler.lineage.types"; const PLANNER_TYPES_MODULE: &str = "sqlbuild._native"; -const REFERENCE_TYPES_MODULE: &str = "sqlbuild.compiler.references.types"; +const REFERENCE_TYPES_MODULE: &str = "sqlbuild._native"; const SCOPE_ERRORS_MODULE: &str = "sqlbuild._native"; const SCOPE_MODELS_MODULE: &str = "sqlbuild._native"; const SCOPE_TYPES_MODULE: &str = "sqlbuild._native"; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compiler_enums/constants.rs b/crates/sqlbuild-python/src/bindings/_helpers/compiler_enums/constants.rs index 80c9460645..cd6583d2f6 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compiler_enums/constants.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compiler_enums/constants.rs @@ -676,6 +676,28 @@ const E_SCENARIO_ARTIFACT_KIND: EnumSpec = EnumSpec { ("MODEL", "model"), ], }; +const E_SQL_REFERENCE_KIND: EnumSpec = EnumSpec { + name: "SqlReferenceKind", + doc: "", + members: &[ + ("REF", "ref"), + ("SEED", "seed"), + ("SOURCE", "source"), + ("DBT_REF", "dbt_ref"), + ("UDF", "udf"), + ("TABLE_FUNCTION", "table_fn"), + ], +}; +const E_HARD_CODED_RELATION_OWNER_KIND: EnumSpec = EnumSpec { + name: "HardCodedRelationOwnerKind", + doc: "The kind of Python code that names a project relation in SQL.", + members: &[("NODE", "node"), ("HOOK", "hook"), ("LOADER", "loader")], +}; +const E_LOADER_CONNECTION_MODE: EnumSpec = EnumSpec { + name: "LoaderConnectionMode", + doc: "How a source loader owns the destination warehouse connection.", + members: &[("SQLBUILD", "sqlbuild"), ("EXTERNAL", "external")], +}; /// Compiler, rule, lint and config string enums. pub(crate) const COMPILER_ENUMS: &[EnumSpec] = &[ @@ -746,4 +768,7 @@ pub(crate) const COMPILER_ENUMS: &[EnumSpec] = &[ E_CURSOR_GRAIN, E_MATERIALIZATION_TYPE, E_SCENARIO_ARTIFACT_KIND, + E_SQL_REFERENCE_KIND, + E_HARD_CODED_RELATION_OWNER_KIND, + E_LOADER_CONNECTION_MODE, ]; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compiler_errors/constants.rs b/crates/sqlbuild-python/src/bindings/_helpers/compiler_errors/constants.rs index fc73ecff02..2493399577 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compiler_errors/constants.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compiler_errors/constants.rs @@ -419,4 +419,103 @@ pub(crate) const COMPILER_ERRORS: &[ErrorSpec] = &[ optional: &["evidence"], fixed: &[], }, + ErrorSpec { + name: "CompileInputError", + parent: VALUE_ERROR_PARENT, + doc: "Raised when discovered inputs cannot be attached into a compile view.", + class_attributes: &[("code", ClassValue::Text("P001"))], + required: &[], + optional: &[], + fixed: &[], + }, + ErrorSpec { + name: "MacroArgumentError", + parent: "CompileInputError", + doc: "Raised when one macro call's arguments do not parse; `offset` is in the expanded SQL.", + class_attributes: &[], + required: &[], + optional: &[], + fixed: &[], + }, + ErrorSpec { + name: "SqlTestReferenceError", + parent: "CompileInputError", + doc: "Raised when a SQL-test CTE calls a reference the test query cannot resolve.", + class_attributes: &[("code", ClassValue::Text("P013"))], + required: &[], + optional: &[], + fixed: &[], + }, + ErrorSpec { + name: "RulesError", + parent: "builtins.RuntimeError", + doc: "Raised for invalid compiler rules or execution.", + class_attributes: &[], + required: &[], + optional: &[], + fixed: &[], + }, + ErrorSpec { + name: "RulesConfigError", + parent: "RulesError", + doc: "Raised when rules configuration breaks a project policy; help shows the setting to change.", + class_attributes: &[], + required: &[], + optional: &[], + fixed: &[], + }, + ErrorSpec { + name: "RuleUsageError", + parent: "RulesError", + doc: "Raised when a rule or rule test uses the public API incorrectly.", + class_attributes: &[], + required: &[], + optional: &[], + fixed: &[], + }, + ErrorSpec { + name: "NonHermeticRuleError", + parent: "RulesError", + doc: "Raised when a custom rule observes state the Rules cache cannot track.", + class_attributes: &[], + required: &[], + optional: &[], + fixed: &[], + }, + ErrorSpec { + name: "RuleAssertionError", + parent: "RulesError", + doc: "Raised when a custom-rule harness case produces an unexpected result.", + class_attributes: &[], + required: &[], + optional: &[], + fixed: &[], + }, + ErrorSpec { + name: "FactDigestError", + parent: "RulesError", + doc: "Raised when a recorded custom-rule fact cannot be recomputed or encoded deterministically.", + class_attributes: &[], + required: &[], + optional: &[], + fixed: &[], + }, + ErrorSpec { + name: "OpaqueModuleStateError", + parent: "RulesError", + doc: "Raised when custom-rule module state holds a value whose mutations cannot be observed.", + class_attributes: &[], + required: &[], + optional: &[], + fixed: &[], + }, + ErrorSpec { + name: "HostCancelledError", + parent: "RulesError", + doc: "Raised in a custom-rule host that stops because another host already failed.", + class_attributes: &[], + required: &[], + optional: &[], + fixed: &[], + }, ]; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compiler_records/constants.rs b/crates/sqlbuild-python/src/bindings/_helpers/compiler_records/constants.rs index 85baffdc92..0e2151ad76 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compiler_records/constants.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compiler_records/constants.rs @@ -584,11 +584,7 @@ const R_DISCOVERED_PYTHON_LOADER_METADATA: RecordSpec = RecordSpec { FieldSpec::new("contract", FieldDefault::None), FieldSpec::new( "connection_mode", - FieldDefault::Member( - "sqlbuild.compiler.discovery.types", - "LoaderConnectionMode", - "SQLBUILD", - ), + FieldDefault::Member("sqlbuild._native", "LoaderConnectionMode", "SQLBUILD"), ), ], }; @@ -957,6 +953,31 @@ const R_SQL_LEXICAL_SYNTAX: RecordSpec = RecordSpec { ), ], }; +const R_PROJECT_GRAPH: RecordSpec = RecordSpec { + name: "ProjectGraph", + doc: "Static compiled project graph held natively; dict views are built only when read.", + fields: &[ + FieldSpec::required("project"), + FieldSpec { + name: "native", + default: FieldDefault::Required, + compared: false, + }, + ], +}; +const R_COMPILE_PIPELINE_RESULT: RecordSpec = RecordSpec { + name: "CompilePipelineResult", + doc: "Complete output from the compile-and-plan pipeline.", + fields: &[ + FieldSpec::required("project"), + FieldSpec::required("plan_output"), + FieldSpec::new("custom_materializations", FieldDefault::EmptyDict), + FieldSpec::new("python_node_names", FieldDefault::EmptyFrozenset), + FieldSpec::new("python_plan_entries", FieldDefault::EmptyTuple), + FieldSpec::new("compile_seconds", FieldDefault::None), + FieldSpec::new("planning_seconds", FieldDefault::None), + ], +}; /// The compiler, lint and config value records. pub(crate) const COMPILER_RECORDS: &[RecordSpec] = &[ @@ -1040,4 +1061,6 @@ pub(crate) const COMPILER_RECORDS: &[RecordSpec] = &[ R_PYTHON_IDENTITY_DEPENDENCY, R_PYTHON_NODE_IDENTITY, R_SQL_LEXICAL_SYNTAX, + R_PROJECT_GRAPH, + R_COMPILE_PIPELINE_RESULT, ]; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/dbt_command/auto_init.rs b/crates/sqlbuild-python/src/bindings/_helpers/dbt_command/auto_init.rs index b5b97e059d..8aa8a8520d 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/dbt_command/auto_init.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/dbt_command/auto_init.rs @@ -17,15 +17,12 @@ use crate::bindings::_helpers::dbt_command::support::{color_enabled, flag, path, const NATIVE_RECORDS_MODULE: &str = "sqlbuild._native"; const NATIVE_MODULE: &str = "sqlbuild._native"; -const DISCOVERY_CONSTANTS: &str = "sqlbuild.compiler.discovery.constants"; +const PROJECT_CONFIG_FILENAME: &str = "sqlbuild_project.toml"; const PROGRESS_REPORTER_MODULE: &str = "sqlbuild._native"; fn project_config_found(directory: &Bound<'_, PyAny>) -> PyResult { - let name: Bound<'_, PyAny> = imported( - directory.py(), - DISCOVERY_CONSTANTS, - "PROJECT_CONFIG_FILENAME", - )?; + let name: Bound<'_, PyAny> = + pyo3::types::PyString::new(directory.py(), PROJECT_CONFIG_FILENAME).into_any(); directory .call_method1("__truediv__", (name,))? .call_method0("exists")? diff --git a/crates/sqlbuild-python/src/bindings/_helpers/dbt_interop/manifest.rs b/crates/sqlbuild-python/src/bindings/_helpers/dbt_interop/manifest.rs index 44d7aa4732..da8afaf6f7 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/dbt_interop/manifest.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/dbt_interop/manifest.rs @@ -24,7 +24,7 @@ use crate::bindings::_helpers::compile_driver::python_calls::{ }; const NATIVE_RECORDS: &str = "sqlbuild._native"; -const COMPILE_ERRORS_MODULE: &str = "sqlbuild.compiler.compile.exceptions"; +const COMPILE_ERRORS_MODULE: &str = "sqlbuild._native"; const DBT_REF_KIND: &str = "dbt_ref"; const RELATION_OPTIONS: (&[&str], &[&str]) = (&["manifest", "ref_kind", "ref_name", "ref_package"], &[]); diff --git a/crates/sqlbuild-python/src/bindings/_helpers/dbt_interop/reference_resolver.rs b/crates/sqlbuild-python/src/bindings/_helpers/dbt_interop/reference_resolver.rs index e9ef7b9675..88847322fd 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/dbt_interop/reference_resolver.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/dbt_interop/reference_resolver.rs @@ -12,7 +12,7 @@ use crate::bindings::_helpers::compile_driver::python_calls::{ }; const NATIVE_MODULE: &str = "sqlbuild._native"; -const COMPILE_ERRORS_MODULE: &str = "sqlbuild.compiler.compile.exceptions"; +const COMPILE_ERRORS_MODULE: &str = "sqlbuild._native"; const COMPILE_MODELS_MODULE: &str = "sqlbuild.compiler.compile.models"; const LIFECYCLE_MODULE: &str = "sqlbuild.runtime.observability.classes.operation_lifecycle"; const VALIDATE_OPTIONS: (&[&str], &[&str]) = ( diff --git a/crates/sqlbuild-python/src/bindings/_helpers/discovery/configuration.rs b/crates/sqlbuild-python/src/bindings/_helpers/discovery/configuration.rs index 939191d8de..ab63518455 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/discovery/configuration.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/discovery/configuration.rs @@ -6,7 +6,7 @@ use pyo3::types::PyString; use crate::bindings::_helpers::compile_driver::python_calls::{call_with, imported, within}; const NATIVE_MODULE: &str = "sqlbuild._native"; -const DISCOVERY_MODELS_MODULE: &str = "sqlbuild.compiler.discovery.models"; +const DISCOVERY_MODELS_MODULE: &str = "sqlbuild._native"; const LIFECYCLE_MODULE: &str = "sqlbuild.runtime.observability.classes.operation_lifecycle"; /// `DiscoveredProjectInputs` with the loaded configuration, inside the project discovery lifecycle. diff --git a/crates/sqlbuild-python/src/bindings/_helpers/discovery/declaration_records.rs b/crates/sqlbuild-python/src/bindings/_helpers/discovery/declaration_records.rs index 75f730b91e..6bc8b68467 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/discovery/declaration_records.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/discovery/declaration_records.rs @@ -33,7 +33,7 @@ use crate::bindings::models::{NativeDiscoverySession, NativeProjectTree}; const NATIVE_RECORDS_MODULE: &str = "sqlbuild._native"; -const MODELS_MODULE: &str = "sqlbuild.compiler.discovery.models"; +const MODELS_MODULE: &str = "sqlbuild._native"; const SCOPE_TYPES_MODULE: &str = "sqlbuild._native"; const AUDITING_MODELS_MODULE: &str = "sqlbuild._native"; const AUDITING_TYPES_MODULE: &str = "sqlbuild.compiler.auditing.types"; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/discovery/description_inputs.rs b/crates/sqlbuild-python/src/bindings/_helpers/discovery/description_inputs.rs index 70c35c02c4..dc303e0764 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/discovery/description_inputs.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/discovery/description_inputs.rs @@ -7,7 +7,7 @@ use pyo3::{pyfunction, wrap_pyfunction}; use crate::bindings::_helpers::compile_driver::python_calls::{call_with, imported, within}; const NATIVE_MODULE: &str = "sqlbuild._native"; -const DISCOVERY_MODELS_MODULE: &str = "sqlbuild.compiler.discovery.models"; +const DISCOVERY_MODELS_MODULE: &str = "sqlbuild._native"; const CONTRACT_MODELS_MODULE: &str = "sqlbuild._native"; /// Models and model schemas, skipping files that cannot be read. diff --git a/crates/sqlbuild-python/src/bindings/_helpers/discovery/header_declarations.rs b/crates/sqlbuild-python/src/bindings/_helpers/discovery/header_declarations.rs index bceaa896b4..08024d074d 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/discovery/header_declarations.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/discovery/header_declarations.rs @@ -22,7 +22,7 @@ use crate::bindings::_helpers::contracts::sql_values::{ const NATIVE_RECORDS_MODULE: &str = "sqlbuild._native"; -const MODELS_MODULE: &str = "sqlbuild.compiler.discovery.models"; +const MODELS_MODULE: &str = "sqlbuild._native"; const DISCOVERY_ERRORS_MODULE: &str = "sqlbuild._native"; const IDENTITY_ERRORS_MODULE: &str = "sqlbuild._native"; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/discovery/hook_entries.rs b/crates/sqlbuild-python/src/bindings/_helpers/discovery/hook_entries.rs index 0b3efe5ed7..10b44186a4 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/discovery/hook_entries.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/discovery/hook_entries.rs @@ -6,7 +6,7 @@ use pyo3::{pyfunction, wrap_pyfunction}; use crate::bindings::_helpers::compile_driver::python_calls::imported; -const DISCOVERY_MODELS_MODULE: &str = "sqlbuild.compiler.discovery.models"; +const DISCOVERY_MODELS_MODULE: &str = "sqlbuild._native"; const RELATIVE_PATH_FIELD: &str = "relative_path"; const SQL_HOOK_FIELDS: [&str; 6] = [ "statement", diff --git a/crates/sqlbuild-python/src/bindings/_helpers/discovery/scope_snapshot.rs b/crates/sqlbuild-python/src/bindings/_helpers/discovery/scope_snapshot.rs index b02c05c0be..a9edcb12ad 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/discovery/scope_snapshot.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/discovery/scope_snapshot.rs @@ -26,7 +26,7 @@ use crate::bindings::_helpers::discovery::yaml_records::discover_source_files; const NATIVE_RECORDS_MODULE: &str = "sqlbuild._native"; -const MODELS_MODULE: &str = "sqlbuild.compiler.discovery.models"; +const MODELS_MODULE: &str = "sqlbuild._native"; const CONTRACT_MODELS_MODULE: &str = "sqlbuild._native"; const PROJECT_CONFIG_FILENAME: &str = "sqlbuild_project.toml"; const LOCAL_CONFIG_FILENAME: &str = "sqlbuild_local.toml"; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/discovery/sql_test_blocks.rs b/crates/sqlbuild-python/src/bindings/_helpers/discovery/sql_test_blocks.rs index 1075c3d825..2913333a00 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/discovery/sql_test_blocks.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/discovery/sql_test_blocks.rs @@ -13,7 +13,7 @@ use crate::bindings::_helpers::contracts::sql_values::{SqlValueClasses, raw_valu const NATIVE_RECORDS_MODULE: &str = "sqlbuild._native"; -const MODELS_MODULE: &str = "sqlbuild.compiler.discovery.models"; +const MODELS_MODULE: &str = "sqlbuild._native"; const ERRORS_MODULE: &str = "sqlbuild._native"; const TYPES_MODULE: &str = "sqlbuild.compiler.compile.types"; const SQL_VALUE_TYPES_MODULE: &str = "sqlbuild._native"; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/discovery/sql_test_records.rs b/crates/sqlbuild-python/src/bindings/_helpers/discovery/sql_test_records.rs index 827da0c072..624322f883 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/discovery/sql_test_records.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/discovery/sql_test_records.rs @@ -30,8 +30,9 @@ use crate::bindings::_helpers::discovery::sql_test_blocks::{ use crate::bindings::_helpers::sqltext::authored_values::map_to_python; use crate::bindings::models::NativeProjectTree; -const MODELS_MODULE: &str = "sqlbuild.compiler.discovery.models"; -const CONSTANTS_MODULE: &str = "sqlbuild.compiler.discovery.constants"; +const MODELS_MODULE: &str = "sqlbuild._native"; +const SQL_TESTS_OWNERSHIP_ROOT: &str = "tests/unit"; +const SQL_SCENARIOS_OWNERSHIP_ROOT: &str = "tests/scenarios"; const PATHLIB_MODULE: &str = "pathlib"; const SCENARIO_KIND: &str = "sql_scenario"; const DESCRIPTION_KEY: &str = "description"; @@ -60,8 +61,8 @@ fn path_class(py: Python<'_>) -> PyResult> { } /// `Path(constants.)`. -fn ownership_root<'py>(py: Python<'py>, name: &str) -> PyResult> { - path_class(py)?.call1((imported(py, CONSTANTS_MODULE, name)?,)) +fn ownership_root<'py>(py: Python<'py>, root: &str) -> PyResult> { + path_class(py)?.call1((root,)) } /// Discover the files natively in the pass's snapshot: each relative path text and outcome. @@ -137,7 +138,7 @@ fn test_file<'py>( ), ( "ownership_root", - ownership_root(py, "SQL_TESTS_OWNERSHIP_ROOT")?, + ownership_root(py, SQL_TESTS_OWNERSHIP_ROOT)?, ), ], ) @@ -190,7 +191,7 @@ fn scenario_file<'py>( ("name", file_path.getattr("stem")?), ( "ownership_root", - ownership_root(py, "SQL_SCENARIOS_OWNERSHIP_ROOT")?, + ownership_root(py, SQL_SCENARIOS_OWNERSHIP_ROOT)?, ), ], ) diff --git a/crates/sqlbuild-python/src/bindings/_helpers/discovery/yaml_records.rs b/crates/sqlbuild-python/src/bindings/_helpers/discovery/yaml_records.rs index 0c0cfa71f0..2f34461ba9 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/discovery/yaml_records.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/discovery/yaml_records.rs @@ -33,7 +33,7 @@ use crate::bindings::models::{NativeProjectTree, YamlDiscoveryRequest}; const NATIVE_RECORDS_MODULE: &str = "sqlbuild._native"; -const MODELS_MODULE: &str = "sqlbuild.compiler.discovery.models"; +const MODELS_MODULE: &str = "sqlbuild._native"; /// The most YAML bytes one native load reads before its files are built and released. const YAML_BATCH_BYTES: u64 = 4 * 1024 * 1024; const PATHLIB_MODULE: &str = "pathlib"; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/format_command/selection.rs b/crates/sqlbuild-python/src/bindings/_helpers/format_command/selection.rs index e7418db0c8..a46e43cc78 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/format_command/selection.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/format_command/selection.rs @@ -19,7 +19,7 @@ use crate::bindings::_helpers::compile_driver::python_calls::{call_with, importe const NATIVE_MODULE: &str = "sqlbuild._native"; const PLANNER_ERRORS_MODULE: &str = "sqlbuild._native"; -const DISCOVERY_MODELS_MODULE: &str = "sqlbuild.compiler.discovery.models"; +const DISCOVERY_MODELS_MODULE: &str = "sqlbuild._native"; const UNIT_TEST_SELECTORS_MODULE: &str = "sqlbuild._native"; const GRAPH_SELECTION_MODULE: &str = "sqlbuild._native"; const COMPILE_TYPES_MODULE: &str = "sqlbuild.compiler.compile.types"; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/frontier/stage_capture.rs b/crates/sqlbuild-python/src/bindings/_helpers/frontier/stage_capture.rs index 8b838282a8..65274bb9de 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/frontier/stage_capture.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/frontier/stage_capture.rs @@ -6,6 +6,7 @@ use pyo3::{pyfunction, wrap_pyfunction}; use sqlbuild_cache::stage_capture::constants::OMITTED_ATTRIBUTES; use sqlbuild_cache::stage_capture::main::mask_capture_noise::mask_capture_noise as masked; +use sqlbuild_cache::stage_capture::main::next_capture_number::next_capture_number; use crate::bindings::_helpers::compile_driver::python_calls::{ call_with, imported, keywords, within, @@ -16,7 +17,6 @@ const CAPTURE_DIR_ENV_VAR: &str = "SQLBUILD_COMPILER_STAGE_CAPTURE_DIR"; const CAPTURE_SUFFIX: &str = ".json"; const PLAN_CAPTURE_DIR_ENV_VAR: &str = "SQLBUILD_PLANNER_CAPTURE_DIR"; const PLAN_CAPTURE_NAME: &str = "plan_output"; -const FRONTIER_CONSTANTS_MODULE: &str = "sqlbuild.compiler.frontier.constants"; /// The capture document of `value`, or the shared nodes written before encoding failed. fn capture_text(py: Python<'_>, value: &Bound<'_, PyAny>) -> Result { @@ -32,6 +32,20 @@ fn capture_text(py: Python<'_>, value: &Bound<'_, PyAny>) -> Result) -> PyResult { + let directory: String = directory.str()?.extract()?; + let names: Vec = match std::fs::read_dir(&directory) { + Ok(entries) => entries + .filter_map(Result::ok) + .map(|entry| entry.file_name().to_string_lossy().into_owned()) + .collect(), + Err(error) if error.kind() == std::io::ErrorKind::NotFound => Vec::new(), + Err(error) => return Err(pyo3::exceptions::PyOSError::new_err(error.to_string())), + }; + Ok(next_capture_number(names.iter().map(String::as_str))) +} + /// Write one frontier object when SQLBUILD_COMPILER_STAGE_CAPTURE_DIR names a directory. pub(crate) fn write_stage_capture( stage: &Bound<'_, PyAny>, @@ -43,13 +57,7 @@ pub(crate) fn write_stage_capture( if !directory.is_truthy()? { return Ok(()); } - let number: u64 = imported(py, "builtins", "next")? - .call1((imported( - py, - FRONTIER_CONSTANTS_MODULE, - "STAGE_CAPTURE_SEQUENCE", - )?,))? - .extract()?; + let number: u64 = capture_number(&directory)?; let stage_name: String = stage.getattr("value")?.str()?.extract()?; let path: Bound<'_, PyAny> = imported(py, "pathlib", "Path")? .call1((directory,))? @@ -68,13 +76,7 @@ pub(crate) fn write_plan_capture(plan_output: &Bound<'_, PyAny>) -> PyResult<()> if !directory.is_truthy()? { return Ok(()); } - let number: u64 = imported(py, "builtins", "next")? - .call1((imported( - py, - FRONTIER_CONSTANTS_MODULE, - "STAGE_CAPTURE_SEQUENCE", - )?,))? - .extract()?; + let number: u64 = capture_number(&directory)?; let path: Bound<'_, PyAny> = imported(py, "pathlib", "Path")? .call1((directory,))? .call_method1( diff --git a/crates/sqlbuild-python/src/bindings/_helpers/graph/execution_gates.rs b/crates/sqlbuild-python/src/bindings/_helpers/graph/execution_gates.rs index c02ad458ba..5352ea008f 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/graph/execution_gates.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/graph/execution_gates.rs @@ -11,7 +11,7 @@ use crate::bindings::_helpers::graph::keys::KeyCache; const GRAPH_MODELS_MODULE: &str = "sqlbuild._native"; const GRAPH_TYPES_MODULE: &str = "sqlbuild._native"; -const DISCOVERY_MODELS_MODULE: &str = "sqlbuild.compiler.discovery.models"; +const DISCOVERY_MODELS_MODULE: &str = "sqlbuild._native"; const COMPILE_TYPES_MODULE: &str = "sqlbuild.compiler.compile.types"; const COMPILE_MODELS_MODULE: &str = "sqlbuild.compiler.compile.models"; const PATHLIB_MODULE: &str = "pathlib"; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/graph/views.rs b/crates/sqlbuild-python/src/bindings/_helpers/graph/views.rs index 6cd147b959..3ea882c339 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/graph/views.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/graph/views.rs @@ -91,7 +91,7 @@ fn graph_views<'py>(py: Python<'py>, graph: &ProjectGraph) -> PyResult( +pub(crate) fn lineage_graph_views<'py>( py: Python<'py>, graph: PyRef<'py, NativeProjectGraph>, ) -> PyResult> { diff --git a/crates/sqlbuild-python/src/bindings/_helpers/manifest/rows.rs b/crates/sqlbuild-python/src/bindings/_helpers/manifest/rows.rs index 5d7f5d9752..d5c16fb94f 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/manifest/rows.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/manifest/rows.rs @@ -14,7 +14,7 @@ use sqlbuild_analysis::python_values::models::PythonValue; use crate::bindings::_helpers::python_values::values::python_value; -const DISCOVERY_MODELS_MODULE: &str = "sqlbuild.compiler.discovery.models"; +const DISCOVERY_MODELS_MODULE: &str = "sqlbuild._native"; const DECIMAL_KIND: &str = "decimal"; const MACRO_KIND: &str = "macro"; const DEFAULT_MATERIALIZATION: &str = "view"; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/model_config/config_errors.rs b/crates/sqlbuild-python/src/bindings/_helpers/model_config/config_errors.rs index 48f6bfb0f2..681cf918ef 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/model_config/config_errors.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/model_config/config_errors.rs @@ -9,7 +9,7 @@ use crate::bindings::_helpers::compile_driver::python_calls::imported; use crate::bindings::_helpers::model_config::config_values::config_value_type_error; use crate::bindings::_helpers::model_config::errors::NativeConfigError; -const COMPILE_ERRORS_MODULE: &str = "sqlbuild.compiler.compile.exceptions"; +const COMPILE_ERRORS_MODULE: &str = "sqlbuild._native"; const DISCOVERY_ERRORS_MODULE: &str = "sqlbuild._native"; const IDENTITY_ERRORS_MODULE: &str = "sqlbuild._native"; const EXPECTED_STRING: &str = "a string"; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/model_config/template_data.rs b/crates/sqlbuild-python/src/bindings/_helpers/model_config/template_data.rs index dcd165dfa1..b78e53cfda 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/model_config/template_data.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/model_config/template_data.rs @@ -20,7 +20,7 @@ use crate::bindings::_helpers::model_config::config_templates::{ }; use crate::bindings::_helpers::model_config::unicode_environment::unicode_environment; -const COMPILE_ERRORS_MODULE: &str = "sqlbuild.compiler.compile.exceptions"; +const COMPILE_ERRORS_MODULE: &str = "sqlbuild._native"; const UUID_MODULE: &str = "uuid"; const ENVIRONMENT_READ: &str = "env"; const RUN_ID_KEY: &str = "run.id"; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/model_config/unicode_environment.rs b/crates/sqlbuild-python/src/bindings/_helpers/model_config/unicode_environment.rs index 5a2970e1dd..64069fdec1 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/model_config/unicode_environment.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/model_config/unicode_environment.rs @@ -8,7 +8,7 @@ use pyo3::{pyclass, pymethods}; use crate::bindings::_helpers::compile_driver::python_calls::imported; -const COMPILE_ERRORS_MODULE: &str = "sqlbuild.compiler.compile.exceptions"; +const COMPILE_ERRORS_MODULE: &str = "sqlbuild._native"; /// `os.environ`, raising a clear error for a value holding undecodable bytes. #[pyclass(module = "sqlbuild._native", frozen, mapping)] diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_changes/renames.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_changes/renames.rs index 729f2e9f83..b6ec6ecca0 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_changes/renames.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_changes/renames.rs @@ -17,7 +17,7 @@ use crate::bindings::_helpers::compile_driver::python_calls::{ use crate::bindings::_helpers::planner_identity::hashing::model_query_hash; const NATIVE_MODULE: &str = "sqlbuild._native"; -const REFERENCE_TYPES_MODULE: &str = "sqlbuild.compiler.references.types"; +const REFERENCE_TYPES_MODULE: &str = "sqlbuild._native"; const GENERIC_DIALECT: &str = "generic"; const FINAL_NEWLINE: &str = "\n"; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_entries/values.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_entries/values.rs index 4f3dbdc918..c2ee234c33 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_entries/values.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_entries/values.rs @@ -12,10 +12,10 @@ pub(super) const PLANNER_TYPES: &str = "sqlbuild._native"; pub(super) const PLANNER_EXCEPTIONS: &str = "sqlbuild._native"; pub(super) const MICROBATCH_CONCURRENCY_MESSAGE: &str = "batch_concurrency > 1 requires concurrent microbatches; selected model(s): "; -pub(super) const COMPILE_EXCEPTIONS: &str = "sqlbuild.compiler.compile.exceptions"; +pub(super) const COMPILE_EXCEPTIONS: &str = "sqlbuild._native"; pub(super) const COMPILE_TYPES: &str = "sqlbuild.compiler.compile.types"; pub(super) const COMPILE_MODELS: &str = "sqlbuild.compiler.compile.models"; -pub(super) const DISCOVERY_MODELS: &str = "sqlbuild.compiler.discovery.models"; +pub(super) const DISCOVERY_MODELS: &str = "sqlbuild._native"; /// `sqlbuild._native.name(**pairs)`. pub(super) fn native<'py>( diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_resolution/cursor_ranges.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_resolution/cursor_ranges.rs index 8b2bcb6e40..9f973de211 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_resolution/cursor_ranges.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_resolution/cursor_ranges.rs @@ -25,7 +25,7 @@ use std::cmp::Ordering; const MODELS: &str = "sqlbuild._native"; const CURSOR_TYPES: &str = "sqlbuild.cursor_algebra.types"; const CURSOR_MODELS: &str = "sqlbuild._native"; -const COMPILE_EXCEPTIONS: &str = "sqlbuild.compiler.compile.exceptions"; +const COMPILE_EXCEPTIONS: &str = "sqlbuild._native"; const INCREMENTAL: &str = "incremental"; const MICROBATCH: &str = "microbatch"; const ROLLING_WINDOW: &str = "rolling_window"; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_resolution/orchestration.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_resolution/orchestration.rs index ee0a98d2fe..9e0b106a42 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_resolution/orchestration.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_resolution/orchestration.rs @@ -9,7 +9,7 @@ use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyModule, PyModuleMethods, PyRes use pyo3::types::{PyBool, PyDict, PyDictMethods, PyList, PyString, PyTuple}; use pyo3::{IntoPyObject, PyErr, pyfunction, wrap_pyfunction}; -const HOOK_MODULE: &str = "sqlbuild.compiler.discovery.models"; +const HOOK_MODULE: &str = "sqlbuild._native"; const ERROR_MODULE: &str = "sqlbuild._native"; struct Resolution<'py> { diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_strategy/values.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_strategy/values.rs index 01d145e27a..14815320f1 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_strategy/values.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_strategy/values.rs @@ -16,7 +16,7 @@ use sqlbuild_analysis::planner_strategy::models::{ use crate::bindings::_helpers::compile_driver::python_calls::{call_with, imported}; pub(crate) const PLANNER_EXCEPTIONS_MODULE: &str = "sqlbuild._native"; -pub(crate) const COMPILE_EXCEPTIONS_MODULE: &str = "sqlbuild.compiler.compile.exceptions"; +pub(crate) const COMPILE_EXCEPTIONS_MODULE: &str = "sqlbuild._native"; const YML_SOURCE: &str = "yml"; const FULL_BACKFILL: &str = "full"; const FORWARD_ONLY_BACKFILL: &str = "forward"; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_warehouse/constants.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_warehouse/constants.rs index f4394e7f9b..3ff81e8ff3 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_warehouse/constants.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_warehouse/constants.rs @@ -42,8 +42,7 @@ pub(crate) const RENDER_SOURCE_READ: (&str, &str) = pub(crate) const YAML_ENTRY_LINE: (&str, &str) = ("sqlbuild._native", "yaml_entry_line"); pub(crate) const RUN_SCOPE: (&str, &str) = ("sqlbuild.runtime.observability.main.run_scope", "run_scope"); -pub(crate) const SQL_HOOK_ENTRY: (&str, &str) = - ("sqlbuild.compiler.discovery.models", "SqlHookEntry"); +pub(crate) const SQL_HOOK_ENTRY: (&str, &str) = ("sqlbuild._native", "SqlHookEntry"); pub(crate) const CURSOR_ALGEBRA_ERROR: (&str, &str) = ("sqlbuild.cursor_algebra.exceptions", "CursorAlgebraError"); pub(crate) const DATE_VALUE: (&str, &str) = ("sqlbuild._native", "DateValue"); diff --git a/crates/sqlbuild-python/src/bindings/_helpers/project_assembly/native_resources.rs b/crates/sqlbuild-python/src/bindings/_helpers/project_assembly/native_resources.rs index 4253a2d8c2..a4cb860f93 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/project_assembly/native_resources.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/project_assembly/native_resources.rs @@ -25,7 +25,7 @@ use crate::bindings::_helpers::project_assembly::models::{ }; use crate::bindings::types::CompilerDetach; -const COMPILE_ERRORS_MODULE: &str = "sqlbuild.compiler.compile.exceptions"; +const COMPILE_ERRORS_MODULE: &str = "sqlbuild._native"; const OS_MODULE: &str = "os"; const PROJECT_ASSEMBLY_CONTEXT: &str = "native project assembly"; const GENERIC_DIALECT: &str = "generic"; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/project_assembly/syntax_validation.rs b/crates/sqlbuild-python/src/bindings/_helpers/project_assembly/syntax_validation.rs index b6b7f7fe44..28fb19a8fc 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/project_assembly/syntax_validation.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/project_assembly/syntax_validation.rs @@ -12,10 +12,10 @@ use sqlbuild_analysis::assembly::project::models::{SyntaxCheck, SyntaxMode}; use crate::bindings::_helpers::project_assembly::resources::syntax_error_message; -const DISCOVERY_MODELS_MODULE: &str = "sqlbuild.compiler.discovery.models"; +const DISCOVERY_MODELS_MODULE: &str = "sqlbuild._native"; const SQL_HOOK_ENTRY: &str = "SqlHookEntry"; const PYTHON_HOOK_ENTRY: &str = "PythonHookEntry"; -const COMPILE_ERRORS_MODULE: &str = "sqlbuild.compiler.compile.exceptions"; +const COMPILE_ERRORS_MODULE: &str = "sqlbuild._native"; const COMPILE_INPUT_ERROR: &str = "CompileInputError"; const GENERIC_DIALECT: &str = "generic"; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/record_behaviours/constants.rs b/crates/sqlbuild-python/src/bindings/_helpers/record_behaviours/constants.rs index 46e169df58..c3b8a68354 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/record_behaviours/constants.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/record_behaviours/constants.rs @@ -9,11 +9,19 @@ use crate::bindings::_helpers::record_behaviours::cursors::{ finish_input_relation, finish_maximum_evidence, input_relation_runtime_owned, snapshot_watermarks_available, }; +use crate::bindings::_helpers::record_behaviours::discovery::{ + constant_logical_type, constant_scalar_type, graph_all_keys, graph_downstream_deps, + graph_lineage_views, graph_path_index, graph_tag_index, graph_upstream_deps, +}; use crate::bindings::_helpers::record_behaviours::durations::{ duration_add_to, duration_fixed_seconds, duration_has_calendar_component, duration_is_single_unit_in, duration_is_zero, duration_parse, duration_subtract_from, duration_total_months, duration_units, finish_aligned_interval, }; +use crate::bindings::_helpers::record_behaviours::error_inits::{ + macro_argument_error_init, macro_argument_error_located, macro_argument_error_shifted, + rules_config_error_init, sql_test_reference_error_init, +}; use crate::bindings::_helpers::record_behaviours::freshness::{ finish_freshness_record, freshness_record_identity, }; @@ -34,6 +42,10 @@ use crate::bindings::_helpers::record_behaviours::planner::{ plan_python_source_entries, retention_decreases, }; use crate::bindings::_helpers::record_behaviours::python_nodes::graph_selected_sql_refs; +use crate::bindings::_helpers::record_behaviours::references::{ + reference_example_call, reference_fixture_cte_prefix, reference_function_name, + reference_placeholder_call, +}; use crate::bindings::_helpers::record_behaviours::scopes::{ completeness_complete, target_query_unknown, }; @@ -163,7 +175,23 @@ pub(crate) const BEHAVIOUR_PROPERTIES: &[(&str, &str, ClassProperty)] = &[ "python_source_entries", plan_python_source_entries, ), + ("SqlReferenceKind", "function_name", reference_function_name), + ( + "SqlReferenceKind", + "fixture_cte_prefix", + reference_fixture_cte_prefix, + ), + ("ConstantDeclaration", "logical_type", constant_logical_type), + ("ConstantDeclaration", "scalar_type", constant_scalar_type), + ("ProjectGraph", "upstream_deps", graph_upstream_deps), + ("ProjectGraph", "downstream_deps", graph_downstream_deps), + ("ProjectGraph", "tag_index", graph_tag_index), + ("ProjectGraph", "path_index", graph_path_index), + ("ProjectGraph", "all_keys", graph_all_keys), ]; +/// Computed attributes of compiler shapes that a record computes once and keeps. +pub(crate) const BEHAVIOUR_CACHED_PROPERTIES: &[(&str, &str, ClassProperty)] = + &[("ProjectGraph", "lineage_views", graph_lineage_views)]; /// Methods of compiler shapes. pub(crate) const BEHAVIOUR_METHODS: &[(&str, &str, ClassMethod)] = &[ ( @@ -190,6 +218,29 @@ pub(crate) const BEHAVIOUR_METHODS: &[(&str, &str, ClassMethod)] = &[ materialization_is_table_backed, ), ("CursorOverrides", "typed_as", overrides_typed_as), + ("SqlReferenceKind", "example_call", reference_example_call), + ( + "SqlReferenceKind", + "placeholder_call", + reference_placeholder_call, + ), + ("MacroArgumentError", "__init__", macro_argument_error_init), + ( + "MacroArgumentError", + "shifted", + macro_argument_error_shifted, + ), + ( + "MacroArgumentError", + "located", + macro_argument_error_located, + ), + ( + "SqlTestReferenceError", + "__init__", + sql_test_reference_error_init, + ), + ("RulesConfigError", "__init__", rules_config_error_init), ("Duration", "subtract_from", duration_subtract_from), ]; /// Compiler records ordered by their compared fields. diff --git a/crates/sqlbuild-python/src/bindings/_helpers/record_behaviours/discovery.rs b/crates/sqlbuild-python/src/bindings/_helpers/record_behaviours/discovery.rs new file mode 100644 index 0000000000..b2bdc75cf3 --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/record_behaviours/discovery.rs @@ -0,0 +1,73 @@ +//! Computed attributes of the natively hosted discovery and pipeline records. + +use pyo3::IntoPyObjectExt; +use pyo3::prelude::{Bound, Py, PyAny, PyAnyMethods, PyResult}; + +use crate::bindings::_helpers::graph::views::lineage_graph_views; + +/// SQL scalar type labels by logical value kind. +const SCALAR_TYPE_LABELS: [(&str, &str); 6] = [ + ("string", "VARCHAR"), + ("integer", "INTEGER"), + ("boolean", "BOOLEAN"), + ("float", "FLOAT"), + ("decimal", "DECIMAL"), + ("null", "NULL"), +]; + +/// `ConstantDeclaration.logical_type`: the normalized logical type. +pub(crate) fn constant_logical_type(declaration: &Bound<'_, PyAny>) -> PyResult> { + Ok(declaration + .getattr("value")? + .getattr("logical_type")? + .unbind()) +} + +/// `ConstantDeclaration.scalar_type`: the SQL scalar type label, or the upper-cased logical type. +pub(crate) fn constant_scalar_type(declaration: &Bound<'_, PyAny>) -> PyResult> { + let value: Bound<'_, PyAny> = declaration.getattr("value")?; + let kind: String = value.getattr("kind")?.str()?.extract()?; + if let Some((_, label)) = SCALAR_TYPE_LABELS.iter().find(|(name, _)| *name == kind) { + return label.into_py_any(declaration.py()); + } + let display: String = value + .getattr("logical_type")? + .getattr("display_name")? + .extract()?; + display.to_uppercase().into_py_any(declaration.py()) +} + +/// `ProjectGraph.lineage_views`: lineage edges and selector indexes, converted once. +pub(crate) fn graph_lineage_views(graph: &Bound<'_, PyAny>) -> PyResult> { + let native = graph.getattr("native")?; + Ok(lineage_graph_views(graph.py(), native.extract()?)?.unbind()) +} + +fn view(graph: &Bound<'_, PyAny>, name: &str) -> PyResult> { + Ok(graph.getattr("lineage_views")?.getattr(name)?.unbind()) +} + +/// `ProjectGraph.upstream_deps`. +pub(crate) fn graph_upstream_deps(graph: &Bound<'_, PyAny>) -> PyResult> { + view(graph, "upstream_deps") +} + +/// `ProjectGraph.downstream_deps`. +pub(crate) fn graph_downstream_deps(graph: &Bound<'_, PyAny>) -> PyResult> { + view(graph, "downstream_deps") +} + +/// `ProjectGraph.tag_index`. +pub(crate) fn graph_tag_index(graph: &Bound<'_, PyAny>) -> PyResult> { + view(graph, "tag_index") +} + +/// `ProjectGraph.path_index`. +pub(crate) fn graph_path_index(graph: &Bound<'_, PyAny>) -> PyResult> { + view(graph, "path_index") +} + +/// `ProjectGraph.all_keys`. +pub(crate) fn graph_all_keys(graph: &Bound<'_, PyAny>) -> PyResult> { + view(graph, "all_keys") +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/record_behaviours/error_inits.rs b/crates/sqlbuild-python/src/bindings/_helpers/record_behaviours/error_inits.rs new file mode 100644 index 0000000000..46314aded6 --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/record_behaviours/error_inits.rs @@ -0,0 +1,174 @@ +//! Constructors and derived copies of the natively hosted expected-error classes. + +use pyo3::prelude::{Bound, Py, PyAny, PyAnyMethods, PyResult, Python}; +use pyo3::types::{PyDict, PyDictMethods, PyString, PyTuple}; + +use crate::bindings::_helpers::record_behaviours::call_arguments::call_argument; + +const HELP_FIELD: &str = "help"; +const LOCATION_FIELD: &str = "location"; +/// `MacroArgumentError` keyword fields, in constructor order. +const MACRO_ARGUMENT_FIELDS: [&str; 7] = [ + "detail", + "help", + "macro_name", + "file_path", + "offset", + "relative_position", + "location", +]; + +/// Set the expected-error attributes every native error carries. +fn initialized( + error: &Bound<'_, PyAny>, + message: &Bound<'_, PyAny>, + help: &Bound<'_, PyAny>, +) -> PyResult<()> { + let py: Python<'_> = error.py(); + error.setattr("args", PyTuple::new(py, [message])?)?; + error.setattr("message", message)?; + if let Ok(code) = error.get_type().getattr("code") { + error.setattr("code", code)?; + } + error.setattr("help", help) +} + +/// A keyword argument, or `None` when absent. +fn keyword<'py>( + py: Python<'py>, + kwargs: Option<&Bound<'py, PyDict>>, + name: &str, +) -> PyResult> { + let value: Option> = match kwargs { + Some(kwargs) => kwargs.get_item(name)?, + None => None, + }; + Ok(value.unwrap_or_else(|| py.None().into_bound(py))) +} + +/// `MacroArgumentError(detail=..., help=..., macro_name=..., file_path=..., offset=..., relative_position=..., location=None)`. +pub(crate) fn macro_argument_error_init( + error: &Bound<'_, PyAny>, + _args: &Bound<'_, PyTuple>, + kwargs: Option<&Bound<'_, PyDict>>, +) -> PyResult> { + let py: Python<'_> = error.py(); + let field = |name: &str| keyword(py, kwargs, name); + let location: Bound<'_, PyAny> = field("location")?; + let file_path: String = field("file_path")?.str()?.extract()?; + let macro_name: String = field("macro_name")?.str()?.extract()?; + let detail: String = field("detail")?.str()?.extract()?; + let (line, column): (Bound<'_, PyAny>, Bound<'_, PyAny>) = if location.is_none() { + let position: Bound<'_, PyAny> = field("relative_position")?; + (position.get_item(0)?, position.get_item(1)?) + } else { + (location.getattr("line")?, location.getattr("column")?) + }; + let (scope, suffix): (String, String) = if location.is_none() { + ( + format!("in '{file_path}'"), + format!(" of the '@{macro_name}' arguments"), + ) + } else { + ( + format!("of '@{macro_name}' in '{file_path}'"), + String::new(), + ) + }; + let message: String = format!( + "Macro arguments {scope} {detail} at line {}, column {}{suffix}", + line.str()?, + column.str()? + ); + initialized(error, PyString::new(py, &message).as_any(), &field("help")?)?; + for name in MACRO_ARGUMENT_FIELDS { + if name != HELP_FIELD { + error.setattr(name, field(name)?)?; + } + } + Ok(py.None()) +} + +/// A copy of `error` with `changes` applied to its constructor fields. +fn macro_argument_copy<'py>( + error: &Bound<'py, PyAny>, + changes: &[(&str, Bound<'py, PyAny>)], +) -> PyResult> { + let py: Python<'py> = error.py(); + let keywords: Bound<'py, PyDict> = PyDict::new(py); + for name in MACRO_ARGUMENT_FIELDS { + if name == LOCATION_FIELD { + continue; + } + keywords.set_item(name, error.getattr(name)?)?; + } + let help: Bound<'py, PyAny> = error.getattr("help")?; + if !help.is_truthy()? { + keywords.set_item("help", "")?; + } + for (name, value) in changes { + keywords.set_item(*name, value)?; + } + error.get_type().call((), Some(&keywords)) +} + +/// `MacroArgumentError.shifted(by)`: the same error, its offset moved into the enclosing SQL. +pub(crate) fn macro_argument_error_shifted( + error: &Bound<'_, PyAny>, + args: &Bound<'_, PyTuple>, + kwargs: Option<&Bound<'_, PyDict>>, +) -> PyResult> { + let by: Bound<'_, PyAny> = call_argument((args, kwargs), "by", 0)?; + let offset: Bound<'_, PyAny> = error.getattr("offset")?.add(by)?; + Ok(macro_argument_copy(error, &[("offset", offset)])?.unbind()) +} + +/// `MacroArgumentError.located(location)`: the same error at its authored file location. +pub(crate) fn macro_argument_error_located( + error: &Bound<'_, PyAny>, + args: &Bound<'_, PyTuple>, + kwargs: Option<&Bound<'_, PyDict>>, +) -> PyResult> { + let location: Bound<'_, PyAny> = call_argument((args, kwargs), "location", 0)?; + Ok(macro_argument_copy(error, &[("location", location)])?.unbind()) +} + +/// `SqlTestReferenceError(message, *, location, help)`: the message prefixed with its location. +pub(crate) fn sql_test_reference_error_init( + error: &Bound<'_, PyAny>, + args: &Bound<'_, PyTuple>, + kwargs: Option<&Bound<'_, PyDict>>, +) -> PyResult> { + let py: Python<'_> = error.py(); + let message: Bound<'_, PyAny> = call_argument((args, kwargs), "message", 0)?; + let location: Bound<'_, PyAny> = keyword(py, kwargs, "location")?; + let path: Bound<'_, PyAny> = location.getattr("path")?.call_method0("as_posix")?; + let text: String = format!( + "{}:{}:{}: {}", + path.str()?, + location.getattr("line")?.str()?, + location.getattr("column")?.str()?, + message.str()? + ); + initialized( + error, + PyString::new(py, &text).as_any(), + &keyword(py, kwargs, "help")?, + )?; + error.setattr("location", location)?; + Ok(py.None()) +} + +/// `RulesConfigError(message, *, help)`. +pub(crate) fn rules_config_error_init( + error: &Bound<'_, PyAny>, + args: &Bound<'_, PyTuple>, + kwargs: Option<&Bound<'_, PyDict>>, +) -> PyResult> { + let py: Python<'_> = error.py(); + let message: Bound<'_, PyAny> = call_argument((args, kwargs), "message", 0)?; + error.setattr("args", PyTuple::new(py, [&message])?)?; + error.setattr("message", message)?; + error.setattr("help", keyword(py, kwargs, "help")?)?; + Ok(py.None()) +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/record_behaviours/mod.rs b/crates/sqlbuild-python/src/bindings/_helpers/record_behaviours/mod.rs index 8c4dadf5a8..482410a68c 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/record_behaviours/mod.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/record_behaviours/mod.rs @@ -3,11 +3,14 @@ pub(crate) mod call_arguments; pub(crate) mod config; pub(crate) mod constants; pub(crate) mod cursors; +pub(crate) mod discovery; pub(crate) mod durations; +pub(crate) mod error_inits; pub(crate) mod freshness; pub(crate) mod lexical; pub(crate) mod migrations; pub(crate) mod planner; pub(crate) mod python_nodes; +pub(crate) mod references; pub(crate) mod scopes; pub(crate) mod values; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/record_behaviours/references.rs b/crates/sqlbuild-python/src/bindings/_helpers/record_behaviours/references.rs new file mode 100644 index 0000000000..3fb813183d --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/record_behaviours/references.rs @@ -0,0 +1,59 @@ +//! Reference call spellings of the natively hosted `SqlReferenceKind`. + +use pyo3::IntoPyObjectExt; +use pyo3::prelude::{Bound, Py, PyAny, PyAnyMethods, PyResult}; +use pyo3::types::{PyDict, PyDictMethods, PyTuple, PyTupleMethods}; + +fn function_name(kind: &Bound<'_, PyAny>) -> PyResult { + let value: String = kind.str()?.extract()?; + Ok(format!("__{value}")) +} + +/// `SqlReferenceKind.function_name`: the reference function, such as `__ref`. +pub(crate) fn reference_function_name(kind: &Bound<'_, PyAny>) -> PyResult> { + function_name(kind)?.into_py_any(kind.py()) +} + +/// `SqlReferenceKind.fixture_cte_prefix`: the prefix of fixture CTEs for this kind. +pub(crate) fn reference_fixture_cte_prefix(kind: &Bound<'_, PyAny>) -> PyResult> { + format!("{}__", function_name(kind)?).into_py_any(kind.py()) +} + +/// `SqlReferenceKind.example_call(*args, quote="'")`: a call with quoted arguments. +pub(crate) fn reference_example_call( + kind: &Bound<'_, PyAny>, + args: &Bound<'_, PyTuple>, + kwargs: Option<&Bound<'_, PyDict>>, +) -> PyResult> { + let quote: String = match kwargs { + Some(kwargs) => match kwargs.get_item("quote")? { + Some(quote) => quote.extract()?, + None => "'".to_owned(), + }, + None => "'".to_owned(), + }; + let doubled: String = format!("{quote}{quote}"); + let mut quoted: Vec = Vec::with_capacity(args.len()); + for argument in args.iter() { + let text: String = argument.extract()?; + quoted.push(format!("{quote}{}{quote}", text.replace("e, &doubled))); + } + format!("{}({})", function_name(kind)?, quoted.join(", ")).into_py_any(kind.py()) +} + +/// `SqlReferenceKind.placeholder_call(placeholder="")`. +pub(crate) fn reference_placeholder_call( + kind: &Bound<'_, PyAny>, + args: &Bound<'_, PyTuple>, + kwargs: Option<&Bound<'_, PyDict>>, +) -> PyResult> { + let placeholder: String = match kwargs + .map(|kwargs| kwargs.get_item("placeholder")) + .transpose()? + { + Some(Some(value)) => value.extract()?, + _ if !args.is_empty() => args.get_item(0)?.extract()?, + _ => String::new(), + }; + format!("{}({placeholder})", function_name(kind)?).into_py_any(kind.py()) +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/records/constants.rs b/crates/sqlbuild-python/src/bindings/_helpers/records/constants.rs index a85bee6a21..02e4ffb586 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/records/constants.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/records/constants.rs @@ -8,7 +8,8 @@ use crate::bindings::_helpers::contract_records::constants::CONTRACT_RECORDS; use crate::bindings::_helpers::freshness::observation::normalize_observation; use crate::bindings::_helpers::planner_records::constants::PLANNER_RECORDS; use crate::bindings::_helpers::record_behaviours::constants::{ - BEHAVIOUR_FINISHERS, BEHAVIOUR_METHODS, BEHAVIOUR_ORDERED, BEHAVIOUR_PROPERTIES, + BEHAVIOUR_CACHED_PROPERTIES, BEHAVIOUR_FINISHERS, BEHAVIOUR_METHODS, BEHAVIOUR_ORDERED, + BEHAVIOUR_PROPERTIES, }; use crate::bindings::_helpers::records::models::{ ClassValue, EnumSpec, ErrorSpec, FieldDefault, FieldSpec, RecordSpec, @@ -165,6 +166,9 @@ pub(crate) const RECORD_FINISHERS: &[&[(&str, RecordFinish)]] = &[ ]; /// `(class, attribute, getter)` computed attributes set on native classes after registration. pub(crate) const CLASS_PROPERTIES: &[&[(&str, &str, ClassProperty)]] = &[BEHAVIOUR_PROPERTIES]; +/// `(class, attribute, getter)` computed attributes a record computes once and keeps. +pub(crate) const CACHED_PROPERTIES: &[&[(&str, &str, ClassProperty)]] = + &[BEHAVIOUR_CACHED_PROPERTIES]; /// `(class, attribute, method)` methods set on native classes after registration. pub(crate) const CLASS_METHODS: &[&[(&str, &str, ClassMethod)]] = &[BEHAVIOUR_METHODS]; /// Records ordered by their compared fields, as `order=True` dataclasses are. diff --git a/crates/sqlbuild-python/src/bindings/_helpers/records/fields.rs b/crates/sqlbuild-python/src/bindings/_helpers/records/fields.rs index 72e86e794e..5ce038f7a2 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/records/fields.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/records/fields.rs @@ -1,7 +1,7 @@ //! Construction, comparison and display of native record field values. use pyo3::IntoPyObjectExt; -use pyo3::exceptions::{PyAttributeError, PyTypeError}; +use pyo3::exceptions::{PyAttributeError, PyRuntimeError, PyTypeError}; use pyo3::prelude::{Bound, Py, PyAny, PyAnyMethods, PyErr, PyResult, Python}; use pyo3::pyclass::CompareOp; use pyo3::types::{ @@ -178,3 +178,38 @@ pub(crate) fn ordered_comparison( .rich_compare(compared_values(other)?, op)? .into_py_any(py) } + +/// The cached value of `name` on `record`, if computed before. +pub(crate) fn cached_value( + record: &Bound<'_, NativeRecord>, + name: &str, +) -> PyResult>> { + let py: Python<'_> = record.py(); + let cached = record + .get() + .cached + .lock() + .map_err(|_| PyRuntimeError::new_err("native record cache is poisoned"))?; + Ok(cached + .iter() + .find(|(cached_name, _)| *cached_name == name) + .map(|(_, value)| value.clone_ref(py))) +} + +/// Keep `value` as the cached value of `name` on `record`. +pub(crate) fn store_cached( + record: &Bound<'_, NativeRecord>, + name: &'static str, + value: &Py, +) -> PyResult<()> { + let py: Python<'_> = record.py(); + let mut cached = record + .get() + .cached + .lock() + .map_err(|_| PyRuntimeError::new_err("native record cache is poisoned"))?; + if !cached.iter().any(|(cached_name, _)| *cached_name == name) { + cached.push((name, value.clone_ref(py))); + } + Ok(()) +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/records/models.rs b/crates/sqlbuild-python/src/bindings/_helpers/records/models.rs index 309b81f8bd..e394ec6148 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/records/models.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/records/models.rs @@ -1,5 +1,7 @@ //! Native frozen value records and the field layouts they are built from. +use std::sync::Mutex; + use pyo3::exceptions::PyAttributeError; use pyo3::prelude::{Bound, Py, PyAny, PyAnyMethods, PyResult, Python}; use pyo3::pyclass::CompareOp; @@ -7,7 +9,8 @@ use pyo3::types::{PyDict, PyTuple, PyType}; use pyo3::{IntoPyObjectExt, pyclass, pymethods}; use crate::bindings::_helpers::records::fields::{ - build_values, compared_values, frozen_error, ordered_comparison, record_spec, repr_text, + build_values, cached_value, compared_values, frozen_error, ordered_comparison, record_spec, + repr_text, store_cached, }; use crate::bindings::_helpers::records::types::{ClassMethod, ClassProperty, RecordFactory}; @@ -89,6 +92,8 @@ pub(crate) struct ErrorSpec { #[pyclass(module = "sqlbuild._native", subclass, frozen)] pub(crate) struct NativeRecord { pub(crate) values: Vec>, + /// Values of the record's cached computed attributes, by attribute name. + pub(crate) cached: Mutex)>>, } #[pymethods] @@ -104,6 +109,7 @@ impl NativeRecord { let spec: &RecordSpec = record_spec(cls)?; Ok(Self { values: build_values(spec, args, kwargs)?, + cached: Mutex::new(Vec::new()), }) } @@ -156,6 +162,8 @@ impl NativeRecord { pub(crate) struct NativeProperty { pub(crate) property: ClassProperty, pub(crate) name: &'static str, + /// Whether a record computes the attribute once and keeps it. + pub(crate) cached: bool, } #[pymethods] @@ -168,7 +176,17 @@ impl NativeProperty { if instance.is_none() { return Ok(slf.clone().into_any().unbind()); } - (slf.get().property)(instance) + let descriptor: &Self = slf.get(); + if !descriptor.cached || !instance.is_instance_of::() { + return (descriptor.property)(instance); + } + let record: &Bound<'_, NativeRecord> = instance.cast::()?; + if let Some(value) = cached_value(record, descriptor.name)? { + return Ok(value); + } + let value: Py = (descriptor.property)(instance)?; + store_cached(record, descriptor.name, &value)?; + Ok(value) } fn __set__(&self, instance: &Bound<'_, PyAny>, _value: &Bound<'_, PyAny>) -> PyResult<()> { diff --git a/crates/sqlbuild-python/src/bindings/_helpers/records/registration.rs b/crates/sqlbuild-python/src/bindings/_helpers/records/registration.rs index 49cfdbaf7e..c550070001 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/records/registration.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/records/registration.rs @@ -5,9 +5,10 @@ use pyo3::prelude::{Bound, Py, PyAny, PyAnyMethods, PyModule, PyModuleMethods, P use pyo3::types::{PyDict, PyDictMethods, PyString, PyTuple}; use crate::bindings::_helpers::records::constants::{ - BUILTIN_PARENT_PREFIX, BUILTINS_MODULE, CLASS_METHODS, CLASS_PROPERTIES, DATACLASS_FIELD_KIND, - DATACLASS_FIELDS_ATTRIBUTE, DATACLASSES_MODULE, ENUM_MODULE, ENUMS, ERROR_SPEC_ATTRIBUTE, - ERRORS, EXCEPTION_PARENT, NATIVE_MODULE, RECORD_SPEC_ATTRIBUTE, STR_ENUM, VALUE_ERROR_PARENT, + BUILTIN_PARENT_PREFIX, BUILTINS_MODULE, CACHED_PROPERTIES, CLASS_METHODS, CLASS_PROPERTIES, + DATACLASS_FIELD_KIND, DATACLASS_FIELDS_ATTRIBUTE, DATACLASSES_MODULE, ENUM_MODULE, ENUMS, + ERROR_SPEC_ATTRIBUTE, ERRORS, EXCEPTION_PARENT, NATIVE_MODULE, RECORD_SPEC_ATTRIBUTE, STR_ENUM, + VALUE_ERROR_PARENT, }; use crate::bindings::_helpers::records::error_classes::{ BoundErrorInitializer, ErrorInitializer, class_value, @@ -59,6 +60,18 @@ pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { NativeProperty { property: *property, name, + cached: false, + }, + )?; + module.getattr(*class)?.setattr(*name, descriptor)?; + } + for (class, name, property) in CACHED_PROPERTIES.iter().flat_map(|group| group.iter()) { + let descriptor: Py = Py::new( + py, + NativeProperty { + property: *property, + name, + cached: true, }, )?; module.getattr(*class)?.setattr(*name, descriptor)?; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/refactor/offline_compile.rs b/crates/sqlbuild-python/src/bindings/_helpers/refactor/offline_compile.rs index 9a940ea320..7d8128bb56 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/refactor/offline_compile.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/refactor/offline_compile.rs @@ -14,9 +14,9 @@ const NATIVE_RECORDS_MODULE: &str = "sqlbuild._native"; const NATIVE_MODULE: &str = "sqlbuild._native"; const COMPILE_MODELS: &str = "sqlbuild.compiler.compile.models"; const COMPILE_TYPES: &str = "sqlbuild.compiler.compile.types"; -const COMPILE_ERRORS: &str = "sqlbuild.compiler.compile.exceptions"; +const COMPILE_ERRORS: &str = "sqlbuild._native"; const PLANNER_ERRORS: &str = "sqlbuild._native"; -const RULES_ERRORS: &str = "sqlbuild.rule_engine.exceptions"; +const RULES_ERRORS: &str = "sqlbuild._native"; const SPEC_ERRORS: &str = "sqlbuild._native"; const ENTRY_MODELS: &str = "sqlbuild._native"; const COMMAND_TYPES: &str = "sqlbuild._native"; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/render/authored_sql.rs b/crates/sqlbuild-python/src/bindings/_helpers/render/authored_sql.rs index fcba3cb7c2..12d7560771 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/render/authored_sql.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/render/authored_sql.rs @@ -23,7 +23,7 @@ const NATIVE_RECORDS_MODULE: &str = "sqlbuild._native"; const NATIVE_MODULE: &str = "sqlbuild._native"; const COMPILE_MODELS_MODULE: &str = "sqlbuild.compiler.compile.models"; -const COMPILE_ERRORS_MODULE: &str = "sqlbuild.compiler.compile.exceptions"; +const COMPILE_ERRORS_MODULE: &str = "sqlbuild._native"; const SCOPE_MODELS_MODULE: &str = "sqlbuild._native"; const FUNCTOOLS_MODULE: &str = "functools"; const DECLARATION_FIELDS: [&str; 4] = [ diff --git a/crates/sqlbuild-python/src/bindings/_helpers/render/declaration_indexes.rs b/crates/sqlbuild-python/src/bindings/_helpers/render/declaration_indexes.rs index 62cadc33e2..2c74115fff 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/render/declaration_indexes.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/render/declaration_indexes.rs @@ -8,7 +8,7 @@ use pyo3::prelude::{ use pyo3::types::{PyDict, PyDictMethods, PyTuple}; use pyo3::{pyfunction, wrap_pyfunction}; -const COMPILE_ERRORS_MODULE: &str = "sqlbuild.compiler.compile.exceptions"; +const COMPILE_ERRORS_MODULE: &str = "sqlbuild._native"; const COMPILE_INPUT_ERROR: &str = "CompileInputError"; const DATACLASSES_MODULE: &str = "dataclasses"; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/render/declaration_values.rs b/crates/sqlbuild-python/src/bindings/_helpers/render/declaration_values.rs index c5d56ad028..696d223fa0 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/render/declaration_values.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/render/declaration_values.rs @@ -12,11 +12,11 @@ use crate::bindings::_helpers::scopes::scope_lookup::build_scope_lookup; const NATIVE_RECORDS_MODULE: &str = "sqlbuild._native"; const COMPILE_MODELS_MODULE: &str = "sqlbuild.compiler.compile.models"; -const COMPILE_ERRORS_MODULE: &str = "sqlbuild.compiler.compile.exceptions"; +const COMPILE_ERRORS_MODULE: &str = "sqlbuild._native"; const SCOPE_MODELS_MODULE: &str = "sqlbuild._native"; const SCOPE_TYPES_MODULE: &str = "sqlbuild._native"; const NATIVE_MODULE: &str = "sqlbuild._native"; -const DISCOVERY_MODELS_MODULE: &str = "sqlbuild.compiler.discovery.models"; +const DISCOVERY_MODELS_MODULE: &str = "sqlbuild._native"; const SCOPE_ERRORS_MODULE: &str = "sqlbuild._native"; const NO_DECLARATION_KIND: u8 = 255; /// Declaration kinds by their native code: enum 0, constant 1, macro 2. diff --git a/crates/sqlbuild-python/src/bindings/_helpers/render/macro_calls.rs b/crates/sqlbuild-python/src/bindings/_helpers/render/macro_calls.rs index 277833513d..208e75a836 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/render/macro_calls.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/render/macro_calls.rs @@ -16,7 +16,7 @@ use crate::bindings::_helpers::render::macro_call_rows::{SiteRow, SpanRow, site_ use crate::bindings::_helpers::render::macro_expansion::raised; use crate::bindings::types::CompilerDetach; -const COMPILE_EXCEPTIONS_MODULE: &str = "sqlbuild.compiler.compile.exceptions"; +const COMPILE_EXCEPTIONS_MODULE: &str = "sqlbuild._native"; const DECLARATION_CALLS: [&str; 2] = ["enum", "const"]; /// `(call start or None, call name or None, message)` of the error Python's scan raises. diff --git a/crates/sqlbuild-python/src/bindings/_helpers/render/macro_expansion_result.rs b/crates/sqlbuild-python/src/bindings/_helpers/render/macro_expansion_result.rs index a69975fb03..1bf37a92d5 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/render/macro_expansion_result.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/render/macro_expansion_result.rs @@ -20,7 +20,7 @@ const NATIVE_RECORDS_MODULE: &str = "sqlbuild._native"; const NATIVE_MODULE: &str = "sqlbuild._native"; const COMPILE_MODELS_MODULE: &str = "sqlbuild.compiler.compile.models"; -const COMPILE_EXCEPTIONS_MODULE: &str = "sqlbuild.compiler.compile.exceptions"; +const COMPILE_EXCEPTIONS_MODULE: &str = "sqlbuild._native"; const FRONTIER_EXCEPTIONS_MODULE: &str = "sqlbuild._native"; const SCOPES_MODELS_MODULE: &str = "sqlbuild._native"; const SCOPES_TYPES_MODULE: &str = "sqlbuild._native"; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/rules/custom_evaluation.rs b/crates/sqlbuild-python/src/bindings/_helpers/rules/custom_evaluation.rs index b436ff7265..991da1aefa 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/rules/custom_evaluation.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/rules/custom_evaluation.rs @@ -28,7 +28,7 @@ const NATIVE_RECORDS_MODULE: &str = "sqlbuild._native"; const NATIVE_MODULE: &str = "sqlbuild._native"; const RULES_CONSTANTS_MODULE: &str = "sqlbuild.rule_engine.constants"; -const RULES_ERRORS_MODULE: &str = "sqlbuild.rule_engine.exceptions"; +const RULES_ERRORS_MODULE: &str = "sqlbuild._native"; const RULE_CONTEXT_MODULE: &str = "sqlbuild.rule_engine.classes.rule_context"; const FACT_REPLAY_MODULE: &str = "sqlbuild.rule_engine._helpers.engine.fact_replay"; const SCOPE_MODELS_MODULE: &str = "sqlbuild._native"; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/rules/custom_hosts.rs b/crates/sqlbuild-python/src/bindings/_helpers/rules/custom_hosts.rs index 93fe2b427e..0ac110b053 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/rules/custom_hosts.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/rules/custom_hosts.rs @@ -32,7 +32,7 @@ use crate::bindings::_helpers::rules::custom_host_observation::{ const NATIVE_MODULE: &str = "sqlbuild._native"; const RULES_CONSTANTS_MODULE: &str = "sqlbuild.rule_engine.constants"; -const RULES_ERRORS_MODULE: &str = "sqlbuild.rule_engine.exceptions"; +const RULES_ERRORS_MODULE: &str = "sqlbuild._native"; const COMPILE_MODELS_MODULE: &str = "sqlbuild.compiler.compile.models"; const HOST_INPUTS: [&str; 3] = ["target", "rules-cache", "host-inputs"]; /// The model fields hosts decode; every other model field is dropped from their payload. diff --git a/crates/sqlbuild-python/src/bindings/_helpers/rules/custom_rule_cache.rs b/crates/sqlbuild-python/src/bindings/_helpers/rules/custom_rule_cache.rs index 211128b211..d935f4bc93 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/rules/custom_rule_cache.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/rules/custom_rule_cache.rs @@ -25,7 +25,7 @@ use crate::bindings::_helpers::boundary::keyword_request::Request; use crate::bindings::_helpers::compiled_project::values::plain_value; use crate::bindings::_helpers::rules::findings::{FindingClasses, invalid_finding}; -const RULES_ERRORS_MODULE: &str = "sqlbuild.rule_engine.exceptions"; +const RULES_ERRORS_MODULE: &str = "sqlbuild._native"; const RULES_ERROR: &str = "RulesError"; /// One custom rule's cache identity from its metadata, source fingerprint and settings. diff --git a/crates/sqlbuild-python/src/bindings/_helpers/rules/findings.rs b/crates/sqlbuild-python/src/bindings/_helpers/rules/findings.rs index 93fd0e3775..361917fa49 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/rules/findings.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/rules/findings.rs @@ -9,7 +9,7 @@ const MODELS_MODULE: &str = "sqlbuild.rule_engine.models"; const FINDING_CLASS: &str = "Finding"; const PATHLIB_MODULE: &str = "pathlib"; const PATH_CLASS: &str = "Path"; -const ERRORS_MODULE: &str = "sqlbuild.rule_engine.exceptions"; +const ERRORS_MODULE: &str = "sqlbuild._native"; const RULES_ERROR: &str = "RulesError"; const INVALID_FINDING: &str = "native rules engine returned an invalid finding"; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/rules/fix_session.rs b/crates/sqlbuild-python/src/bindings/_helpers/rules/fix_session.rs index 3e9a741619..261eb0bf41 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/rules/fix_session.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/rules/fix_session.rs @@ -21,7 +21,7 @@ use crate::bindings::_helpers::rules::fix_verdict::fix_verdict; const NATIVE: &str = "sqlbuild._native"; const MODELS: &str = "sqlbuild._native"; const ADAPTER: &str = "sqlbuild.adapter.contract.classes.base_adapter"; -const COMPILE_ERRORS: &str = "sqlbuild.compiler.compile.exceptions"; +const COMPILE_ERRORS: &str = "sqlbuild._native"; const LINT_ERRORS: &str = "sqlbuild._native"; const VERIFICATION_FAILURE: &str = "Compilation could not prove unchanged output columns, types, dependencies and lineage"; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/rules/lint_expansion.rs b/crates/sqlbuild-python/src/bindings/_helpers/rules/lint_expansion.rs index 3e835c3457..9fde98d0f3 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/rules/lint_expansion.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/rules/lint_expansion.rs @@ -14,7 +14,7 @@ use crate::bindings::_helpers::compile_driver::python_calls::{ const NATIVE_MODULE: &str = "sqlbuild._native"; const RESOLVE_ADAPTER_MODULE: &str = "sqlbuild.adapter.discovery.main.resolve_adapter"; const ADAPTER_EXCEPTIONS_MODULE: &str = "sqlbuild.adapter.contract.exceptions"; -const COMPILE_EXCEPTIONS_MODULE: &str = "sqlbuild.compiler.compile.exceptions"; +const COMPILE_EXCEPTIONS_MODULE: &str = "sqlbuild._native"; const DISCOVERY_EXCEPTIONS_MODULE: &str = "sqlbuild._native"; const LINT_EXCEPTIONS_MODULE: &str = "sqlbuild._native"; const COMPILE_FIRST_MESSAGE: &str = "compiler-integrated Rules check the SQL your project \ diff --git a/crates/sqlbuild-python/src/bindings/_helpers/rules/lint_preparation.rs b/crates/sqlbuild-python/src/bindings/_helpers/rules/lint_preparation.rs index 44023b8dd8..4adab35a94 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/rules/lint_preparation.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/rules/lint_preparation.rs @@ -16,7 +16,7 @@ const NATIVE_RECORDS_MODULE: &str = "sqlbuild._native"; const NATIVE: &str = "sqlbuild._native"; const MODELS: &str = "sqlbuild._native"; -const COMPILE_ERRORS: &str = "sqlbuild.compiler.compile.exceptions"; +const COMPILE_ERRORS: &str = "sqlbuild._native"; const LINT_ERRORS: &str = "sqlbuild._native"; /// Expand a single body and finish its native SQL preparation. diff --git a/crates/sqlbuild-python/src/bindings/_helpers/rules/rules_config.rs b/crates/sqlbuild-python/src/bindings/_helpers/rules/rules_config.rs index a428a3a4c3..196cf383c5 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/rules/rules_config.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/rules/rules_config.rs @@ -15,7 +15,7 @@ use sqlbuild_rules::models::RulesConfig; use crate::bindings::_helpers::boundary::panics::compiler_guard; const MODELS_MODULE: &str = "sqlbuild.rule_engine.models"; -const ERRORS_MODULE: &str = "sqlbuild.rule_engine.exceptions"; +const ERRORS_MODULE: &str = "sqlbuild._native"; const NATIVE_MODULE: &str = "sqlbuild._native"; /// The rules configuration models that stay Python for the custom-rule API. const PYTHON_CLASSES: &[&str] = &["RulesConfig", "LayoutConfig"]; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/rules/selection.rs b/crates/sqlbuild-python/src/bindings/_helpers/rules/selection.rs index 289a085c0e..1e98140149 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/rules/selection.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/rules/selection.rs @@ -11,7 +11,7 @@ use sqlbuild_rules::rules::main::selected_codes::selected_codes_json; use crate::bindings::_helpers::boundary::panics::compiler_guard; use crate::bindings::_helpers::rules::constants::RULES_NATIVE_API_VERSION; -const RULES_EXCEPTIONS_MODULE: &str = "sqlbuild.rule_engine.exceptions"; +const RULES_EXCEPTIONS_MODULE: &str = "sqlbuild._native"; const DATACLASSES_MODULE: &str = "dataclasses"; const JSON_MODULE: &str = "json"; const BUILTINS_MODULE: &str = "builtins"; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/scopes/relationship_names.rs b/crates/sqlbuild-python/src/bindings/_helpers/scopes/relationship_names.rs index dcfde4fa8c..9ac4b234e4 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/scopes/relationship_names.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/scopes/relationship_names.rs @@ -18,7 +18,7 @@ use crate::bindings::types::CompilerDetach; const NATIVE_RECORDS_MODULE: &str = "sqlbuild._native"; -const COMPILE_ERRORS_MODULE: &str = "sqlbuild.compiler.compile.exceptions"; +const COMPILE_ERRORS_MODULE: &str = "sqlbuild._native"; /// One scan's error, or its values. type Scanned = (Option, Vec); diff --git a/crates/sqlbuild-python/src/bindings/_helpers/semantic_checks/resource_sql.rs b/crates/sqlbuild-python/src/bindings/_helpers/semantic_checks/resource_sql.rs index d403c7155f..a58cd6d948 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/semantic_checks/resource_sql.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/semantic_checks/resource_sql.rs @@ -17,7 +17,7 @@ use crate::bindings::_helpers::boundary::panics::compiler_error; use crate::bindings::models::ProjectCatalog; use crate::bindings::types::CompilerDetach; -const COMPILE_ERRORS_MODULE: &str = "sqlbuild.compiler.compile.exceptions"; +const COMPILE_ERRORS_MODULE: &str = "sqlbuild._native"; const COMPILE_MODELS_MODULE: &str = "sqlbuild.compiler.compile.models"; const CONTRACT_MODELS_MODULE: &str = "sqlbuild._native"; const SQL_ANALYSIS_CONFIG_KEY: &str = "sql_analysis"; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/semantic_checks/stage_objects.rs b/crates/sqlbuild-python/src/bindings/_helpers/semantic_checks/stage_objects.rs index e9ba4f5b49..1f9aca6bf6 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/semantic_checks/stage_objects.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/semantic_checks/stage_objects.rs @@ -16,7 +16,7 @@ const BINDING_DIAGNOSTICS_MODULE: &str = "sqlbuild._native"; const NATIVE_MODULE: &str = "sqlbuild._native"; const NORMALIZE_TYPE_MODULE: &str = "sqlbuild.adapter.type_system.main.normalize_type"; const DATACLASSES_MODULE: &str = "dataclasses"; -const REFERENCE_TYPES_MODULE: &str = "sqlbuild.compiler.references.types"; +const REFERENCE_TYPES_MODULE: &str = "sqlbuild._native"; /// Imported once per stage run. pub(crate) struct StageClasses<'py> { diff --git a/crates/sqlbuild-python/src/bindings/_helpers/sql_test_glue/authored_ctes.rs b/crates/sqlbuild-python/src/bindings/_helpers/sql_test_glue/authored_ctes.rs index d0adb4dbcd..f1b7ed40f4 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/sql_test_glue/authored_ctes.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/sql_test_glue/authored_ctes.rs @@ -14,7 +14,7 @@ use sqlbuild_sqltext::sql_scan::models::LexicalSyntax; use crate::bindings::_helpers::sqltext::lexical_syntax::LexicalSyntaxInput; -const COMPILE_ERRORS_MODULE: &str = "sqlbuild.compiler.compile.exceptions"; +const COMPILE_ERRORS_MODULE: &str = "sqlbuild._native"; const COMPILE_INPUT_ERROR: &str = "CompileInputError"; const MODEL_MODE: &str = "model"; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/sql_test_glue/plan_errors.rs b/crates/sqlbuild-python/src/bindings/_helpers/sql_test_glue/plan_errors.rs index 54513e5664..043e11832f 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/sql_test_glue/plan_errors.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/sql_test_glue/plan_errors.rs @@ -79,16 +79,11 @@ pub(super) fn planning_error( "CompileInputError" }; return Ok(PyErr::from_value( - imported(py, "sqlbuild.compiler.compile.exceptions", class)? - .call((message,), Some(&kwargs))?, + imported(py, "sqlbuild._native", class)?.call((message,), Some(&kwargs))?, )); } let (module, class, message) = if let Some(message) = message.strip_prefix("compile_input:") { - ( - "sqlbuild.compiler.compile.exceptions", - "CompileInputError", - message.to_owned(), - ) + ("sqlbuild._native", "CompileInputError", message.to_owned()) } else if let Some(message) = message.strip_prefix("planner_input:") { ("sqlbuild._native", "PlannerInputError", message.to_owned()) } else { diff --git a/crates/sqlbuild-python/src/bindings/_helpers/sqltext/authored_values.rs b/crates/sqlbuild-python/src/bindings/_helpers/sqltext/authored_values.rs index f8a5141d1d..77659c1912 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/sqltext/authored_values.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/sqltext/authored_values.rs @@ -7,7 +7,7 @@ use sqlbuild_sqltext::compiler::main::header_word_value::header_word_value; use sqlbuild_sqltext::compiler::models::HeaderWord; const SQL_VALUES_MODULE: &str = "sqlbuild._native"; -const DISCOVERY_MODELS_MODULE: &str = "sqlbuild.compiler.discovery.models"; +const DISCOVERY_MODELS_MODULE: &str = "sqlbuild._native"; fn word_to_python(py: Python<'_>, word: String) -> PyResult> { match header_word_value(&word) { diff --git a/crates/sqlbuild-python/src/bindings/_helpers/sqltext/code_scanner.rs b/crates/sqlbuild-python/src/bindings/_helpers/sqltext/code_scanner.rs index cf4ea638ed..594654873d 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/sqltext/code_scanner.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/sqltext/code_scanner.rs @@ -12,7 +12,7 @@ use sqlbuild_sqltext::sql_scan::models::Unclosed; use crate::bindings::_helpers::sqltext::lexical_syntax::LexicalSyntaxInput; -const COMPILE_ERRORS_MODULE: &str = "sqlbuild.compiler.compile.exceptions"; +const COMPILE_ERRORS_MODULE: &str = "sqlbuild._native"; const COMPILE_INPUT_ERROR: &str = "CompileInputError"; /// One SQL text scanned under an adapter's lexical rules, or the generic compiler rules. diff --git a/crates/sqlbuild-python/src/bindings/_helpers/sqltext/sql_references.rs b/crates/sqlbuild-python/src/bindings/_helpers/sqltext/sql_references.rs index 5a1046c80b..70520985c6 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/sqltext/sql_references.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/sqltext/sql_references.rs @@ -20,8 +20,8 @@ const NATIVE_RECORDS_MODULE: &str = "sqlbuild._native"; const COMPILE_MODELS_MODULE: &str = "sqlbuild.compiler.compile.models"; const COMPILE_TYPES_MODULE: &str = "sqlbuild.compiler.compile.types"; -const COMPILE_ERRORS_MODULE: &str = "sqlbuild.compiler.compile.exceptions"; -const REFERENCE_TYPES_MODULE: &str = "sqlbuild.compiler.references.types"; +const COMPILE_ERRORS_MODULE: &str = "sqlbuild._native"; +const REFERENCE_TYPES_MODULE: &str = "sqlbuild._native"; const COLLECTOR_MODULE: &str = "sqlbuild._native"; const REFERENCE_CALL_SYNTAX_CODE: &str = "P012"; diff --git a/python-boundary.toml b/python-boundary.toml index a92193fd2d..c6de58e414 100644 --- a/python-boundary.toml +++ b/python-boundary.toml @@ -2,7 +2,7 @@ # migration work that moves to Rust. `remaining_lines` must equal the current count of that work: # a PR that moves code lowers it, and adding movable Python fails the check. # Native failures must surface as errors; `handled_native_errors` lists the reviewed exceptions. -remaining_lines = 2319 +remaining_lines = 1586 [[stays]] path = "executor" diff --git a/scripts/compiler_differential/_helpers/corpus/emitted_codes.py b/scripts/compiler_differential/_helpers/corpus/emitted_codes.py index b02f3a7bbe..c762ffda85 100644 --- a/scripts/compiler_differential/_helpers/corpus/emitted_codes.py +++ b/scripts/compiler_differential/_helpers/corpus/emitted_codes.py @@ -39,9 +39,13 @@ FailureCase, ProjectRun, ) -from sqlbuild._native import DiscoveryError, ResourceIdentityError, ScopeDiagnosticCode +from sqlbuild._native import ( + CompileInputError, + DiscoveryError, + ResourceIdentityError, + ScopeDiagnosticCode, +) from sqlbuild.compiler.compile import constants as compile_constants -from sqlbuild.compiler.compile.exceptions import CompileInputError _SOURCE_DIRECTORY: str = "source" _CODE_KEYWORD: str = "code" diff --git a/scripts/compiler_differential/_helpers/coverage/discovery.py b/scripts/compiler_differential/_helpers/coverage/discovery.py index b4febeba93..da0e2810aa 100644 --- a/scripts/compiler_differential/_helpers/coverage/discovery.py +++ b/scripts/compiler_differential/_helpers/coverage/discovery.py @@ -35,12 +35,11 @@ STRATEGY_HEADER_KEYS, TAB, ) -from sqlbuild._native import STAGE_CAPTURE_OMITTED_ATTRIBUTES +from sqlbuild._native import STAGE_CAPTURE_OMITTED_ATTRIBUTES, DiscoveredProjectInputs from sqlbuild.adapter.discovery.constants import BUILTIN_ADAPTER_IMPORTS -from sqlbuild.compiler.discovery.models import DiscoveredProjectInputs _OMITTED_DISCOVERY_FIELDS: frozenset[str] = STAGE_CAPTURE_OMITTED_ATTRIBUTES.get( - f"{DiscoveredProjectInputs.__module__}:{DiscoveredProjectInputs.__qualname__}", frozenset() + "sqlbuild.compiler.discovery.models:DiscoveredProjectInputs", frozenset() ) _CROSS_FILE_MACRO_IMPORT: re.Pattern[str] = re.compile(r"^(?:from|import) macros\.", re.MULTILINE) diff --git a/src/sqlbuild/_native.pyi b/src/sqlbuild/_native.pyi index 9b97418121..d4affd3247 100644 --- a/src/sqlbuild/_native.pyi +++ b/src/sqlbuild/_native.pyi @@ -76,6 +76,7 @@ from sqlbuild.compiler.compile.types import ( CompiledResourceType, CompiledSeedView, CompiledSourceView, + ExternalSqlReferenceResolver, FunctionLanguage, RelationKeyRow, SourceInput, @@ -106,10 +107,6 @@ from sqlbuild.compiler.discovery.types import LoaderConnectionMode from sqlbuild.compiler.lineage.types import ( InferredNullability, ) -from sqlbuild.compiler.pipeline.models import ( - CompilePipelineResult, - ProjectGraph, -) from sqlbuild.compiler.python_nodes.types import PythonNodeKind from sqlbuild.compiler.references.models import ( ProjectRelation, @@ -172,10 +169,10 @@ from sqlbuild.integrations.dbt.models import ( ) from sqlbuild.integrations.dbt.types import DbtInvoker from sqlbuild.microbatches.models import MicrobatchInterval -from sqlbuild.presentation.types import TransientLineOwner from sqlbuild.provider.classes.container import ProviderContainer from sqlbuild.provider.classes.session import ProviderSession from sqlbuild.python_nodes.models import ( + AuditCase, ColumnLineageRef, RetryPolicy, SqlResourceRef, @@ -405,9 +402,16 @@ type NativeSqlTestPlanRow = tuple[ type SqlTestAssemblyFailureKind = Literal["input", "internal", "decimal_overflow"] +type ProjectProvider = Provider type SqlScalar = str | int | bool | float | Decimal | None type SqlValuePayload = SqlScalar | tuple[object, ...] +class TransientLineOwner(Protocol): + """A renderer that owns the transient line currently drawn in place on a terminal.""" + + def clear_transient_line(self) -> None: ... + def redraw_transient_line(self) -> None: ... + class DbtInvoker(Protocol): """Callable dbt process invocation contract.""" @@ -16783,3 +16787,694 @@ class PlanOutput(NativeRecord): ) -> None: ... @property def python_source_entries(self) -> dict[str, SourceEntry]: ... + +class CompileInputError(ValueError): + """Raised when discovered inputs cannot be attached into a compile view.""" + + code: str + message: str + help: str | None + def __init__( + self, message: str, *, code: str | None = None, help: str | None = None + ) -> None: ... + +class MacroArgumentError(CompileInputError): + """Raised when one macro call's arguments do not parse; `offset` is in the expanded SQL.""" + + detail: str + macro_name: str + file_path: Path + offset: int + relative_position: tuple[int, int] + location: SourceLocation | None + def __init__( + self, + *, + detail: str, + help: str, + macro_name: str, + file_path: Path, + offset: int, + relative_position: tuple[int, int], + location: SourceLocation | None = None, + ) -> None: ... + def shifted(self, by: int) -> MacroArgumentError: ... + def located(self, location: SourceLocation) -> MacroArgumentError: ... + +class SqlTestReferenceError(CompileInputError): + """Raised when a SQL-test CTE calls a reference the test query cannot resolve.""" + + location: SourceLocation + def __init__(self, message: str, *, location: SourceLocation, help: str) -> None: ... + +class RulesError(RuntimeError): + """Raised for invalid compiler rules or execution.""" + +class RulesConfigError(RulesError): + """Raised when rules configuration breaks a project policy; help shows the setting to change.""" + + message: str + help: str + def __init__(self, message: str, *, help: str) -> None: ... + +class RuleUsageError(RulesError): + """Raised when a rule or rule test uses the public API incorrectly.""" + +class NonHermeticRuleError(RulesError): + """Raised when a custom rule observes state the Rules cache cannot track.""" + +class RuleAssertionError(RulesError): + """Raised when a custom-rule harness case produces an unexpected result.""" + +class FactDigestError(RulesError): + """Raised when a recorded custom-rule fact cannot be recomputed or encoded deterministically.""" + +class OpaqueModuleStateError(RulesError): + """Raised when custom-rule module state holds a value whose mutations cannot be observed.""" + +class HostCancelledError(RulesError): + """Raised in a custom-rule host that stops because another host already failed.""" + +class SqlReferenceKind(StrEnum): + REF = "ref" + SEED = "seed" + SOURCE = "source" + DBT_REF = "dbt_ref" + UDF = "udf" + TABLE_FUNCTION = "table_fn" + @property + def function_name(self) -> str: ... + @property + def fixture_cte_prefix(self) -> str: ... + def example_call(self, *args: str, quote: Literal["'", '"'] = "'") -> str: ... + def placeholder_call(self, placeholder: str = "") -> str: ... + +class HardCodedRelationOwnerKind(StrEnum): + """The kind of Python code that names a project relation in SQL.""" + + NODE = "node" + HOOK = "hook" + LOADER = "loader" + +class LoaderConnectionMode(StrEnum): + """How a source loader owns the destination warehouse connection.""" + + SQLBUILD = "sqlbuild" + EXTERNAL = "external" + +class SqlHookEntry(NativeRecord): + """A model lifecycle hook that executes SQL.""" + + statement: str + name: str | None + relative_path: Path | None + definition_sql: str | None + kwargs: dict[str, object] | None + description: str | None + reads: tuple[SqlResourceRef, ...] + def __init__( + self, + statement: str, + name: str | None = ..., + relative_path: Path | None = ..., + definition_sql: str | None = ..., + kwargs: dict[str, object] | None = ..., + description: str | None = ..., + reads: tuple[SqlResourceRef, ...] = ..., + ) -> None: ... + +class PythonHookEntry(NativeRecord): + """A model lifecycle hook that invokes a discovered Python hook.""" + + name: str + kwargs: dict[str, object] + def __init__( + self, + name: str, + kwargs: dict[str, object], + ) -> None: ... + +class EnumDeclaration(NativeRecord): + """One validated public or model-local enum declaration.""" + + name: str + members: tuple[EnumMember, ...] + scalar_type: str + relative_path: Path + model_name: str | None + def __init__( + self, + name: str, + members: tuple[EnumMember, ...], + scalar_type: str, + relative_path: Path, + model_name: str | None = ..., + ) -> None: ... + +class ConstantDeclaration(NativeRecord): + """One validated public or model-local constant declaration.""" + + name: str + value: SqlValue + relative_path: Path + model_name: str | None + render_as: CollectionRendering | None + def __init__( + self, + name: str, + value: SqlValue, + relative_path: Path, + model_name: str | None = ..., + render_as: CollectionRendering | None = ..., + ) -> None: ... + @property + def logical_type(self) -> SqlLogicalType: ... + @property + def scalar_type(self) -> str: ... + +class DiscoveredSourceFile(NativeRecord): + """A discovered source declaration file and its raw contents.""" + + file_path: Path + relative_path: Path + contents: str + source_entries: tuple[SourceEntry, ...] + def __init__( + self, + file_path: Path, + relative_path: Path, + contents: str, + source_entries: tuple[SourceEntry, ...], + ) -> None: ... + +class DiscoveredSqlTestFile(NativeRecord): + """A discovered SQL-native test file and its raw contents.""" + + file_path: Path + relative_path: Path + contents: str + blocks: tuple[DiscoveredSqlTestBlock, ...] + ownership_root: Path + def __init__( + self, + file_path: Path, + relative_path: Path, + contents: str, + blocks: tuple[DiscoveredSqlTestBlock, ...], + ownership_root: Path = ..., + ) -> None: ... + +class SqlTestParameterDeclaration(NativeRecord): + """One typed parameter declared by a SQL-native test template.""" + + name: str + value_type: SqlValueKind + nullable: bool + def __init__( + self, + name: str, + value_type: SqlValueKind, + nullable: bool = ..., + ) -> None: ... + +class DiscoveredSqlScenarioFile(NativeRecord): + """A discovered SQL-native scenario file and its raw contents.""" + + file_path: Path + relative_path: Path + contents: str + header_values: dict[str, object] + sql_body: str + name: str + ownership_root: Path + sql_body_span: tuple[int, int] | None + def __init__( + self, + file_path: Path, + relative_path: Path, + contents: str, + header_values: dict[str, object], + sql_body: str, + name: str, + ownership_root: Path = ..., + sql_body_span: tuple[int, int] | None = ..., + ) -> None: ... + +class DiscoveredMacroFile(NativeRecord): + """A discovered project macro file and its raw contents.""" + + file_path: Path + relative_path: Path + contents: str + scope_kind: ScopeKind + ownership_root: Path | None + owning_path: Path | None + declaration_root: Path | None + def __init__( + self, + file_path: Path, + relative_path: Path, + contents: str, + scope_kind: ScopeKind = ..., + ownership_root: Path | None = ..., + owning_path: Path | None = ..., + declaration_root: Path | None = ..., + ) -> None: ... + +class DiscoveredMaterializationFile(NativeRecord): + """A discovered custom materialization Python file.""" + + file_path: Path + relative_path: Path + name: str + provider_usages: tuple[DiscoveredProviderUsage, ...] + def __init__( + self, + file_path: Path, + relative_path: Path, + name: str, + provider_usages: tuple[DiscoveredProviderUsage, ...] = ..., + ) -> None: ... + +class DiscoveredProviderUsage(NativeRecord): + """Framework-owned metadata for provider usage by a Python callable surface.""" + + provider_name: str + parameter_name: str + annotation_class_name: str | None + annotation_module: str | None + def __init__( + self, + provider_name: str, + parameter_name: str, + annotation_class_name: str | None = ..., + annotation_module: str | None = ..., + ) -> None: ... + +class DiscoveredLoaderFunction(NativeRecord): + """A discovered project source loader function.""" + + file_path: Path + relative_path: Path + name: str + function: Callable[..., object] + depends_on: tuple[Callable[..., object] | SqlResourceRef, ...] + destination: str | None + write_strategy: SourceWriteStrategy | None + cursor_column: str | None + unique_key: tuple[str, ...] + columns: tuple[SourceColumnEntry, ...] + contract: str | None + description: str | None + connection_mode: LoaderConnectionMode + provider_usages: tuple[DiscoveredProviderUsage, ...] + def __init__( + self, + file_path: Path, + relative_path: Path, + name: str, + function: Callable[..., object], + depends_on: tuple[Callable[..., object] | SqlResourceRef, ...] = ..., + destination: str | None = ..., + write_strategy: SourceWriteStrategy | None = ..., + cursor_column: str | None = ..., + unique_key: tuple[str, ...] = ..., + columns: tuple[SourceColumnEntry, ...] = ..., + contract: str | None = ..., + description: str | None = ..., + connection_mode: LoaderConnectionMode = ..., + provider_usages: tuple[DiscoveredProviderUsage, ...] = ..., + ) -> None: ... + +class DiscoveredTaskFunction(NativeRecord): + """A discovered project task function.""" + + file_path: Path + relative_path: Path + name: str + function: Callable[..., object] + depends_on: tuple[Callable[..., object] | SqlResourceRef, ...] + tags: tuple[str, ...] + group: str | None + description: str | None + meta: dict[str, object] | None + retry: RetryPolicy | None + provider_usages: tuple[DiscoveredProviderUsage, ...] + def __init__( + self, + file_path: Path, + relative_path: Path, + name: str, + function: Callable[..., object], + depends_on: tuple[Callable[..., object] | SqlResourceRef, ...] = ..., + tags: tuple[str, ...] = ..., + group: str | None = ..., + description: str | None = ..., + meta: dict[str, object] | None = ..., + retry: RetryPolicy | None = ..., + provider_usages: tuple[DiscoveredProviderUsage, ...] = ..., + ) -> None: ... + +class DiscoveredAssetFunction(NativeRecord): + """A discovered project asset function.""" + + file_path: Path + relative_path: Path + name: str + function: Callable[..., object] + depends_on: tuple[Callable[..., object] | SqlResourceRef, ...] + tags: tuple[str, ...] + group: str | None + description: str | None + meta: dict[str, object] | None + columns: tuple[SourceColumnEntry, ...] + column_lineage: dict[str, tuple[ColumnLineageRef, ...]] | None + retry: RetryPolicy | None + provider_usages: tuple[DiscoveredProviderUsage, ...] + def __init__( + self, + file_path: Path, + relative_path: Path, + name: str, + function: Callable[..., object], + depends_on: tuple[Callable[..., object] | SqlResourceRef, ...] = ..., + tags: tuple[str, ...] = ..., + group: str | None = ..., + description: str | None = ..., + meta: dict[str, object] | None = ..., + columns: tuple[SourceColumnEntry, ...] = ..., + column_lineage: dict[str, tuple[ColumnLineageRef, ...]] | None = ..., + retry: RetryPolicy | None = ..., + provider_usages: tuple[DiscoveredProviderUsage, ...] = ..., + ) -> None: ... + +class DiscoveredCheckFunction(NativeRecord): + """A discovered project check function.""" + + file_path: Path + relative_path: Path + name: str + function: Callable[..., object] + depends_on: tuple[Callable[..., object] | SqlResourceRef, ...] + severity: PythonCheckSeverity + tags: tuple[str, ...] + group: str | None + description: str | None + meta: dict[str, object] | None + provider_usages: tuple[DiscoveredProviderUsage, ...] + def __init__( + self, + file_path: Path, + relative_path: Path, + name: str, + function: Callable[..., object], + depends_on: tuple[Callable[..., object] | SqlResourceRef, ...], + severity: PythonCheckSeverity = ..., + tags: tuple[str, ...] = ..., + group: str | None = ..., + description: str | None = ..., + meta: dict[str, object] | None = ..., + provider_usages: tuple[DiscoveredProviderUsage, ...] = ..., + ) -> None: ... + +class DiscoveredHookFunction(NativeRecord): + """A discovered project model lifecycle hook function.""" + + file_path: Path + relative_path: Path + name: str + function: Callable[..., object] + description: str | None + reads: tuple[SqlResourceRef, ...] + provider_usages: tuple[DiscoveredProviderUsage, ...] + scope_kind: ScopeKind + ownership_root: Path | None + owning_path: Path | None + declaration_root: Path | None + def __init__( + self, + file_path: Path, + relative_path: Path, + name: str, + function: Callable[..., object], + description: str | None = ..., + reads: tuple[SqlResourceRef, ...] = ..., + provider_usages: tuple[DiscoveredProviderUsage, ...] = ..., + scope_kind: ScopeKind = ..., + ownership_root: Path | None = ..., + owning_path: Path | None = ..., + declaration_root: Path | None = ..., + ) -> None: ... + +class DiscoveredAuditFactory(NativeRecord): + """A discovered audit factory and its normalized cases.""" + + name: str + function: Callable[..., object] + file_path: Path + relative_path: Path + line: int + cases: tuple[AuditCase, ...] + def __init__( + self, + name: str, + function: Callable[..., object], + file_path: Path, + relative_path: Path, + line: int, + cases: tuple[AuditCase, ...] = ..., + ) -> None: ... + +class DiscoveredEventExporterDeclaration(NativeRecord): + """A discovered exporter declaration before provider binding.""" + + file_path: Path + relative_path: Path + name: str + function: Callable[..., object] + event_kinds: frozenset[str] + min_severity: EventExportSeverity + def __init__( + self, + file_path: Path, + relative_path: Path, + name: str, + function: Callable[..., object], + event_kinds: frozenset[str], + min_severity: EventExportSeverity, + ) -> None: ... + +class DiscoveredEventExporter(NativeRecord): + """A discovered canonical lifecycle event exporter function.""" + + file_path: Path + relative_path: Path + name: str + function: Callable[..., object] + event_kinds: frozenset[str] + min_severity: EventExportSeverity + provider_usages: tuple[DiscoveredProviderUsage, ...] + def __init__( + self, + file_path: Path, + relative_path: Path, + name: str, + function: Callable[..., object], + event_kinds: frozenset[str] = ..., + min_severity: EventExportSeverity = ..., + provider_usages: tuple[DiscoveredProviderUsage, ...] = ..., + ) -> None: ... + +class DiscoveredCommandOutputSinkDeclaration(NativeRecord): + """A command-output sink declaration before provider binding.""" + + file_path: Path + relative_path: Path + name: str + function: Callable[..., object] + streams: frozenset[CommandOutputStream] + def __init__( + self, + file_path: Path, + relative_path: Path, + name: str, + function: Callable[..., object], + streams: frozenset[CommandOutputStream], + ) -> None: ... + +class DiscoveredCommandOutputSink(NativeRecord): + """A discovered command-output sink function.""" + + file_path: Path + relative_path: Path + name: str + function: Callable[..., object] + streams: frozenset[CommandOutputStream] + provider_usages: tuple[DiscoveredProviderUsage, ...] + def __init__( + self, + file_path: Path, + relative_path: Path, + name: str, + function: Callable[..., object], + streams: frozenset[CommandOutputStream], + provider_usages: tuple[DiscoveredProviderUsage, ...] = ..., + ) -> None: ... + +class DiscoveredProvider(NativeRecord): + """A discovered project provider class and validated settings object.""" + + file_path: Path + relative_path: Path + name: str + provider_class: type[ProjectProvider] + settings: ProjectProvider + def __init__( + self, + file_path: Path, + relative_path: Path, + name: str, + provider_class: type[ProjectProvider], + settings: ProjectProvider, + ) -> None: ... + +class DiscoveredPythonNodeFunctions(NativeRecord): + """Discovered project Python DAG node functions grouped by kind.""" + + loaders: tuple[DiscoveredLoaderFunction, ...] + tasks: tuple[DiscoveredTaskFunction, ...] + assets: tuple[DiscoveredAssetFunction, ...] + checks: tuple[DiscoveredCheckFunction, ...] + audit_factories: tuple[DiscoveredAuditFactory, ...] + def __init__( + self, + loaders: tuple[DiscoveredLoaderFunction, ...] = ..., + tasks: tuple[DiscoveredTaskFunction, ...] = ..., + assets: tuple[DiscoveredAssetFunction, ...] = ..., + checks: tuple[DiscoveredCheckFunction, ...] = ..., + audit_factories: tuple[DiscoveredAuditFactory, ...] = ..., + ) -> None: ... + +class DiscoveredProjectInputs(NativeRecord): + """All raw project inputs discovered from disk before semantic resolution.""" + + project_config: ProjectConfig + local_config: LocalConfig + project_dir: Path | None + model_files: tuple[DiscoveredSqlModelFile, ...] + enum_files: tuple[DiscoveredEnumFile, ...] + constant_files: tuple[DiscoveredConstantFile, ...] + model_schema_files: tuple[DiscoveredModelSchemaFile, ...] + sql_function_files: tuple[DiscoveredSqlFunctionFile, ...] + sql_hook_files: tuple[DiscoveredSqlHookFile, ...] + python_function_files: tuple[DiscoveredPythonFunctionFile, ...] + schema_files: tuple[DiscoveredSchemaFile, ...] + source_files: tuple[DiscoveredSourceFile, ...] + seed_files: tuple[DiscoveredSeedFile, ...] + test_files: tuple[DiscoveredSqlTestFile, ...] + scenario_files: tuple[DiscoveredSqlScenarioFile, ...] + audit_files: tuple[DiscoveredAuditFile, ...] + macro_files: tuple[DiscoveredMacroFile, ...] + materialization_files: tuple[DiscoveredMaterializationFile, ...] + loader_functions: tuple[DiscoveredLoaderFunction, ...] + task_functions: tuple[DiscoveredTaskFunction, ...] + asset_functions: tuple[DiscoveredAssetFunction, ...] + check_functions: tuple[DiscoveredCheckFunction, ...] + audit_factories: tuple[DiscoveredAuditFactory, ...] + hook_functions: tuple[DiscoveredHookFunction, ...] + event_exporters: tuple[DiscoveredEventExporter, ...] + command_output_sinks: tuple[DiscoveredCommandOutputSink, ...] + providers: tuple[DiscoveredProvider, ...] + adapter_file: DiscoveredAdapterFile | None + native_session: NativeDiscoverySession | None + def __init__( + self, + project_config: ProjectConfig, + local_config: LocalConfig, + project_dir: Path | None = ..., + model_files: tuple[DiscoveredSqlModelFile, ...] = ..., + enum_files: tuple[DiscoveredEnumFile, ...] = ..., + constant_files: tuple[DiscoveredConstantFile, ...] = ..., + model_schema_files: tuple[DiscoveredModelSchemaFile, ...] = ..., + sql_function_files: tuple[DiscoveredSqlFunctionFile, ...] = ..., + sql_hook_files: tuple[DiscoveredSqlHookFile, ...] = ..., + python_function_files: tuple[DiscoveredPythonFunctionFile, ...] = ..., + schema_files: tuple[DiscoveredSchemaFile, ...] = ..., + source_files: tuple[DiscoveredSourceFile, ...] = ..., + seed_files: tuple[DiscoveredSeedFile, ...] = ..., + test_files: tuple[DiscoveredSqlTestFile, ...] = ..., + scenario_files: tuple[DiscoveredSqlScenarioFile, ...] = ..., + audit_files: tuple[DiscoveredAuditFile, ...] = ..., + macro_files: tuple[DiscoveredMacroFile, ...] = ..., + materialization_files: tuple[DiscoveredMaterializationFile, ...] = ..., + loader_functions: tuple[DiscoveredLoaderFunction, ...] = ..., + task_functions: tuple[DiscoveredTaskFunction, ...] = ..., + asset_functions: tuple[DiscoveredAssetFunction, ...] = ..., + check_functions: tuple[DiscoveredCheckFunction, ...] = ..., + audit_factories: tuple[DiscoveredAuditFactory, ...] = ..., + hook_functions: tuple[DiscoveredHookFunction, ...] = ..., + event_exporters: tuple[DiscoveredEventExporter, ...] = ..., + command_output_sinks: tuple[DiscoveredCommandOutputSink, ...] = ..., + providers: tuple[DiscoveredProvider, ...] = ..., + adapter_file: DiscoveredAdapterFile | None = ..., + native_session: NativeDiscoverySession | None = ..., + ) -> None: ... + +class DiscoveredRuntimeExtensions(NativeRecord): + """Command-startup providers, typed sinks, and lifecycle delivery settings.""" + + providers: tuple[DiscoveredProvider, ...] + event_exporters: tuple[DiscoveredEventExporter, ...] + command_output_sinks: tuple[DiscoveredCommandOutputSink, ...] + lifecycle_shutdown_timeout_seconds: int | None + def __init__( + self, + providers: tuple[DiscoveredProvider, ...] = ..., + event_exporters: tuple[DiscoveredEventExporter, ...] = ..., + command_output_sinks: tuple[DiscoveredCommandOutputSink, ...] = ..., + lifecycle_shutdown_timeout_seconds: int | None = ..., + ) -> None: ... + +class ProjectGraph(NativeRecord): + """Static compiled project graph held natively; dict views are built only when read.""" + + project: CompiledProjectView + native: _native.NativeProjectGraph + def __init__( + self, + project: CompiledProjectView, + native: _native.NativeProjectGraph, + ) -> None: ... + @property + def lineage_views(self) -> LineageGraphViews: ... + @property + def upstream_deps(self) -> dict[CompiledObjectKey, tuple[CompiledObjectKey, ...]]: ... + @property + def downstream_deps(self) -> dict[CompiledObjectKey, tuple[CompiledObjectKey, ...]]: ... + @property + def tag_index(self) -> dict[str, frozenset[CompiledObjectKey]]: ... + @property + def path_index(self) -> dict[CompiledObjectKey, str]: ... + @property + def all_keys(self) -> dict[str, CompiledObjectKey]: ... + +class CompilePipelineResult(NativeRecord): + """Complete output from the compile-and-plan pipeline.""" + + project: CompiledProjectView + plan_output: PlanOutput + custom_materializations: dict[str, Callable[..., Any]] + python_node_names: frozenset[str] + python_plan_entries: tuple[PythonPlanEntry, ...] + compile_seconds: float | None + planning_seconds: float | None + def __init__( + self, + project: CompiledProjectView, + plan_output: PlanOutput, + custom_materializations: dict[str, Callable[..., Any]] = ..., + python_node_names: frozenset[str] = ..., + python_plan_entries: tuple[PythonPlanEntry, ...] = ..., + compile_seconds: float | None = ..., + planning_seconds: float | None = ..., + ) -> None: ... diff --git a/src/sqlbuild/compiler/compile/_helpers/diagnostics/python_resources.py b/src/sqlbuild/compiler/compile/_helpers/diagnostics/python_resources.py index 121f5334a9..c550729a37 100644 --- a/src/sqlbuild/compiler/compile/_helpers/diagnostics/python_resources.py +++ b/src/sqlbuild/compiler/compile/_helpers/diagnostics/python_resources.py @@ -7,9 +7,13 @@ from functools import partial from pathlib import Path -from sqlbuild._native import SourceLocation, snippet_help +from sqlbuild._native import ( + DiscoveredProjectInputs, + DiscoveredProvider, + SourceLocation, + snippet_help, +) from sqlbuild.compiler.compile.types import PythonResourceRow, SourceInput -from sqlbuild.compiler.discovery.models import DiscoveredProjectInputs, DiscoveredProvider def python_resource_rows( diff --git a/src/sqlbuild/compiler/compile/_helpers/macro_bridge/call_store.py b/src/sqlbuild/compiler/compile/_helpers/macro_bridge/call_store.py index e62351fb10..297577b5ab 100644 --- a/src/sqlbuild/compiler/compile/_helpers/macro_bridge/call_store.py +++ b/src/sqlbuild/compiler/compile/_helpers/macro_bridge/call_store.py @@ -5,10 +5,10 @@ from pathlib import Path from sqlbuild._native import ( + DiscoveredProjectInputs, TargetConfig, build_compile_target_context, ) -from sqlbuild.compiler.discovery.models import DiscoveredProjectInputs from sqlbuild.compiler.macro_bridge.classes.macro_bridge import MacroBridge from sqlbuild.compiler.macro_bridge.main._claim_first_compile import claim_first_compile from sqlbuild.compiler.macro_bridge.main.active_macro_bridge import active_macro_bridge diff --git a/src/sqlbuild/compiler/compile/_helpers/render/user_macros.py b/src/sqlbuild/compiler/compile/_helpers/render/user_macros.py index 2c325615f7..29a17d9887 100644 --- a/src/sqlbuild/compiler/compile/_helpers/render/user_macros.py +++ b/src/sqlbuild/compiler/compile/_helpers/render/user_macros.py @@ -15,17 +15,18 @@ import sqlbuild._native as _native from sqlbuild._native import ( + CompileInputError, + ConstantDeclaration, DeclarationIdentity, DeclarationKind, DeclarationRecord, + DiscoveredMacroFile, + EnumDeclaration, SqlValue, SqlValueKind, validate_resource_identity, ) -from sqlbuild.compiler.compile.exceptions import ( - CompileInputError, - MacroDeclarationLookupError, -) +from sqlbuild.compiler.compile.exceptions import MacroDeclarationLookupError from sqlbuild.compiler.compile.models import ( DeclarationResolutionContext, LoadedMacro, @@ -34,11 +35,6 @@ StaticMacroFault, StaticMacroInventory, ) -from sqlbuild.compiler.discovery.models import ( - ConstantDeclaration, - DiscoveredMacroFile, - EnumDeclaration, -) _OBJECT_ENTRY_LENGTH: int = 2 _MACRO_MODULE_LOAD_LOCK: threading.RLock = threading.RLock() diff --git a/src/sqlbuild/compiler/compile/exceptions.py b/src/sqlbuild/compiler/compile/exceptions.py index 3d4caa356b..184084b274 100644 --- a/src/sqlbuild/compiler/compile/exceptions.py +++ b/src/sqlbuild/compiler/compile/exceptions.py @@ -2,97 +2,7 @@ from __future__ import annotations -from pathlib import Path - -from sqlbuild._native import SourceLocation - - -class CompileInputError(ValueError): - """Raised when discovered inputs cannot be attached into a compile view.""" - - code: str = "P001" - - def __init__( - self, - message: str, - *, - code: str | None = None, - help: str | None = None, - ) -> None: - super().__init__(message) - self.message = message - self.code = code if code is not None else self.code - self.help = help - - -class MacroArgumentError(CompileInputError): - """Raised when one macro call's arguments do not parse; `offset` is in the expanded SQL.""" - - def __init__( # noqa: PLR0913 - self, - *, - detail: str, - help: str, - macro_name: str, - file_path: Path, - offset: int, - relative_position: tuple[int, int], - location: SourceLocation | None = None, - ) -> None: - line, column = ( - (location.line, location.column) if location is not None else relative_position - ) - scope: str = ( - f"of '@{macro_name}' in '{file_path}'" if location is not None else f"in '{file_path}'" - ) - suffix: str = "" if location is not None else f" of the '@{macro_name}' arguments" - super().__init__( - f"Macro arguments {scope} {detail} at line {line}, column {column}{suffix}", - help=help, - ) - self.detail: str = detail - self.macro_name: str = macro_name - self.file_path: Path = file_path - self.offset: int = offset - self.relative_position: tuple[int, int] = relative_position - self.location: SourceLocation | None = location - - def shifted(self, by: int) -> MacroArgumentError: - """The same error, its offset moved by `by` into the enclosing SQL.""" - - return MacroArgumentError( - detail=self.detail, - help=self.help or "", - macro_name=self.macro_name, - file_path=self.file_path, - offset=self.offset + by, - relative_position=self.relative_position, - ) - - def located(self, location: SourceLocation) -> MacroArgumentError: - """The same error at its authored file location.""" - - return MacroArgumentError( - detail=self.detail, - help=self.help or "", - macro_name=self.macro_name, - file_path=self.file_path, - offset=self.offset, - relative_position=self.relative_position, - location=location, - ) - - -class SqlTestReferenceError(CompileInputError): - """Raised when a SQL-test CTE calls a reference the test query cannot resolve.""" - - code: str = "P013" - - def __init__(self, message: str, *, location: SourceLocation, help: str) -> None: - super().__init__( - f"{location.path.as_posix()}:{location.line}:{location.column}: {message}", help=help - ) - self.location: SourceLocation = location +from sqlbuild._native import CompileInputError class MacroDeclarationLookupError(CompileInputError, KeyError): diff --git a/src/sqlbuild/compiler/compile/models.py b/src/sqlbuild/compiler/compile/models.py index 46b9adeb17..0aef4bed50 100644 --- a/src/sqlbuild/compiler/compile/models.py +++ b/src/sqlbuild/compiler/compile/models.py @@ -19,8 +19,10 @@ CompiledModelSqlTestPayload, CompiledSqlExpansion, CompiledSqlTestResource, + CompileInputError, CompileModelConfig, CompileSqlScenarioCte, + ConstantDeclaration, DeclarationExpansionContext, DeclarationIdentity, DeclarationRecord, @@ -29,10 +31,17 @@ DiagnosticSeverity, DiscoveredAuditBlock, DiscoveredAuditFile, + DiscoveredHookFunction, + DiscoveredLoaderFunction, + DiscoveredMaterializationFile, DiscoveredSchemaFile, DiscoveredSqlHookFile, + DiscoveredSqlModelFile, + DiscoveredSqlScenarioFile, DiscoveredSqlTestBlock, + DiscoveredSqlTestFile, DynamicColumnContractProof, + EnumDeclaration, LiteralSqlRelation, MeasurementContract, MeasurementThresholds, @@ -45,6 +54,8 @@ SettingsConfig, SourceLocation, SqlLexicalSyntax, + SqlReferenceKind, + SqlTestParameterDeclaration, SqlValue, TargetConfig, UsageRecord, @@ -52,7 +63,6 @@ ) from sqlbuild.compiler.auditing.types import AuditEvaluationMode, AuditSeverity from sqlbuild.compiler.compile.constants import DEFAULT_SQL_TEST_MODE, MACRO_CONTEXT_PARAMETER_NAME -from sqlbuild.compiler.compile.exceptions import CompileInputError from sqlbuild.compiler.compile.types import ( AttachedAuditTargetKind, CompiledAuditView, @@ -61,23 +71,12 @@ CompiledResourceType, CompiledSeedView, CompiledSourceView, + ExternalSqlReferenceResolver, RelationKeyRow, SqlTestMode, TypedSqlValueRenderer, ) -from sqlbuild.compiler.discovery.models import ( - ConstantDeclaration, - DiscoveredHookFunction, - DiscoveredLoaderFunction, - DiscoveredMaterializationFile, - DiscoveredSqlModelFile, - DiscoveredSqlScenarioFile, - DiscoveredSqlTestFile, - EnumDeclaration, - SqlTestParameterDeclaration, -) from sqlbuild.compiler.lineage.types import InferredNullability -from sqlbuild.compiler.references.types import ExternalSqlReferenceResolver, SqlReferenceKind @dataclass(frozen=True) diff --git a/src/sqlbuild/compiler/compile/types.py b/src/sqlbuild/compiler/compile/types.py index d60ad95c12..711ccd3c20 100644 --- a/src/sqlbuild/compiler/compile/types.py +++ b/src/sqlbuild/compiler/compile/types.py @@ -18,12 +18,18 @@ CompiledSource, CompiledSqlExpansion, CompileModelConfig, + ConstantDeclaration, DiscoveredAuditBlock, DiscoveredAuditFile, + DiscoveredHookFunction, + DiscoveredLoaderFunction, + DiscoveredMaterializationFile, DiscoveredSchemaFile, DiscoveredSeedFile, + DiscoveredSourceFile, DiscoveredSqlHookFile, DynamicColumnContractProof, + EnumDeclaration, FunctionArgument, FunctionReturnColumn, LiteralSqlRelation, @@ -50,15 +56,6 @@ InferredColumn, LoadedMacro, ) - from sqlbuild.compiler.discovery.models import ( - ConstantDeclaration, - DiscoveredHookFunction, - DiscoveredLoaderFunction, - DiscoveredMaterializationFile, - DiscoveredSourceFile, - EnumDeclaration, - ) - from sqlbuild.compiler.references.types import ExternalSqlReferenceResolver type RelationKeyRow = tuple[ @@ -458,3 +455,38 @@ def name(self) -> str | None: ... def assembly_row( self, ) -> tuple[list[tuple[str, str, str | None]], tuple[str, str] | None]: ... + + +class ExternalSqlReferenceResolver(Protocol): + """Resolve first-class SQLBuild references backed by external metadata.""" + + def validate_model_names(self, *, known_model_names: set[str]) -> None: + """Validate SQLBuild model names against external integration resources.""" + + def extend_sql_test_model_names(self, *, known_model_names: set[str]) -> set[str]: + """Return extra model names valid for SQL-native test targets.""" + + def extend_sql_test_source_names(self, *, known_source_names: set[str]) -> set[str]: + """Return extra source names valid for SQL-native test mocks.""" + + def extend_sql_test_seed_names(self, *, known_seed_names: set[str]) -> set[str]: + """Return extra seed names valid for SQL-native test mocks.""" + + def validate_reference( + self, + *, + ref_kind: str, + ref_name: str, + ref_package: str | None, + owner_relative_sql_path: Path, + ) -> None: + """Validate one external SQL reference from a SQLBuild-owned file.""" + + def resolve_reference( + self, + *, + ref_kind: str, + ref_name: str, + ref_package: str | None, + ) -> str | None: + """Return the physical relation for an external reference, if resolvable.""" diff --git a/src/sqlbuild/compiler/discovery/_helpers/filesystem/command_output_sinks.py b/src/sqlbuild/compiler/discovery/_helpers/filesystem/command_output_sinks.py index b961b51995..73fb50bb7a 100644 --- a/src/sqlbuild/compiler/discovery/_helpers/filesystem/command_output_sinks.py +++ b/src/sqlbuild/compiler/discovery/_helpers/filesystem/command_output_sinks.py @@ -20,6 +20,12 @@ DiscoveredEventExporter, DiscoveredProvider, DiscoveredProviderUsage, + EventExporterDiscoveryError, +) +from sqlbuild.compiler.discovery._helpers.filesystem.core import ( + _load_sink_module, + _provider_by_name, + _provider_usages, ) from sqlbuild.runtime.event_exporting.main.get_command_output_sink_definition import ( get_command_output_sink_definition, diff --git a/src/sqlbuild/compiler/discovery/_helpers/validation/discovery.py b/src/sqlbuild/compiler/discovery/_helpers/validation/discovery.py index 46d967b628..9ea32d11dd 100644 --- a/src/sqlbuild/compiler/discovery/_helpers/validation/discovery.py +++ b/src/sqlbuild/compiler/discovery/_helpers/validation/discovery.py @@ -6,7 +6,7 @@ from typing import Protocol import sqlbuild._native as _native -from sqlbuild.compiler.discovery.models import ( +from sqlbuild._native import ( DiscoveredAssetFunction, DiscoveredCheckFunction, DiscoveredLoaderFunction, diff --git a/src/sqlbuild/compiler/discovery/constants.py b/src/sqlbuild/compiler/discovery/constants.py index 4eca4a8a6c..cb4fcc00e1 100644 --- a/src/sqlbuild/compiler/discovery/constants.py +++ b/src/sqlbuild/compiler/discovery/constants.py @@ -11,6 +11,3 @@ SETTINGS_SECTION: str = "settings" TABLE_PROMOTION_MODE_SETTING_KEY: str = "table_promotion_mode" STAGED_PROMOTION_MODE: str = "staged" - -SQL_TESTS_OWNERSHIP_ROOT: str = "tests/unit" -SQL_SCENARIOS_OWNERSHIP_ROOT: str = "tests/scenarios" diff --git a/src/sqlbuild/compiler/discovery/models.py b/src/sqlbuild/compiler/discovery/models.py deleted file mode 100644 index b5b1732ec8..0000000000 --- a/src/sqlbuild/compiler/discovery/models.py +++ /dev/null @@ -1,409 +0,0 @@ -"""Structured models for discovered project inputs.""" - -from __future__ import annotations - -from collections.abc import Callable -from dataclasses import dataclass, field -from pathlib import Path - -from sqlbuild._native import ( - CollectionRendering, - DiscoveredAdapterFile, - DiscoveredAuditFile, - DiscoveredConstantFile, - DiscoveredEnumFile, - DiscoveredModelSchemaFile, - DiscoveredPythonFunctionFile, - DiscoveredSchemaFile, - DiscoveredSeedFile, - DiscoveredSqlFunctionFile, - DiscoveredSqlHookFile, - DiscoveredSqlModelFile, - DiscoveredSqlTestBlock, - EnumMember, - EventExportSeverity, - LocalConfig, - NativeDiscoverySession, - ProjectConfig, - ScopeKind, - SourceColumnEntry, - SourceEntry, - SourceWriteStrategy, - SqlLogicalType, - SqlValue, - SqlValueKind, -) -from sqlbuild.compiler.discovery.constants import ( - SQL_SCENARIOS_OWNERSHIP_ROOT, - SQL_TESTS_OWNERSHIP_ROOT, -) -from sqlbuild.compiler.discovery.types import ( - LoaderConnectionMode, - ProjectProvider, -) -from sqlbuild.python_nodes.models import AuditCase, ColumnLineageRef, RetryPolicy, SqlResourceRef -from sqlbuild.python_nodes.types import PythonCheckSeverity -from sqlbuild.runtime.event_exporting.constants import EVENT_EXPORT_KINDS -from sqlbuild.runtime.output_capture.types import CommandOutputStream - - -@dataclass(frozen=True) -class SqlHookEntry: - """A model lifecycle hook that executes SQL.""" - - statement: str - name: str | None = None - relative_path: Path | None = None - definition_sql: str | None = None - kwargs: dict[str, object] | None = None - description: str | None = None - reads: tuple[SqlResourceRef, ...] = () - - -@dataclass(frozen=True) -class PythonHookEntry: - """A model lifecycle hook that invokes a discovered Python hook.""" - - name: str - kwargs: dict[str, object] - - -@dataclass(frozen=True) -class EnumDeclaration: - """One validated public or model-local enum declaration.""" - - name: str - members: tuple[EnumMember, ...] - scalar_type: str - relative_path: Path - model_name: str | None = None - - -@dataclass(frozen=True) -class ConstantDeclaration: - """One validated public or model-local constant declaration.""" - - name: str - value: SqlValue - relative_path: Path - model_name: str | None = None - render_as: CollectionRendering | None = None - - @property - def logical_type(self) -> SqlLogicalType: - """Return the normalized logical type.""" - - return self.value.logical_type - - @property - def scalar_type(self) -> str: - """Retain the existing SQL scalar type label for scalar declarations.""" - - names: dict[SqlValueKind, str] = { - SqlValueKind.STRING: "VARCHAR", - SqlValueKind.INTEGER: "INTEGER", - SqlValueKind.BOOLEAN: "BOOLEAN", - SqlValueKind.FLOAT: "FLOAT", - SqlValueKind.DECIMAL: "DECIMAL", - SqlValueKind.NULL: "NULL", - } - return names.get(self.value.kind, self.value.logical_type.display_name.upper()) - - -@dataclass(frozen=True) -class DiscoveredSourceFile: - """A discovered source declaration file and its raw contents.""" - - file_path: Path - relative_path: Path - contents: str - source_entries: tuple[SourceEntry, ...] - - -@dataclass(frozen=True) -class DiscoveredSqlTestFile: - """A discovered SQL-native test file and its raw contents.""" - - file_path: Path - relative_path: Path - contents: str - blocks: tuple[DiscoveredSqlTestBlock, ...] - ownership_root: Path = Path(SQL_TESTS_OWNERSHIP_ROOT) - - -@dataclass(frozen=True) -class SqlTestParameterDeclaration: - """One typed parameter declared by a SQL-native test template.""" - - name: str - value_type: SqlValueKind - nullable: bool = False - - -@dataclass(frozen=True) -class DiscoveredSqlScenarioFile: - """A discovered SQL-native scenario file and its raw contents.""" - - file_path: Path - relative_path: Path - contents: str - header_values: dict[str, object] - sql_body: str - name: str - ownership_root: Path = Path(SQL_SCENARIOS_OWNERSHIP_ROOT) - sql_body_span: tuple[int, int] | None = None - - -@dataclass(frozen=True) -class DiscoveredMacroFile: - """A discovered project macro file and its raw contents.""" - - file_path: Path - relative_path: Path - contents: str - scope_kind: ScopeKind = ScopeKind.GLOBAL - ownership_root: Path | None = None - owning_path: Path | None = None - declaration_root: Path | None = None - - -@dataclass(frozen=True) -class DiscoveredMaterializationFile: - """A discovered custom materialization Python file.""" - - file_path: Path - relative_path: Path - name: str - provider_usages: tuple[DiscoveredProviderUsage, ...] = field(default_factory=tuple) - - -@dataclass(frozen=True) -class DiscoveredProviderUsage: - """Framework-owned metadata for provider usage by a Python callable surface.""" - - provider_name: str - parameter_name: str - annotation_class_name: str | None = None - annotation_module: str | None = None - - -@dataclass(frozen=True) -class DiscoveredLoaderFunction: - """A discovered project source loader function.""" - - file_path: Path - relative_path: Path - name: str - function: Callable[..., object] - depends_on: tuple[Callable[..., object] | SqlResourceRef, ...] = field(default_factory=tuple) - destination: str | None = None - write_strategy: SourceWriteStrategy | None = None - cursor_column: str | None = None - unique_key: tuple[str, ...] = field(default_factory=tuple) - columns: tuple[SourceColumnEntry, ...] = field(default_factory=tuple) - contract: str | None = None - description: str | None = None - connection_mode: LoaderConnectionMode = LoaderConnectionMode.SQLBUILD - provider_usages: tuple[DiscoveredProviderUsage, ...] = field(default_factory=tuple) - - -@dataclass(frozen=True) -class DiscoveredTaskFunction: - """A discovered project task function.""" - - file_path: Path - relative_path: Path - name: str - function: Callable[..., object] - depends_on: tuple[Callable[..., object] | SqlResourceRef, ...] = field(default_factory=tuple) - tags: tuple[str, ...] = field(default_factory=tuple) - group: str | None = None - description: str | None = None - meta: dict[str, object] | None = None - retry: RetryPolicy | None = None - provider_usages: tuple[DiscoveredProviderUsage, ...] = field(default_factory=tuple) - - -@dataclass(frozen=True) -class DiscoveredAssetFunction: - """A discovered project asset function.""" - - file_path: Path - relative_path: Path - name: str - function: Callable[..., object] - depends_on: tuple[Callable[..., object] | SqlResourceRef, ...] = field(default_factory=tuple) - tags: tuple[str, ...] = field(default_factory=tuple) - group: str | None = None - description: str | None = None - meta: dict[str, object] | None = None - columns: tuple[SourceColumnEntry, ...] = field(default_factory=tuple) - column_lineage: dict[str, tuple[ColumnLineageRef, ...]] | None = None - retry: RetryPolicy | None = None - provider_usages: tuple[DiscoveredProviderUsage, ...] = field(default_factory=tuple) - - -@dataclass(frozen=True) -class DiscoveredCheckFunction: - """A discovered project check function.""" - - file_path: Path - relative_path: Path - name: str - function: Callable[..., object] - depends_on: tuple[Callable[..., object] | SqlResourceRef, ...] - severity: PythonCheckSeverity = PythonCheckSeverity.ERROR - tags: tuple[str, ...] = field(default_factory=tuple) - group: str | None = None - description: str | None = None - meta: dict[str, object] | None = None - provider_usages: tuple[DiscoveredProviderUsage, ...] = field(default_factory=tuple) - - -@dataclass(frozen=True) -class DiscoveredHookFunction: - """A discovered project model lifecycle hook function.""" - - file_path: Path - relative_path: Path - name: str - function: Callable[..., object] - description: str | None = None - reads: tuple[SqlResourceRef, ...] = field(default_factory=tuple) - provider_usages: tuple[DiscoveredProviderUsage, ...] = field(default_factory=tuple) - scope_kind: ScopeKind = ScopeKind.GLOBAL - ownership_root: Path | None = None - owning_path: Path | None = None - declaration_root: Path | None = None - - -@dataclass(frozen=True) -class DiscoveredAuditFactory: - """A discovered audit factory and its normalized cases.""" - - name: str - function: Callable[..., object] - file_path: Path - relative_path: Path - line: int - cases: tuple[AuditCase, ...] = field(default_factory=tuple) - - -@dataclass(frozen=True) -class DiscoveredEventExporterDeclaration: - """A discovered exporter declaration before provider binding.""" - - file_path: Path - relative_path: Path - name: str - function: Callable[..., object] - event_kinds: frozenset[str] - min_severity: EventExportSeverity - - -@dataclass(frozen=True) -class DiscoveredEventExporter: - """A discovered canonical lifecycle event exporter function.""" - - file_path: Path - relative_path: Path - name: str - function: Callable[..., object] - event_kinds: frozenset[str] = EVENT_EXPORT_KINDS - min_severity: EventExportSeverity = EventExportSeverity.DEBUG - provider_usages: tuple[DiscoveredProviderUsage, ...] = field(default_factory=tuple) - - -@dataclass(frozen=True) -class DiscoveredCommandOutputSinkDeclaration: - """A command-output sink declaration before provider binding.""" - - file_path: Path - relative_path: Path - name: str - function: Callable[..., object] - streams: frozenset[CommandOutputStream] - - -@dataclass(frozen=True) -class DiscoveredCommandOutputSink: - """A discovered command-output sink function.""" - - file_path: Path - relative_path: Path - name: str - function: Callable[..., object] - streams: frozenset[CommandOutputStream] - provider_usages: tuple[DiscoveredProviderUsage, ...] = field(default_factory=tuple) - - -@dataclass(frozen=True) -class DiscoveredProvider: - """A discovered project provider class and validated settings object.""" - - file_path: Path - relative_path: Path - name: str - provider_class: type[ProjectProvider] - settings: ProjectProvider - - -@dataclass(frozen=True) -class DiscoveredPythonNodeFunctions: - """Discovered project Python DAG node functions grouped by kind.""" - - loaders: tuple[DiscoveredLoaderFunction, ...] = field(default_factory=tuple) - tasks: tuple[DiscoveredTaskFunction, ...] = field(default_factory=tuple) - assets: tuple[DiscoveredAssetFunction, ...] = field(default_factory=tuple) - checks: tuple[DiscoveredCheckFunction, ...] = field(default_factory=tuple) - audit_factories: tuple[DiscoveredAuditFactory, ...] = field(default_factory=tuple) - - -@dataclass(frozen=True) -class DiscoveredProjectInputs: - """All raw project inputs discovered from disk before semantic resolution.""" - - project_config: ProjectConfig - local_config: LocalConfig - project_dir: Path | None = None - model_files: tuple[DiscoveredSqlModelFile, ...] = field(default_factory=tuple) - enum_files: tuple[DiscoveredEnumFile, ...] = field(default_factory=tuple) - constant_files: tuple[DiscoveredConstantFile, ...] = field(default_factory=tuple) - model_schema_files: tuple[DiscoveredModelSchemaFile, ...] = field(default_factory=tuple) - sql_function_files: tuple[DiscoveredSqlFunctionFile, ...] = field(default_factory=tuple) - sql_hook_files: tuple[DiscoveredSqlHookFile, ...] = field(default_factory=tuple) - python_function_files: tuple[DiscoveredPythonFunctionFile, ...] = field(default_factory=tuple) - schema_files: tuple[DiscoveredSchemaFile, ...] = field(default_factory=tuple) - source_files: tuple[DiscoveredSourceFile, ...] = field(default_factory=tuple) - seed_files: tuple[DiscoveredSeedFile, ...] = field(default_factory=tuple) - test_files: tuple[DiscoveredSqlTestFile, ...] = field(default_factory=tuple) - scenario_files: tuple[DiscoveredSqlScenarioFile, ...] = field(default_factory=tuple) - audit_files: tuple[DiscoveredAuditFile, ...] = field(default_factory=tuple) - macro_files: tuple[DiscoveredMacroFile, ...] = field(default_factory=tuple) - materialization_files: tuple[DiscoveredMaterializationFile, ...] = field(default_factory=tuple) - loader_functions: tuple[DiscoveredLoaderFunction, ...] = field(default_factory=tuple) - task_functions: tuple[DiscoveredTaskFunction, ...] = field(default_factory=tuple) - asset_functions: tuple[DiscoveredAssetFunction, ...] = field(default_factory=tuple) - check_functions: tuple[DiscoveredCheckFunction, ...] = field(default_factory=tuple) - audit_factories: tuple[DiscoveredAuditFactory, ...] = field(default_factory=tuple) - hook_functions: tuple[DiscoveredHookFunction, ...] = field(default_factory=tuple) - event_exporters: tuple[DiscoveredEventExporter, ...] = field(default_factory=tuple) - command_output_sinks: tuple[DiscoveredCommandOutputSink, ...] = field(default_factory=tuple) - providers: tuple[DiscoveredProvider, ...] = field(default_factory=tuple) - adapter_file: DiscoveredAdapterFile | None = None - native_session: NativeDiscoverySession | None = field(default=None, compare=False, repr=False) - """The native session that read these declaration files, for later native stages. - - `None` means no session: readers must use the Python objects. Any copy that changes a - discovered collection must set it to `None` so it never describes other files. - """ - - -@dataclass(frozen=True) -class DiscoveredRuntimeExtensions: - """Command-startup providers, typed sinks, and lifecycle delivery settings.""" - - providers: tuple[DiscoveredProvider, ...] = field(default_factory=tuple) - event_exporters: tuple[DiscoveredEventExporter, ...] = field(default_factory=tuple) - command_output_sinks: tuple[DiscoveredCommandOutputSink, ...] = field(default_factory=tuple) - lifecycle_shutdown_timeout_seconds: int | None = None diff --git a/src/sqlbuild/compiler/discovery/types.py b/src/sqlbuild/compiler/discovery/types.py deleted file mode 100644 index 133eee9d52..0000000000 --- a/src/sqlbuild/compiler/discovery/types.py +++ /dev/null @@ -1,18 +0,0 @@ -"""Discovery domain types.""" - -from __future__ import annotations - -from enum import StrEnum -from typing import TYPE_CHECKING - -if TYPE_CHECKING: - from sqlbuild.provider.classes.provider import Provider - -type ProjectProvider = Provider - - -class LoaderConnectionMode(StrEnum): - """How a source loader owns the destination warehouse connection.""" - - SQLBUILD = "sqlbuild" - EXTERNAL = "external" diff --git a/src/sqlbuild/compiler/frontier/constants.py b/src/sqlbuild/compiler/frontier/constants.py index 02232e4b6c..791af21afd 100644 --- a/src/sqlbuild/compiler/frontier/constants.py +++ b/src/sqlbuild/compiler/frontier/constants.py @@ -2,8 +2,5 @@ from __future__ import annotations -import itertools - STAGE_CAPTURE_DIR_ENV_VAR: str = "SQLBUILD_COMPILER_STAGE_CAPTURE_DIR" COMPILE_REUSE_DISABLE_ENV_VAR: str = "SQLBUILD_DISABLE_COMPILE_REUSE" -STAGE_CAPTURE_SEQUENCE: itertools.count[int] = itertools.count(1) diff --git a/src/sqlbuild/compiler/pipeline/_helpers/materializations.py b/src/sqlbuild/compiler/pipeline/_helpers/materializations.py index e40fee03a5..c8225f220e 100644 --- a/src/sqlbuild/compiler/pipeline/_helpers/materializations.py +++ b/src/sqlbuild/compiler/pipeline/_helpers/materializations.py @@ -7,8 +7,7 @@ from types import ModuleType from typing import Any -from sqlbuild._native import PlannerInputError -from sqlbuild.compiler.discovery.models import DiscoveredMaterializationFile +from sqlbuild._native import DiscoveredMaterializationFile, PlannerInputError from sqlbuild.executor.custom.models import MaterializationContext, MaterializationResult diff --git a/src/sqlbuild/compiler/pipeline/models.py b/src/sqlbuild/compiler/pipeline/models.py deleted file mode 100644 index 1d198eee9e..0000000000 --- a/src/sqlbuild/compiler/pipeline/models.py +++ /dev/null @@ -1,75 +0,0 @@ -"""Compiler pipeline models.""" - -from __future__ import annotations - -from collections.abc import Callable -from dataclasses import dataclass, field -from functools import cached_property -from typing import Any - -import sqlbuild._native as _native -from sqlbuild._native import ( - LineageGraphViews, - PlanOutput, - PythonPlanEntry, - lineage_graph_views, -) -from sqlbuild.compiler.compile.models import CompiledObjectKey -from sqlbuild.compiler.compile.types import CompiledProjectView - - -@dataclass(frozen=True) -class ProjectGraph: - """Static compiled project graph held natively; dict views are built only when read.""" - - project: CompiledProjectView - native: _native.NativeProjectGraph = field(compare=False, repr=False) - - @cached_property - def lineage_views(self) -> LineageGraphViews: - """Lineage edges and selector indexes as dicts, converted from the native graph once.""" - - return lineage_graph_views(self.native) - - @property - def upstream_deps(self) -> dict[CompiledObjectKey, tuple[CompiledObjectKey, ...]]: - """Lineage upstream edges, SQL tests stripped.""" - - return self.lineage_views.upstream_deps - - @property - def downstream_deps(self) -> dict[CompiledObjectKey, tuple[CompiledObjectKey, ...]]: - """Lineage downstream edges sorted by `(resource type, name)`.""" - - return self.lineage_views.downstream_deps - - @property - def tag_index(self) -> dict[str, frozenset[CompiledObjectKey]]: - """Tag to tagged models, seeds and functions.""" - - return self.lineage_views.tag_index - - @property - def path_index(self) -> dict[CompiledObjectKey, str]: - """Model key to its folder below `models/`.""" - - return self.lineage_views.path_index - - @property - def all_keys(self) -> dict[str, CompiledObjectKey]: - """Selector name to key.""" - - return self.lineage_views.all_keys - - -@dataclass(frozen=True) -class CompilePipelineResult: - """Complete output from the compile-and-plan pipeline.""" - - project: CompiledProjectView - plan_output: PlanOutput - custom_materializations: dict[str, Callable[..., Any]] = field(default_factory=dict) - python_node_names: frozenset[str] = field(default_factory=frozenset) - python_plan_entries: tuple[PythonPlanEntry, ...] = field(default_factory=tuple) - compile_seconds: float | None = None - planning_seconds: float | None = None diff --git a/src/sqlbuild/compiler/python_nodes/main/_hook_identities.py b/src/sqlbuild/compiler/python_nodes/main/_hook_identities.py index 9d802bcc6c..6ce05786b1 100644 --- a/src/sqlbuild/compiler/python_nodes/main/_hook_identities.py +++ b/src/sqlbuild/compiler/python_nodes/main/_hook_identities.py @@ -4,8 +4,7 @@ from pathlib import Path -from sqlbuild._native import PythonNodeIdentity -from sqlbuild.compiler.discovery.models import DiscoveredHookFunction +from sqlbuild._native import DiscoveredHookFunction, PythonNodeIdentity from sqlbuild.compiler.fingerprints.constants import NODE_TYPE_HOOK from sqlbuild.compiler.python_nodes.main.identity import build_python_node_identity diff --git a/src/sqlbuild/compiler/references/types.py b/src/sqlbuild/compiler/references/types.py deleted file mode 100644 index 468a5b7108..0000000000 --- a/src/sqlbuild/compiler/references/types.py +++ /dev/null @@ -1,76 +0,0 @@ -"""SQL reference type-layer declarations.""" - -from __future__ import annotations - -from enum import StrEnum -from pathlib import Path -from typing import Literal, Protocol - - -class SqlReferenceKind(StrEnum): - REF = "ref" - SEED = "seed" - SOURCE = "source" - DBT_REF = "dbt_ref" - UDF = "udf" - TABLE_FUNCTION = "table_fn" - - @property - def function_name(self) -> str: - return f"__{self.value}" - - @property - def fixture_cte_prefix(self) -> str: - return f"{self.function_name}__" - - def example_call(self, *args: str, quote: Literal["'", '"'] = "'") -> str: - quoted_args: str = ", ".join( - f"{quote}{arg.replace(quote, quote + quote)}{quote}" for arg in args - ) - return f"{self.function_name}({quoted_args})" - - def placeholder_call(self, placeholder: str = "") -> str: - return f"{self.function_name}({placeholder})" - - -class ExternalSqlReferenceResolver(Protocol): - """Resolve first-class SQLBuild references backed by external metadata.""" - - def validate_model_names(self, *, known_model_names: set[str]) -> None: - """Validate SQLBuild model names against external integration resources.""" - - def extend_sql_test_model_names(self, *, known_model_names: set[str]) -> set[str]: - """Return extra model names valid for SQL-native test targets.""" - - def extend_sql_test_source_names(self, *, known_source_names: set[str]) -> set[str]: - """Return extra source names valid for SQL-native test mocks.""" - - def extend_sql_test_seed_names(self, *, known_seed_names: set[str]) -> set[str]: - """Return extra seed names valid for SQL-native test mocks.""" - - def validate_reference( - self, - *, - ref_kind: str, - ref_name: str, - ref_package: str | None, - owner_relative_sql_path: Path, - ) -> None: - """Validate one external SQL reference from a SQLBuild-owned file.""" - - def resolve_reference( - self, - *, - ref_kind: str, - ref_name: str, - ref_package: str | None, - ) -> str | None: - """Return the physical relation for an external reference, if resolvable.""" - - -class HardCodedRelationOwnerKind(StrEnum): - """The kind of Python code that names a project relation in SQL.""" - - NODE = "node" - HOOK = "hook" - LOADER = "loader" diff --git a/src/sqlbuild/executor/build/_helpers/external_source_loads.py b/src/sqlbuild/executor/build/_helpers/external_source_loads.py index 32fcfa2642..cfbfda20c9 100644 --- a/src/sqlbuild/executor/build/_helpers/external_source_loads.py +++ b/src/sqlbuild/executor/build/_helpers/external_source_loads.py @@ -4,11 +4,15 @@ from collections.abc import Callable -from sqlbuild._native import PlanOutput, SourceEntry, SourceLoadPlanEntry +from sqlbuild._native import ( + DiscoveredLoaderFunction, + LoaderConnectionMode, + PlanOutput, + SourceEntry, + SourceLoadPlanEntry, +) from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter from sqlbuild.compiler.compile.models import CompiledObjectKey -from sqlbuild.compiler.discovery.models import DiscoveredLoaderFunction -from sqlbuild.compiler.discovery.types import LoaderConnectionMode from sqlbuild.errors.contracts.exceptions import ExecutorInputError from sqlbuild.executor.build._helpers.source_node import execute_build_source_node from sqlbuild.executor.build.models import ( diff --git a/src/sqlbuild/executor/build/_helpers/source_node.py b/src/sqlbuild/executor/build/_helpers/source_node.py index 152bedf3ea..78375bd589 100644 --- a/src/sqlbuild/executor/build/_helpers/source_node.py +++ b/src/sqlbuild/executor/build/_helpers/source_node.py @@ -7,11 +7,16 @@ from collections.abc import Callable from typing import Any -from sqlbuild._native import CostContext, PlanOutput, SourceEntry, SourceLoadPlanEntry +from sqlbuild._native import ( + CostContext, + DiscoveredLoaderFunction, + PlanOutput, + SourceEntry, + SourceLoadPlanEntry, +) from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter from sqlbuild.adapter.contract.classes.statement_recorder import StatementRecorder from sqlbuild.compiler.compile.models import CompiledObjectKey -from sqlbuild.compiler.discovery.models import DiscoveredLoaderFunction from sqlbuild.executor.build.models import BuildCallbacks, BuildRuntimeParams from sqlbuild.executor.load.main._execute import execute_source_load from sqlbuild.executor.load.models import LoadExecutionResult, LoadRuntimeParams diff --git a/src/sqlbuild/executor/build/classes/build_scheduler.py b/src/sqlbuild/executor/build/classes/build_scheduler.py index 3f6fd1fac2..dcb64aef78 100644 --- a/src/sqlbuild/executor/build/classes/build_scheduler.py +++ b/src/sqlbuild/executor/build/classes/build_scheduler.py @@ -18,6 +18,7 @@ from sqlbuild._native import ( AuditPlanEntry, CostContext, + DiscoveredLoaderFunction, FunctionPlanEntry, IncrementalMode, IncrementalStrategy, @@ -38,7 +39,6 @@ from sqlbuild.compiler.auditing.types import AuditOutcome from sqlbuild.compiler.compile.models import CompiledObjectKey from sqlbuild.compiler.compile.types import CompiledResourceType -from sqlbuild.compiler.discovery.models import DiscoveredLoaderFunction from sqlbuild.cost.models import StatementExecutionTelemetry from sqlbuild.diagnostics.main.diagnostics_context import diagnostics_context from sqlbuild.diagnostics.main.log_debug_event import log_debug_event diff --git a/src/sqlbuild/executor/build/main/_external_source_loads.py b/src/sqlbuild/executor/build/main/_external_source_loads.py index 347ba7ff9d..752cc09978 100644 --- a/src/sqlbuild/executor/build/main/_external_source_loads.py +++ b/src/sqlbuild/executor/build/main/_external_source_loads.py @@ -2,10 +2,9 @@ from __future__ import annotations -from sqlbuild._native import PlanOutput +from sqlbuild._native import DiscoveredLoaderFunction, PlanOutput from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter from sqlbuild.compiler.compile.models import CompiledObjectKey -from sqlbuild.compiler.discovery.models import DiscoveredLoaderFunction from sqlbuild.executor.build._helpers.external_source_loads import ( run_external_source_loads_before_connections as _run_external_source_loads_before_connections, ) diff --git a/src/sqlbuild/executor/build/models.py b/src/sqlbuild/executor/build/models.py index e308b6fd9f..691ab14055 100644 --- a/src/sqlbuild/executor/build/models.py +++ b/src/sqlbuild/executor/build/models.py @@ -9,6 +9,7 @@ from sqlbuild._native import ( AuditPlanEntry, + DiscoveredLoaderFunction, FunctionPlanEntry, ModelPlanEntry, SeedPlanEntry, @@ -19,7 +20,6 @@ from sqlbuild.adapter.contract.models import LifeCycleEvent, RelationInfo from sqlbuild.adapter.contract.types import TablePromotionMode from sqlbuild.compiler.compile.models import CompiledObjectKey, CompiledRelationLocation -from sqlbuild.compiler.discovery.models import DiscoveredLoaderFunction from sqlbuild.cost.models import StatementExecutionTelemetry from sqlbuild.executor.auditing.models import AuditExecutionResult from sqlbuild.executor.build.types import BeforeModelMaterializeCallback, BuildStatus diff --git a/src/sqlbuild/executor/load/_helpers/execution.py b/src/sqlbuild/executor/load/_helpers/execution.py index 4e516e8c92..955a267d89 100644 --- a/src/sqlbuild/executor/load/_helpers/execution.py +++ b/src/sqlbuild/executor/load/_helpers/execution.py @@ -4,8 +4,7 @@ from collections.abc import Callable -from sqlbuild._native import SourceEntry -from sqlbuild.compiler.discovery.models import DiscoveredLoaderFunction +from sqlbuild._native import DiscoveredLoaderFunction, SourceEntry from sqlbuild.compiler.python_nodes.types import SkipMode from sqlbuild.executor.load.models import LoadExecutionIndexes, LoadExecutionResult from sqlbuild.executor.scheduling.types import ExecutionStatus diff --git a/src/sqlbuild/executor/load/_helpers/external.py b/src/sqlbuild/executor/load/_helpers/external.py index 5b041b73b1..b0f4773988 100644 --- a/src/sqlbuild/executor/load/_helpers/external.py +++ b/src/sqlbuild/executor/load/_helpers/external.py @@ -6,10 +6,9 @@ import time from collections.abc import Callable -from sqlbuild._native import SourceEntry +from sqlbuild._native import DiscoveredLoaderFunction, SourceEntry from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter from sqlbuild.adapter.contract.classes.statement_recorder import StatementRecorder -from sqlbuild.compiler.discovery.models import DiscoveredLoaderFunction from sqlbuild.errors.contracts.exceptions import ExecutorInputError from sqlbuild.executor.load.models import ( LoaderContext, diff --git a/src/sqlbuild/executor/load/_helpers/loader_invocation.py b/src/sqlbuild/executor/load/_helpers/loader_invocation.py index 193c8dccfe..c07f0fd419 100644 --- a/src/sqlbuild/executor/load/_helpers/loader_invocation.py +++ b/src/sqlbuild/executor/load/_helpers/loader_invocation.py @@ -7,11 +7,14 @@ from collections.abc import Callable, Mapping from typing import Any -from sqlbuild._native import SourceEntry, SourceWriteStrategy +from sqlbuild._native import ( + DiscoveredLoaderFunction, + HardCodedRelationOwnerKind, + SourceEntry, + SourceWriteStrategy, +) from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter from sqlbuild.adapter.contract.classes.statement_recorder import StatementRecorder -from sqlbuild.compiler.discovery.models import DiscoveredLoaderFunction -from sqlbuild.compiler.references.types import HardCodedRelationOwnerKind from sqlbuild.errors.contracts.exceptions import ExecutorInputError from sqlbuild.executor.load._helpers.execution import ( is_untargeted_self_managed_intermediate, diff --git a/src/sqlbuild/executor/load/_helpers/scheduling.py b/src/sqlbuild/executor/load/_helpers/scheduling.py index 074e624949..bedf7263e1 100644 --- a/src/sqlbuild/executor/load/_helpers/scheduling.py +++ b/src/sqlbuild/executor/load/_helpers/scheduling.py @@ -10,10 +10,8 @@ from functools import partial from typing import Any -from sqlbuild._native import SourceEntry +from sqlbuild._native import DiscoveredLoaderFunction, LoaderConnectionMode, SourceEntry from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter -from sqlbuild.compiler.discovery.models import DiscoveredLoaderFunction -from sqlbuild.compiler.discovery.types import LoaderConnectionMode from sqlbuild.compiler.python_nodes.types import SkipMode from sqlbuild.errors.contracts.exceptions import ExecutorInputError from sqlbuild.executor.load._helpers.dag_runtime import ( diff --git a/src/sqlbuild/executor/load/main/_build_execution_indexes.py b/src/sqlbuild/executor/load/main/_build_execution_indexes.py index 68b1f01212..4bb37bf351 100644 --- a/src/sqlbuild/executor/load/main/_build_execution_indexes.py +++ b/src/sqlbuild/executor/load/main/_build_execution_indexes.py @@ -2,8 +2,7 @@ from __future__ import annotations -from sqlbuild._native import SourceEntry -from sqlbuild.compiler.discovery.models import DiscoveredLoaderFunction +from sqlbuild._native import DiscoveredLoaderFunction, SourceEntry from sqlbuild.executor.load._helpers.execution import ( build_load_execution_indexes as _build_load_execution_indexes, ) diff --git a/src/sqlbuild/executor/load/main/_execute.py b/src/sqlbuild/executor/load/main/_execute.py index 48d8d44d90..f4ccaa0360 100644 --- a/src/sqlbuild/executor/load/main/_execute.py +++ b/src/sqlbuild/executor/load/main/_execute.py @@ -7,13 +7,17 @@ from dataclasses import replace from typing import Any -from sqlbuild._native import CostContext, SourceEntry, SourceWriteStrategy +from sqlbuild._native import ( + CostContext, + DiscoveredLoaderFunction, + LoaderConnectionMode, + SourceEntry, + SourceWriteStrategy, +) from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter from sqlbuild.adapter.contract.classes.statement_recorder import StatementRecorder from sqlbuild.adapter.contract.models import ColumnInfo from sqlbuild.adapter.contract.types import CursorKind -from sqlbuild.compiler.discovery.models import DiscoveredLoaderFunction -from sqlbuild.compiler.discovery.types import LoaderConnectionMode from sqlbuild.cursor_algebra.types import CursorScalar from sqlbuild.errors.contracts.exceptions import ExecutorInputError from sqlbuild.executor.load._helpers.cursors import ( diff --git a/src/sqlbuild/executor/load/main/run.py b/src/sqlbuild/executor/load/main/run.py index 26697faa72..676e1ed458 100644 --- a/src/sqlbuild/executor/load/main/run.py +++ b/src/sqlbuild/executor/load/main/run.py @@ -4,9 +4,8 @@ from typing import Any -from sqlbuild._native import SourceEntry +from sqlbuild._native import DiscoveredLoaderFunction, SourceEntry from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter -from sqlbuild.compiler.discovery.models import DiscoveredLoaderFunction from sqlbuild.executor.load._helpers.execution import build_load_execution_indexes from sqlbuild.executor.load._helpers.scheduling import ( open_load_connections, diff --git a/src/sqlbuild/executor/load/models.py b/src/sqlbuild/executor/load/models.py index 2eeb1349b1..39097300b7 100644 --- a/src/sqlbuild/executor/load/models.py +++ b/src/sqlbuild/executor/load/models.py @@ -9,7 +9,7 @@ from pathlib import Path from typing import Any -from sqlbuild._native import SourceEntry +from sqlbuild._native import DiscoveredLoaderFunction, SourceEntry from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter from sqlbuild.adapter.contract.classes.statement_recorder import StatementRecorder from sqlbuild.adapter.contract.models import ColumnInfo, LifeCycleEvent @@ -17,7 +17,6 @@ from sqlbuild.adapter.relations.main.resolve_qualified_name_parts import ( resolve_qualified_name_parts, ) -from sqlbuild.compiler.discovery.models import DiscoveredLoaderFunction from sqlbuild.compiler.python_nodes.types import PythonNodeKind, SkipMode from sqlbuild.errors.contracts.exceptions import ExecutorInputError from sqlbuild.executor.load.constants import LOADER_RELATION_QUALIFIER_SEPARATOR diff --git a/src/sqlbuild/executor/pipeline/_helpers/scenario.py b/src/sqlbuild/executor/pipeline/_helpers/scenario.py index cf20fcf2f4..9d1736fc0d 100644 --- a/src/sqlbuild/executor/pipeline/_helpers/scenario.py +++ b/src/sqlbuild/executor/pipeline/_helpers/scenario.py @@ -16,6 +16,7 @@ from typing import Any, ClassVar, Protocol from sqlbuild._native import ( + CompilePipelineResult, NativeProgressProjector, ScenarioExecutionPlan, SqlLexicalSyntax, @@ -29,7 +30,6 @@ from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter from sqlbuild.adapter.contract.types import TablePromotionMode from sqlbuild.compiler.compile.models import CompiledSqlScenario -from sqlbuild.compiler.pipeline.models import CompilePipelineResult from sqlbuild.diagnostics.main.log_debug_event import log_debug_event from sqlbuild.executor.pipeline._helpers.connections import ( close_connections, diff --git a/src/sqlbuild/executor/python_nodes/_helpers/execution.py b/src/sqlbuild/executor/python_nodes/_helpers/execution.py index 1c36b1e9dd..43bd63753c 100644 --- a/src/sqlbuild/executor/python_nodes/_helpers/execution.py +++ b/src/sqlbuild/executor/python_nodes/_helpers/execution.py @@ -8,9 +8,8 @@ from dataclasses import replace from typing import Any -from sqlbuild._native import CostContext +from sqlbuild._native import CostContext, DiscoveredAssetFunction from sqlbuild.adapter.contract.classes.statement_recorder import StatementRecorder -from sqlbuild.compiler.discovery.models import DiscoveredAssetFunction from sqlbuild.compiler.python_nodes.types import ( PythonNodeFanInAction, PythonNodeKind, diff --git a/src/sqlbuild/executor/python_nodes/_helpers/ingress_execution.py b/src/sqlbuild/executor/python_nodes/_helpers/ingress_execution.py index ef2a5fed83..6a24e47e14 100644 --- a/src/sqlbuild/executor/python_nodes/_helpers/ingress_execution.py +++ b/src/sqlbuild/executor/python_nodes/_helpers/ingress_execution.py @@ -10,17 +10,15 @@ from sqlbuild._native import ( CostContext, + DiscoveredAssetFunction, + DiscoveredLoaderFunction, DiscoveredPythonNode, + DiscoveredTaskFunction, PythonNodeGraph, PythonSqlRunLifecyclePlan, SourceEntry, ) from sqlbuild.adapter.contract.classes.statement_recorder import StatementRecorder -from sqlbuild.compiler.discovery.models import ( - DiscoveredAssetFunction, - DiscoveredLoaderFunction, - DiscoveredTaskFunction, -) from sqlbuild.compiler.python_nodes.types import PythonNodeKind, PythonNodeStatus, SkipMode from sqlbuild.errors.contracts.exceptions import ExecutorInputError from sqlbuild.executor.load.main._execute import execute_source_load diff --git a/src/sqlbuild/executor/python_nodes/_helpers/python_checks.py b/src/sqlbuild/executor/python_nodes/_helpers/python_checks.py index ed48cde51b..87027f7f5c 100644 --- a/src/sqlbuild/executor/python_nodes/_helpers/python_checks.py +++ b/src/sqlbuild/executor/python_nodes/_helpers/python_checks.py @@ -6,10 +6,15 @@ from collections.abc import Mapping from typing import Any -from sqlbuild._native import CostContext, DiscoveredPythonNode, PythonNodeGraph, PythonNodeIdentity +from sqlbuild._native import ( + CostContext, + DiscoveredCheckFunction, + DiscoveredPythonNode, + PythonNodeGraph, + PythonNodeIdentity, +) from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter from sqlbuild.adapter.contract.classes.statement_recorder import StatementRecorder -from sqlbuild.compiler.discovery.models import DiscoveredCheckFunction from sqlbuild.compiler.python_nodes.types import PythonNodeKind, PythonNodeStatus from sqlbuild.errors.contracts.exceptions import ExecutorInputError from sqlbuild.executor.load.models import LoadExecutionResult diff --git a/src/sqlbuild/executor/python_nodes/_helpers/relation_guard.py b/src/sqlbuild/executor/python_nodes/_helpers/relation_guard.py index 5e6eead8b3..fd755d2af9 100644 --- a/src/sqlbuild/executor/python_nodes/_helpers/relation_guard.py +++ b/src/sqlbuild/executor/python_nodes/_helpers/relation_guard.py @@ -2,7 +2,7 @@ from __future__ import annotations -from sqlbuild.compiler.references.types import HardCodedRelationOwnerKind +from sqlbuild._native import HardCodedRelationOwnerKind from sqlbuild.executor.python_nodes.classes.runtime_relation_guard import RuntimeRelationGuard from sqlbuild.executor.python_nodes.models import PythonNodeRuntime diff --git a/src/sqlbuild/executor/python_nodes/classes/runtime_relation_guard.py b/src/sqlbuild/executor/python_nodes/classes/runtime_relation_guard.py index 532a4f37b8..a5efd87f3d 100644 --- a/src/sqlbuild/executor/python_nodes/classes/runtime_relation_guard.py +++ b/src/sqlbuild/executor/python_nodes/classes/runtime_relation_guard.py @@ -6,6 +6,7 @@ from collections.abc import Mapping from sqlbuild._native import ( + HardCodedRelationOwnerKind, extract_relation_names, hard_coded_relation_remedy, match_project_relation, @@ -16,7 +17,6 @@ ProjectRelationIndex, RelationName, ) -from sqlbuild.compiler.references.types import HardCodedRelationOwnerKind from sqlbuild.python_nodes.models import SqlResourceRef _LOGGER: logging.Logger = logging.getLogger(__name__) diff --git a/src/sqlbuild/executor/python_nodes/main/_checks.py b/src/sqlbuild/executor/python_nodes/main/_checks.py index 6d1b0e7b9e..91ac05fd7c 100644 --- a/src/sqlbuild/executor/python_nodes/main/_checks.py +++ b/src/sqlbuild/executor/python_nodes/main/_checks.py @@ -4,8 +4,7 @@ from collections.abc import Mapping -from sqlbuild._native import PythonNodeGraph -from sqlbuild.compiler.discovery.models import DiscoveredCheckFunction +from sqlbuild._native import DiscoveredCheckFunction, PythonNodeGraph from sqlbuild.executor.load.models import LoadExecutionResult from sqlbuild.executor.python_nodes._helpers.python_checks import execute_python_check_nodes from sqlbuild.executor.python_nodes.models import ( diff --git a/src/sqlbuild/executor/python_nodes/main/_ingress.py b/src/sqlbuild/executor/python_nodes/main/_ingress.py index 71322e2201..ea1d03a5f5 100644 --- a/src/sqlbuild/executor/python_nodes/main/_ingress.py +++ b/src/sqlbuild/executor/python_nodes/main/_ingress.py @@ -4,8 +4,7 @@ from collections.abc import Mapping -from sqlbuild._native import PythonNodeGraph, SourceEntry -from sqlbuild.compiler.discovery.models import DiscoveredLoaderFunction +from sqlbuild._native import DiscoveredLoaderFunction, PythonNodeGraph, SourceEntry from sqlbuild.executor.python_nodes._helpers.ingress_execution import ( execute_ingress_python_loader_nodes, ) diff --git a/src/sqlbuild/executor/python_nodes/types.py b/src/sqlbuild/executor/python_nodes/types.py index 8c6c2ac47f..222d443440 100644 --- a/src/sqlbuild/executor/python_nodes/types.py +++ b/src/sqlbuild/executor/python_nodes/types.py @@ -4,8 +4,7 @@ from typing import TYPE_CHECKING, Protocol -from sqlbuild._native import PythonNodeIdentity -from sqlbuild.compiler.discovery.models import DiscoveredAssetFunction, DiscoveredTaskFunction +from sqlbuild._native import DiscoveredAssetFunction, DiscoveredTaskFunction, PythonNodeIdentity type ExecutablePythonNode = DiscoveredTaskFunction | DiscoveredAssetFunction diff --git a/src/sqlbuild/executor/run/_helpers/execution/hooks.py b/src/sqlbuild/executor/run/_helpers/execution/hooks.py index ac5d0c572e..b1cce44223 100644 --- a/src/sqlbuild/executor/run/_helpers/execution/hooks.py +++ b/src/sqlbuild/executor/run/_helpers/execution/hooks.py @@ -5,22 +5,25 @@ from collections.abc import Mapping from typing import Any -from sqlbuild._native import CostContext, PythonNodeIdentity, SourceEntry, render_source_relation +from sqlbuild._native import ( + CostContext, + DiscoveredHookFunction, + HardCodedRelationOwnerKind, + PythonHookEntry, + PythonNodeIdentity, + SourceEntry, + SqlHookEntry, + render_source_relation, +) from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter from sqlbuild.adapter.contract.classes.statement_recorder import StatementRecorder from sqlbuild.adapter.relations.main.resolve_relation_location_qualified_name import ( resolve_relation_location_qualified_name, ) from sqlbuild.compiler.compile.models import CompiledRelationLocation -from sqlbuild.compiler.discovery.models import ( - DiscoveredHookFunction, - PythonHookEntry, - SqlHookEntry, -) from sqlbuild.compiler.fingerprints.constants import NODE_TYPE_HOOK from sqlbuild.compiler.python_nodes.main.identity import build_python_node_identity from sqlbuild.compiler.python_nodes.types import SkipMode -from sqlbuild.compiler.references.types import HardCodedRelationOwnerKind from sqlbuild.cost.models import CostResourceContext from sqlbuild.errors.contracts.exceptions import ExecutorInputError from sqlbuild.executor.python_nodes.classes.runtime_relation_guard import RuntimeRelationGuard diff --git a/src/sqlbuild/executor/run/_helpers/materializations/view.py b/src/sqlbuild/executor/run/_helpers/materializations/view.py index 5e44371070..3e3a682cb3 100644 --- a/src/sqlbuild/executor/run/_helpers/materializations/view.py +++ b/src/sqlbuild/executor/run/_helpers/materializations/view.py @@ -5,7 +5,7 @@ from collections.abc import Mapping from typing import Any -from sqlbuild._native import AuditPlanEntry, ModelPlanEntry, SourceEntry +from sqlbuild._native import AuditPlanEntry, DiscoveredHookFunction, ModelPlanEntry, SourceEntry from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter from sqlbuild.adapter.contract.classes.statement_recorder import StatementRecorder from sqlbuild.adapter.relations.main.resolve_relation_location_qualified_name import ( @@ -13,7 +13,6 @@ ) from sqlbuild.compiler.auditing.types import AuditOutcome, AuditRunScope from sqlbuild.compiler.compile.models import CompiledRelationLocation -from sqlbuild.compiler.discovery.models import DiscoveredHookFunction from sqlbuild.diagnostics.main.diagnostics_context import diagnostics_context from sqlbuild.executor.auditing.main._execute import execute_audit from sqlbuild.executor.auditing.models import AuditExecutionResult diff --git a/src/sqlbuild/executor/run/models.py b/src/sqlbuild/executor/run/models.py index 6079ae2a79..eb0c923a80 100644 --- a/src/sqlbuild/executor/run/models.py +++ b/src/sqlbuild/executor/run/models.py @@ -14,6 +14,7 @@ CursorGrain, CursorType, DateValue, + DiscoveredHookFunction, Duration, FutureCursorSafetyEvidence, FutureCursorsConfig, @@ -21,6 +22,7 @@ MaximumStartSafetyEvidence, MicrobatchLimitAction, ModelPlanEntry, + PythonHookEntry, SourceEntry, StartCursorsConfig, TimestampValue, @@ -31,7 +33,6 @@ from sqlbuild.adapter.contract.classes.statement_recorder import StatementRecorder from sqlbuild.adapter.contract.models import LifeCycleEvent, QueryResult from sqlbuild.compiler.compile.models import CompiledRelationLocation -from sqlbuild.compiler.discovery.models import DiscoveredHookFunction, PythonHookEntry from sqlbuild.compiler.python_nodes.types import SkipMode from sqlbuild.cursor_algebra.exceptions import CursorAlgebraError from sqlbuild.cursor_algebra.types import BoundSentinel, CursorScalar diff --git a/src/sqlbuild/executor/testing/main/_execute.py b/src/sqlbuild/executor/testing/main/_execute.py index 69951e0ae0..e751c8351b 100644 --- a/src/sqlbuild/executor/testing/main/_execute.py +++ b/src/sqlbuild/executor/testing/main/_execute.py @@ -4,9 +4,8 @@ from typing import Any -from sqlbuild._native import SqlTestPlanEntry, render_sql_test_comparison_sql +from sqlbuild._native import CompileInputError, SqlTestPlanEntry, render_sql_test_comparison_sql from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter -from sqlbuild.compiler.compile.exceptions import CompileInputError from sqlbuild.executor.testing._helpers.difference_samples import add_difference_samples from sqlbuild.executor.testing._helpers.expected_column_probe import ( missing_expected_columns_message, diff --git a/src/sqlbuild/executor/testing/main/_sql_length.py b/src/sqlbuild/executor/testing/main/_sql_length.py index 3cb9b9c5e8..1ae52e7a79 100644 --- a/src/sqlbuild/executor/testing/main/_sql_length.py +++ b/src/sqlbuild/executor/testing/main/_sql_length.py @@ -2,9 +2,9 @@ from __future__ import annotations +from sqlbuild._native import CompileInputError from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter from sqlbuild.adapter.contract.types import StatementSizeLimit -from sqlbuild.compiler.compile.exceptions import CompileInputError from sqlbuild.executor.testing.constants import SQL_TEST_SIZE_BYTES diff --git a/src/sqlbuild/executor/testing/models.py b/src/sqlbuild/executor/testing/models.py index e3ce3d1abf..40247d1a60 100644 --- a/src/sqlbuild/executor/testing/models.py +++ b/src/sqlbuild/executor/testing/models.py @@ -5,8 +5,7 @@ from dataclasses import dataclass, field from pathlib import Path -from sqlbuild._native import SqlValue -from sqlbuild.compiler.discovery.models import SqlTestParameterDeclaration +from sqlbuild._native import SqlTestParameterDeclaration, SqlValue from sqlbuild.executor.testing.types import SqlTestOutcome diff --git a/src/sqlbuild/presentation/constants.py b/src/sqlbuild/presentation/constants.py deleted file mode 100644 index ef8eccfe8f..0000000000 --- a/src/sqlbuild/presentation/constants.py +++ /dev/null @@ -1,5 +0,0 @@ -"""Presentation constants.""" - -from __future__ import annotations - -DEFAULT_MAX_DISPLAY_ENTRIES: int = 20 diff --git a/src/sqlbuild/presentation/types.py b/src/sqlbuild/presentation/types.py deleted file mode 100644 index 467d0c0cfb..0000000000 --- a/src/sqlbuild/presentation/types.py +++ /dev/null @@ -1,15 +0,0 @@ -"""Type-layer declarations for presentation.""" - -from typing import Protocol - - -class TransientLineOwner(Protocol): - """A renderer that owns the transient line currently drawn in place on a terminal.""" - - def clear_transient_line(self) -> None: - """Erase the drawn transient line so a persistent line can take its place.""" - ... - - def redraw_transient_line(self) -> None: - """Draw the transient line again below the persistent output.""" - ... diff --git a/src/sqlbuild/provider/_helpers/session.py b/src/sqlbuild/provider/_helpers/session.py index 6776f19815..732740b559 100644 --- a/src/sqlbuild/provider/_helpers/session.py +++ b/src/sqlbuild/provider/_helpers/session.py @@ -4,7 +4,7 @@ from typing import TYPE_CHECKING -from sqlbuild.compiler.discovery.models import DiscoveredProvider +from sqlbuild._native import DiscoveredProvider from sqlbuild.provider.classes.session import ProviderSession if TYPE_CHECKING: diff --git a/src/sqlbuild/provider/main/session.py b/src/sqlbuild/provider/main/session.py index 231e165740..31c2712bb6 100644 --- a/src/sqlbuild/provider/main/session.py +++ b/src/sqlbuild/provider/main/session.py @@ -2,7 +2,7 @@ from __future__ import annotations -from sqlbuild.compiler.discovery.models import DiscoveredProvider +from sqlbuild._native import DiscoveredProvider from sqlbuild.provider._helpers.session import build_provider_session as _build_provider_session from sqlbuild.provider.classes.session import ProviderSession diff --git a/src/sqlbuild/rule_engine/_helpers/engine/_custom_catalogue.py b/src/sqlbuild/rule_engine/_helpers/engine/_custom_catalogue.py index c86d8bafe6..3e995d1a69 100644 --- a/src/sqlbuild/rule_engine/_helpers/engine/_custom_catalogue.py +++ b/src/sqlbuild/rule_engine/_helpers/engine/_custom_catalogue.py @@ -9,12 +9,12 @@ from pathlib import Path from types import ModuleType +from sqlbuild._native import RulesError from sqlbuild.rule_engine._helpers.engine.definition import ( resolve_rule_signature, rule_from_value, ) from sqlbuild.rule_engine.constants import INIT_MODULE_NAME, RULE_DECORATOR_TOKEN -from sqlbuild.rule_engine.exceptions import RulesError from sqlbuild.rule_engine.models import Rule diff --git a/src/sqlbuild/rule_engine/_helpers/engine/definition.py b/src/sqlbuild/rule_engine/_helpers/engine/definition.py index 973ded3f39..1df2c5fefc 100644 --- a/src/sqlbuild/rule_engine/_helpers/engine/definition.py +++ b/src/sqlbuild/rule_engine/_helpers/engine/definition.py @@ -6,7 +6,7 @@ from collections.abc import Callable from typing import cast, get_type_hints -from sqlbuild.rule_engine.exceptions import RulesError +from sqlbuild._native import RulesError from sqlbuild.rule_engine.models import Model, Project, Rule, RuleOption from sqlbuild.rule_engine.types import RuleCheck, RuleContext, RuleSubject diff --git a/src/sqlbuild/rule_engine/_helpers/engine/fact_replay.py b/src/sqlbuild/rule_engine/_helpers/engine/fact_replay.py index 0832f7e1a2..97600b0354 100644 --- a/src/sqlbuild/rule_engine/_helpers/engine/fact_replay.py +++ b/src/sqlbuild/rule_engine/_helpers/engine/fact_replay.py @@ -9,6 +9,7 @@ import orjson +from sqlbuild._native import FactDigestError from sqlbuild.rule_engine.classes.shared_fact_encoder import SharedFactEncoder from sqlbuild.rule_engine.classes.sql_document import SqlDocument from sqlbuild.rule_engine.constants import ( @@ -42,7 +43,6 @@ FACT_TREE_RELATIVE_PARTS, FACT_TREE_RESOURCES_UNDER, ) -from sqlbuild.rule_engine.exceptions import FactDigestError from sqlbuild.rule_engine.models import Model, RuleFactViews from sqlbuild.rule_engine.types import FactKey diff --git a/src/sqlbuild/rule_engine/_helpers/engine/hermeticity.py b/src/sqlbuild/rule_engine/_helpers/engine/hermeticity.py index 22173a8d50..efd3c40536 100644 --- a/src/sqlbuild/rule_engine/_helpers/engine/hermeticity.py +++ b/src/sqlbuild/rule_engine/_helpers/engine/hermeticity.py @@ -5,9 +5,8 @@ import ast from pathlib import Path -from sqlbuild._native import custom_rule_import_closure +from sqlbuild._native import NonHermeticRuleError, RulesError, custom_rule_import_closure from sqlbuild.rule_engine.constants import CUSTOM_RULE_IMPORT_ROOTS -from sqlbuild.rule_engine.exceptions import NonHermeticRuleError, RulesError from sqlbuild.rule_engine.models import Rule diff --git a/src/sqlbuild/rule_engine/_helpers/engine/rule_harness.py b/src/sqlbuild/rule_engine/_helpers/engine/rule_harness.py index fb31d74cc8..c0f3e10635 100644 --- a/src/sqlbuild/rule_engine/_helpers/engine/rule_harness.py +++ b/src/sqlbuild/rule_engine/_helpers/engine/rule_harness.py @@ -7,6 +7,9 @@ from tempfile import TemporaryDirectory from sqlbuild._native import ( + DiscoveredProjectInputs, + RuleAssertionError, + RuleUsageError, build_compiled_project, discover_project_inputs, evaluate_project_rules, @@ -14,13 +17,11 @@ ) from sqlbuild.adapters.duckdb.classes.duckdb_adapter import DuckDbAdapter from sqlbuild.compiler.compile.types import CompiledProjectView -from sqlbuild.compiler.discovery.models import DiscoveredProjectInputs from sqlbuild.rule_engine._helpers.engine.definition import ( resolve_rule_signature, rule_from_value, ) from sqlbuild.rule_engine.constants import MIN_CUSTOM_RULE_TEST_CASES -from sqlbuild.rule_engine.exceptions import RuleAssertionError, RuleUsageError from sqlbuild.rule_engine.models import ( Rule, RuleCase, diff --git a/src/sqlbuild/rule_engine/_helpers/host/custom_evaluation.py b/src/sqlbuild/rule_engine/_helpers/host/custom_evaluation.py index 0f40861c9e..6c3f676b7a 100644 --- a/src/sqlbuild/rule_engine/_helpers/host/custom_evaluation.py +++ b/src/sqlbuild/rule_engine/_helpers/host/custom_evaluation.py @@ -9,6 +9,7 @@ from itertools import chain from pathlib import Path +from sqlbuild._native import HostCancelledError, NonHermeticRuleError, RulesError from sqlbuild.compiler.compile.types import CompiledModelView, CompiledProjectView from sqlbuild.rule_engine._helpers.host.module_state import rule_namespaces from sqlbuild.rule_engine._helpers.host.tracked_facts import tracked_fact_views @@ -25,11 +26,6 @@ CUSTOM_HOST_CANCELLED_MESSAGE, CUSTOM_RULE_PROJECT_SUBJECT, ) -from sqlbuild.rule_engine.exceptions import ( - HostCancelledError, - NonHermeticRuleError, - RulesError, -) from sqlbuild.rule_engine.models import ( CustomHostPartition, CustomRuleEvaluation, diff --git a/src/sqlbuild/rule_engine/_helpers/host/custom_host.py b/src/sqlbuild/rule_engine/_helpers/host/custom_host.py index 76d0c2cbf9..cdddea35a9 100644 --- a/src/sqlbuild/rule_engine/_helpers/host/custom_host.py +++ b/src/sqlbuild/rule_engine/_helpers/host/custom_host.py @@ -14,7 +14,7 @@ from pathlib import Path from typing import Any, BinaryIO -from sqlbuild._native import build_rule_catalogue +from sqlbuild._native import HostCancelledError, RulesError, build_rule_catalogue from sqlbuild.compiler.compile.models import CompiledProject from sqlbuild.compiler.compile.types import CompiledProjectView from sqlbuild.rule_engine._helpers.engine.fact_replay import fact_key_payload @@ -33,7 +33,6 @@ CUSTOM_HOST_PROTOCOL_VERSION, CUSTOM_HOST_RUNTIME_VERSION, ) -from sqlbuild.rule_engine.exceptions import HostCancelledError, RulesError from sqlbuild.rule_engine.models import ( CustomHostPartition, CustomRuleRun, diff --git a/src/sqlbuild/rule_engine/_helpers/host/module_state.py b/src/sqlbuild/rule_engine/_helpers/host/module_state.py index eba9b96c5c..c72d0c4160 100644 --- a/src/sqlbuild/rule_engine/_helpers/host/module_state.py +++ b/src/sqlbuild/rule_engine/_helpers/host/module_state.py @@ -16,7 +16,7 @@ from types import BuiltinFunctionType, CellType, FunctionType, MethodType, ModuleType from typing import Any, cast -from sqlbuild.rule_engine.exceptions import OpaqueModuleStateError +from sqlbuild._native import OpaqueModuleStateError from sqlbuild.rule_engine.models import ModuleStateFingerprint, ModuleStateTrace _MAX_STATE_NODES: int = 50_000 diff --git a/src/sqlbuild/rule_engine/_helpers/host/tracked_facts.py b/src/sqlbuild/rule_engine/_helpers/host/tracked_facts.py index 9cdd9567fa..f5be531640 100644 --- a/src/sqlbuild/rule_engine/_helpers/host/tracked_facts.py +++ b/src/sqlbuild/rule_engine/_helpers/host/tracked_facts.py @@ -6,9 +6,9 @@ from pathlib import Path from typing import Any +from sqlbuild._native import ConstantDeclaration, EnumDeclaration, FactDigestError from sqlbuild.compiler.compile.models import CompiledSqlTest, InferredColumn from sqlbuild.compiler.compile.types import CompiledAuditView -from sqlbuild.compiler.discovery.models import ConstantDeclaration, EnumDeclaration from sqlbuild.rule_engine._helpers.engine.fact_replay import fact_error_digest, fact_value_digest from sqlbuild.rule_engine.classes.fact_reads import FactReads from sqlbuild.rule_engine.constants import ( @@ -42,7 +42,6 @@ FACT_TREE_RELATIVE_PARTS, FACT_TREE_RESOURCES_UNDER, ) -from sqlbuild.rule_engine.exceptions import FactDigestError from sqlbuild.rule_engine.models import Model, ModelSql, ProjectPath, RuleFactViews diff --git a/src/sqlbuild/rule_engine/classes/declaration_facts.py b/src/sqlbuild/rule_engine/classes/declaration_facts.py index c410d315b0..e0724599f7 100644 --- a/src/sqlbuild/rule_engine/classes/declaration_facts.py +++ b/src/sqlbuild/rule_engine/classes/declaration_facts.py @@ -1,7 +1,7 @@ """Declaration facts for custom rules.""" +from sqlbuild._native import ConstantDeclaration, EnumDeclaration from sqlbuild.compiler.compile.types import CompiledProjectView -from sqlbuild.compiler.discovery.models import ConstantDeclaration, EnumDeclaration class DeclarationFacts: diff --git a/src/sqlbuild/rule_engine/classes/module_state_monitor.py b/src/sqlbuild/rule_engine/classes/module_state_monitor.py index 98435f2506..ac4788dd2f 100644 --- a/src/sqlbuild/rule_engine/classes/module_state_monitor.py +++ b/src/sqlbuild/rule_engine/classes/module_state_monitor.py @@ -5,12 +5,12 @@ from itertools import chain from pathlib import Path +from sqlbuild._native import OpaqueModuleStateError from sqlbuild.rule_engine._helpers.host.module_state import ( module_state_fingerprint, module_state_snapshot, rules_root_prefix, ) -from sqlbuild.rule_engine.exceptions import OpaqueModuleStateError from sqlbuild.rule_engine.models import ModuleStateFingerprint, ModuleStateTrace _EMPTY_TRACE: ModuleStateTrace = ModuleStateTrace( diff --git a/src/sqlbuild/rule_engine/classes/project_tree.py b/src/sqlbuild/rule_engine/classes/project_tree.py index f192e64dae..9e37c1463b 100644 --- a/src/sqlbuild/rule_engine/classes/project_tree.py +++ b/src/sqlbuild/rule_engine/classes/project_tree.py @@ -8,9 +8,9 @@ from pathlib import Path import sqlbuild._native as _native +from sqlbuild._native import RuleUsageError from sqlbuild.compiler.compile.types import CompiledProjectView from sqlbuild.rule_engine.constants import PARENT_DIRECTORY_TOKEN -from sqlbuild.rule_engine.exceptions import RuleUsageError from sqlbuild.rule_engine.models import Model, ProjectPath _EXCLUDED_ROOTS: frozenset[str] = frozenset({".git", ".venv", "logs", "target", "venv"}) diff --git a/src/sqlbuild/rule_engine/classes/rule_context.py b/src/sqlbuild/rule_engine/classes/rule_context.py index 0fb4e32544..66fec806ea 100644 --- a/src/sqlbuild/rule_engine/classes/rule_context.py +++ b/src/sqlbuild/rule_engine/classes/rule_context.py @@ -5,6 +5,7 @@ from pathlib import Path from typing import Any, cast +from sqlbuild._native import RuleUsageError from sqlbuild.compiler.compile.types import CompiledModelView, CompiledProjectView from sqlbuild.rule_engine.classes.audit_facts import AuditFacts from sqlbuild.rule_engine.classes.column_facts import ColumnFacts @@ -16,7 +17,6 @@ from sqlbuild.rule_engine.classes.project_facts import ProjectFacts from sqlbuild.rule_engine.classes.sql_facts import SqlFacts from sqlbuild.rule_engine.constants import PARENT_DIRECTORY_TOKEN -from sqlbuild.rule_engine.exceptions import RuleUsageError from sqlbuild.rule_engine.models import ( Finding, Model, diff --git a/src/sqlbuild/rule_engine/classes/runtime_guard.py b/src/sqlbuild/rule_engine/classes/runtime_guard.py index 847fc4b1a2..d54026716f 100644 --- a/src/sqlbuild/rule_engine/classes/runtime_guard.py +++ b/src/sqlbuild/rule_engine/classes/runtime_guard.py @@ -12,9 +12,9 @@ from pathlib import Path from types import CodeType, FrameType, FunctionType, ModuleType +from sqlbuild._native import NonHermeticRuleError from sqlbuild.rule_engine._helpers.engine.hermeticity import allowed_custom_rule_import from sqlbuild.rule_engine.classes.project_tree import ProjectTree -from sqlbuild.rule_engine.exceptions import NonHermeticRuleError _IMPORT_EVENT: str = "import" _OPEN_EVENT: str = "open" diff --git a/src/sqlbuild/rule_engine/classes/sql_document.py b/src/sqlbuild/rule_engine/classes/sql_document.py index 29456d171a..e8eed481ec 100644 --- a/src/sqlbuild/rule_engine/classes/sql_document.py +++ b/src/sqlbuild/rule_engine/classes/sql_document.py @@ -2,9 +2,9 @@ from typing import Any +from sqlbuild._native import RuleUsageError from sqlbuild.compiler.sql_analysis.main.import_polyglot_sql import import_polyglot_sql from sqlbuild.rule_engine.constants import AST_CTE_KIND -from sqlbuild.rule_engine.exceptions import RuleUsageError from sqlbuild.rule_engine.models import SqlNode diff --git a/src/sqlbuild/rule_engine/classes/sql_facts.py b/src/sqlbuild/rule_engine/classes/sql_facts.py index d55c346673..7267d1003e 100644 --- a/src/sqlbuild/rule_engine/classes/sql_facts.py +++ b/src/sqlbuild/rule_engine/classes/sql_facts.py @@ -2,9 +2,9 @@ from functools import cached_property +from sqlbuild._native import RuleUsageError from sqlbuild.compiler.compile.types import CompiledModelView, CompiledProjectView from sqlbuild.rule_engine.classes.sql_document import SqlDocument -from sqlbuild.rule_engine.exceptions import RuleUsageError from sqlbuild.rule_engine.models import Model, ModelSql diff --git a/src/sqlbuild/rule_engine/exceptions.py b/src/sqlbuild/rule_engine/exceptions.py deleted file mode 100644 index 5e32b7cb94..0000000000 --- a/src/sqlbuild/rule_engine/exceptions.py +++ /dev/null @@ -1,38 +0,0 @@ -"""Rules public errors.""" - - -class RulesError(RuntimeError): - """Raised for invalid compiler rules or execution.""" - - -class RulesConfigError(RulesError): - """Raised when rules configuration breaks a project policy; help shows the setting to change.""" - - def __init__(self, message: str, *, help: str) -> None: - super().__init__(message) - self.message = message - self.help = help - - -class RuleUsageError(RulesError): - """Raised when a rule or rule test uses the public API incorrectly.""" - - -class NonHermeticRuleError(RulesError): - """Raised when a custom rule observes state the Rules cache cannot track.""" - - -class RuleAssertionError(RulesError): - """Raised when a custom-rule harness case produces an unexpected result.""" - - -class FactDigestError(RulesError): - """Raised when a recorded custom-rule fact cannot be recomputed or encoded deterministically.""" - - -class OpaqueModuleStateError(RulesError): - """Raised when custom-rule module state holds a value whose mutations cannot be observed.""" - - -class HostCancelledError(RulesError): - """Raised in a custom-rule host that stops because another host already failed.""" diff --git a/tests/integration/src/sqlbuild/cli/commands/main/test_load_command.py b/tests/integration/src/sqlbuild/cli/commands/main/test_load_command.py index d5cbf5b9b5..5e172c1c6b 100644 --- a/tests/integration/src/sqlbuild/cli/commands/main/test_load_command.py +++ b/tests/integration/src/sqlbuild/cli/commands/main/test_load_command.py @@ -18,6 +18,8 @@ BuildCommandRequest, CliUserError, CursorOverrides, + DiscoveredProjectInputs, + DiscoveredSourceFile, LoadCommandRequest, PlanCommandRequest, PlannerInputError, @@ -36,7 +38,6 @@ ) from sqlbuild.adapter.contract.types import CursorKind from sqlbuild.adapters.duckdb.classes.duckdb_adapter import DuckDbAdapter -from sqlbuild.compiler.discovery.models import DiscoveredProjectInputs, DiscoveredSourceFile from sqlbuild.executor.load.main.run import run_load_pipeline from sqlbuild.executor.load.models import ( LoadCallbacks, diff --git a/tests/integration/src/sqlbuild/cli/commands/main/test_rules.py b/tests/integration/src/sqlbuild/cli/commands/main/test_rules.py index 165d694d04..51dad358ea 100644 --- a/tests/integration/src/sqlbuild/cli/commands/main/test_rules.py +++ b/tests/integration/src/sqlbuild/cli/commands/main/test_rules.py @@ -17,6 +17,8 @@ from sqlbuild._native import ( CustomHostObserver, DeclarationScopeBuild, + DiscoveredMacroFile, + DiscoveredProjectInputs, LintRunResult, SqlLexicalSyntax, active_lint_stop, @@ -24,7 +26,6 @@ ) from sqlbuild._native import run_cli as main from sqlbuild.compiler.compile.models import LoadedMacro -from sqlbuild.compiler.discovery.models import DiscoveredMacroFile, DiscoveredProjectInputs from tests.integration.src.sqlbuild.cli.commands.main._test_types import ( CustomHostFailureIntegrationTestCase, CustomHostInterruptIntegrationTestCase, diff --git a/tests/integration/src/sqlbuild/cli/commands/main/test_startup_import_footprint.py b/tests/integration/src/sqlbuild/cli/commands/main/test_startup_import_footprint.py index d0b78d5354..ac3a1a0ea9 100644 --- a/tests/integration/src/sqlbuild/cli/commands/main/test_startup_import_footprint.py +++ b/tests/integration/src/sqlbuild/cli/commands/main/test_startup_import_footprint.py @@ -42,7 +42,6 @@ "yaml", "sqlbuild.adapter.contract.classes.base_adapter", "sqlbuild.compiler.compile.models", - "sqlbuild.compiler.discovery.models", "sqlbuild.lint", "sqlbuild.rule_engine", "sqlbuild.runtime.compute_logs", diff --git a/tests/integration/src/sqlbuild/compiler/analysis_session/helpers.py b/tests/integration/src/sqlbuild/compiler/analysis_session/helpers.py index 9d5bf390b9..da3677e75a 100644 --- a/tests/integration/src/sqlbuild/compiler/analysis_session/helpers.py +++ b/tests/integration/src/sqlbuild/compiler/analysis_session/helpers.py @@ -19,6 +19,7 @@ CollectionRendering, ColumnLineageMode, CompileAdapterContext, + CompileInputError, DynamicColumnContractProof, ModelSqlAnalysis, NativeModelAnalyses, @@ -40,7 +41,6 @@ from sqlbuild.adapter.contract.models import ExpressionInferenceProfile from sqlbuild.adapters.duckdb.classes.duckdb_adapter import DuckDbAdapter from sqlbuild.cli.entry.main.entry import main -from sqlbuild.compiler.compile.exceptions import CompileInputError from sqlbuild.compiler.compile.models import CompactLineageFacts, InferredColumn from sqlbuild.compiler.compile.types import CompiledProjectView from sqlbuild.compiler.lineage.types import InferredNullability diff --git a/tests/integration/src/sqlbuild/compiler/analysis_session/test_native_query_columns.py b/tests/integration/src/sqlbuild/compiler/analysis_session/test_native_query_columns.py index 78bfa1f58a..5e04593722 100644 --- a/tests/integration/src/sqlbuild/compiler/analysis_session/test_native_query_columns.py +++ b/tests/integration/src/sqlbuild/compiler/analysis_session/test_native_query_columns.py @@ -12,9 +12,13 @@ import pytest import sqlbuild._native as native_module -from sqlbuild._native import NativeColumnQuery, NativeModelAnalyses, infer_native_query_columns +from sqlbuild._native import ( + CompileInputError, + NativeColumnQuery, + NativeModelAnalyses, + infer_native_query_columns, +) from sqlbuild.adapter.contract.models import ExpressionInferenceProfile -from sqlbuild.compiler.compile.exceptions import CompileInputError from sqlbuild.compiler.compile.types import CompiledModelView, CompiledProjectView from tests.integration.src.sqlbuild.compiler.analysis_session._test_types import ( FallbackColumnsTestCase, diff --git a/tests/integration/src/sqlbuild/compiler/attachments/helpers.py b/tests/integration/src/sqlbuild/compiler/attachments/helpers.py index ac846e8dba..c382bd4f56 100644 --- a/tests/integration/src/sqlbuild/compiler/attachments/helpers.py +++ b/tests/integration/src/sqlbuild/compiler/attachments/helpers.py @@ -9,12 +9,12 @@ import sqlbuild._native as _native from sqlbuild._native import ( + CompileInputError, SqlLexicalSyntax, SqlReferenceScan, find_macro_call_names, scan_sql_reference_calls, ) -from sqlbuild.compiler.compile.exceptions import CompileInputError from sqlbuild.compiler.compile.types import SqlTestMode _SQL_PIECES: tuple[str, ...] = ( diff --git a/tests/integration/src/sqlbuild/compiler/compile/_helpers/refs/test_native_references.py b/tests/integration/src/sqlbuild/compiler/compile/_helpers/refs/test_native_references.py index 9edef9e095..5bc6993ece 100644 --- a/tests/integration/src/sqlbuild/compiler/compile/_helpers/refs/test_native_references.py +++ b/tests/integration/src/sqlbuild/compiler/compile/_helpers/refs/test_native_references.py @@ -9,8 +9,7 @@ import pytest import sqlbuild._native as _native -from sqlbuild._native import SqlLexicalSyntax, extract_sql_references -from sqlbuild.compiler.compile.exceptions import CompileInputError +from sqlbuild._native import CompileInputError, SqlLexicalSyntax, extract_sql_references from tests.integration.src.sqlbuild.compiler.compile._helpers.refs._test_types import ( CraftedReferenceTestCase, ReferenceExtractionTestCase, diff --git a/tests/integration/src/sqlbuild/compiler/compile/helpers.py b/tests/integration/src/sqlbuild/compiler/compile/helpers.py index a083d118c8..da62fbbb46 100644 --- a/tests/integration/src/sqlbuild/compiler/compile/helpers.py +++ b/tests/integration/src/sqlbuild/compiler/compile/helpers.py @@ -16,7 +16,9 @@ from sqlbuild._native import ( CompileAdapterContext, + CompileInputError, CompilerStage, + DiscoveredProjectInputs, MacroExpansionResult, ResourceIdentity, ResourceKind, @@ -29,9 +31,7 @@ resolve_effective_collection_rendering, ) from sqlbuild.adapters.duckdb.classes.duckdb_adapter import DuckDbAdapter -from sqlbuild.compiler.compile.exceptions import CompileInputError from sqlbuild.compiler.compile.models import LoadedMacro, MacroContext -from sqlbuild.compiler.discovery.models import DiscoveredProjectInputs from sqlbuild.compiler.macro_bridge.classes.macro_bridge import MacroBridge from sqlbuild.compiler.macro_bridge.constants import ACTIVE_MACRO_BRIDGE diff --git a/tests/integration/src/sqlbuild/compiler/compile/native_model_config/helpers.py b/tests/integration/src/sqlbuild/compiler/compile/native_model_config/helpers.py index 51f29fb712..e4bd4b8b4d 100644 --- a/tests/integration/src/sqlbuild/compiler/compile/native_model_config/helpers.py +++ b/tests/integration/src/sqlbuild/compiler/compile/native_model_config/helpers.py @@ -8,15 +8,15 @@ import sqlbuild._native as _native from sqlbuild._native import ( CompileModelConfig, + DiscoveredSqlModelFile, ModelValidationRequest, NativeModelConfigInputs, NativeModelConfigSession, ProjectConfig, + SqlReferenceKind, native_model_config_session, ) from sqlbuild.compiler.compile.models import CompileSqlReference -from sqlbuild.compiler.discovery.models import DiscoveredSqlModelFile -from sqlbuild.compiler.references.types import SqlReferenceKind _ACCEPTED: str = "accepted" diff --git a/tests/integration/src/sqlbuild/compiler/compiled_project/helpers.py b/tests/integration/src/sqlbuild/compiler/compiled_project/helpers.py index 45aee5d974..ce1fd9e280 100644 --- a/tests/integration/src/sqlbuild/compiler/compiled_project/helpers.py +++ b/tests/integration/src/sqlbuild/compiler/compiled_project/helpers.py @@ -19,6 +19,8 @@ CompileAdapterContext, CompileModelConfig, CompileProjectInputs, + DiscoveredProjectInputs, + DiscoveredSqlModelFile, LocalConfig, MeasurementThresholdBound, MeasurementThresholds, @@ -41,7 +43,6 @@ CompiledModelView, CompiledProjectView, ) -from sqlbuild.compiler.discovery.models import DiscoveredProjectInputs, DiscoveredSqlModelFile from tests.integration.src.sqlbuild.compiler.graph.helpers import write_files if TYPE_CHECKING: diff --git a/tests/integration/src/sqlbuild/compiler/compiled_project/test_native_audits.py b/tests/integration/src/sqlbuild/compiler/compiled_project/test_native_audits.py index d10fcc3fb3..3d593b327a 100644 --- a/tests/integration/src/sqlbuild/compiler/compiled_project/test_native_audits.py +++ b/tests/integration/src/sqlbuild/compiler/compiled_project/test_native_audits.py @@ -7,7 +7,7 @@ import pytest -from sqlbuild.compiler.compile.exceptions import CompileInputError +from sqlbuild._native import CompileInputError from sqlbuild.compiler.compile.types import CompiledAuditView from tests.integration.src.sqlbuild.compiler.compiled_project._test_types import ( NativeAuditAttachmentTestCase, diff --git a/tests/integration/src/sqlbuild/compiler/compiled_project/test_native_compiled_project.py b/tests/integration/src/sqlbuild/compiler/compiled_project/test_native_compiled_project.py index e20cb23d88..8505f0d92b 100644 --- a/tests/integration/src/sqlbuild/compiler/compiled_project/test_native_compiled_project.py +++ b/tests/integration/src/sqlbuild/compiler/compiled_project/test_native_compiled_project.py @@ -11,13 +11,13 @@ import pytest from sqlbuild._native import ( + CompileInputError, InvalidQualifiedIdentityError, build_compiled_project, discover_project_inputs, ) from sqlbuild.adapters.duckdb.classes.duckdb_adapter import DuckDbAdapter from sqlbuild.cli.entry.main.entry import main -from sqlbuild.compiler.compile.exceptions import CompileInputError from sqlbuild.compiler.compile.models import InferredColumn from sqlbuild.compiler.compile.types import CompiledProjectView from tests.integration.src.sqlbuild.compiler.compiled_project._test_types import ( diff --git a/tests/integration/src/sqlbuild/compiler/compiled_project/test_native_functions.py b/tests/integration/src/sqlbuild/compiler/compiled_project/test_native_functions.py index bd2c76ed63..631b4338db 100644 --- a/tests/integration/src/sqlbuild/compiler/compiled_project/test_native_functions.py +++ b/tests/integration/src/sqlbuild/compiler/compiled_project/test_native_functions.py @@ -7,7 +7,7 @@ import pytest -from sqlbuild.compiler.compile.exceptions import CompileInputError +from sqlbuild._native import CompileInputError from sqlbuild.compiler.compile.types import CompiledFunctionView from tests.integration.src.sqlbuild.compiler.compiled_project._test_types import ( NativeFunctionAttachmentTestCase, diff --git a/tests/integration/src/sqlbuild/compiler/compiled_project/test_native_parameter_boundaries.py b/tests/integration/src/sqlbuild/compiler/compiled_project/test_native_parameter_boundaries.py index 6ecd0837a2..e8dac20c6e 100644 --- a/tests/integration/src/sqlbuild/compiler/compiled_project/test_native_parameter_boundaries.py +++ b/tests/integration/src/sqlbuild/compiler/compiled_project/test_native_parameter_boundaries.py @@ -11,6 +11,7 @@ from sqlbuild._native import ( CollectionRendering, CompileAdapterContext, + SqlHookEntry, build_compile_inputs, discover_project_inputs, ) @@ -18,7 +19,6 @@ from sqlbuild.adapters.bigquery.classes.bigquery_adapter import BigQueryAdapter from sqlbuild.adapters.duckdb.classes.duckdb_adapter import DuckDbAdapter from sqlbuild.adapters.sqlserver.classes.sqlserver_adapter import SqlServerAdapter -from sqlbuild.compiler.discovery.models import SqlHookEntry from tests.integration.src.sqlbuild.compiler.compiled_project._test_types import ( AdapterParameterBoundaryTestCase, QuotedHookBoundaryTestCase, diff --git a/tests/integration/src/sqlbuild/compiler/compiled_project/test_native_sources.py b/tests/integration/src/sqlbuild/compiler/compiled_project/test_native_sources.py index d34587030e..eed81341b3 100644 --- a/tests/integration/src/sqlbuild/compiler/compiled_project/test_native_sources.py +++ b/tests/integration/src/sqlbuild/compiler/compiled_project/test_native_sources.py @@ -6,7 +6,7 @@ import pytest import sqlbuild._native as _native -from sqlbuild.compiler.compile.exceptions import CompileInputError +from sqlbuild._native import CompileInputError from sqlbuild.compiler.compile.types import CompiledProjectView, CompiledSourceView, SourceInput from tests.integration.src.sqlbuild.compiler.compiled_project._test_types import ( NativeSourceAttachmentTestCase, diff --git a/tests/integration/src/sqlbuild/compiler/discovery/_test_types.py b/tests/integration/src/sqlbuild/compiler/discovery/_test_types.py index 95f2f46ee1..3d3e0ba719 100644 --- a/tests/integration/src/sqlbuild/compiler/discovery/_test_types.py +++ b/tests/integration/src/sqlbuild/compiler/discovery/_test_types.py @@ -3,7 +3,7 @@ from collections.abc import Callable from dataclasses import dataclass -from sqlbuild.compiler.discovery.models import DiscoveredSqlModelFile +from sqlbuild._native import DiscoveredSqlModelFile @dataclass(frozen=True) diff --git a/tests/integration/src/sqlbuild/compiler/discovery/helpers.py b/tests/integration/src/sqlbuild/compiler/discovery/helpers.py index 195e1d6238..f6996f4d4b 100644 --- a/tests/integration/src/sqlbuild/compiler/discovery/helpers.py +++ b/tests/integration/src/sqlbuild/compiler/discovery/helpers.py @@ -23,8 +23,13 @@ from sqlbuild._native import ( AuthoredSqlSet, AuthoredSqlValueCall, + DiscoveredProjectInputs, + DiscoveredSourceFile, + DiscoveredSqlModelFile, DiscoveryFileFault, NamedSqlHookEntry, + PythonHookEntry, + SqlHookEntry, TolerantScopeDiscovery, discover_audit_files, discover_constant_files, @@ -37,13 +42,6 @@ discover_sql_hook_files, render_stage_capture, ) -from sqlbuild.compiler.discovery.models import ( - DiscoveredProjectInputs, - DiscoveredSourceFile, - DiscoveredSqlModelFile, - PythonHookEntry, - SqlHookEntry, -) from tests.integration.src.sqlbuild.compiler.discovery._test_types import ( GeneratedDeclarationFileTestCase, ) diff --git a/tests/integration/src/sqlbuild/compiler/discovery/test_native_declaration_files.py b/tests/integration/src/sqlbuild/compiler/discovery/test_native_declaration_files.py index 8e5e73f284..e4902eed90 100644 --- a/tests/integration/src/sqlbuild/compiler/discovery/test_native_declaration_files.py +++ b/tests/integration/src/sqlbuild/compiler/discovery/test_native_declaration_files.py @@ -8,8 +8,7 @@ import pytest from sqlbuild import _native -from sqlbuild._native import DiscoveryFileFault, discover_project_inputs -from sqlbuild.compiler.discovery.models import DiscoveredProjectInputs +from sqlbuild._native import DiscoveredProjectInputs, DiscoveryFileFault, discover_project_inputs from tests.integration.src.sqlbuild.compiler.discovery._test_types import ( DeclarationFilesParityTestCase, DeepDeclarationHeaderTestCase, diff --git a/tests/integration/src/sqlbuild/compiler/discovery/test_native_model_discovery.py b/tests/integration/src/sqlbuild/compiler/discovery/test_native_model_discovery.py index b3bb0494b7..a38c6bb87d 100644 --- a/tests/integration/src/sqlbuild/compiler/discovery/test_native_model_discovery.py +++ b/tests/integration/src/sqlbuild/compiler/discovery/test_native_model_discovery.py @@ -8,11 +8,11 @@ import pytest from sqlbuild._native import ( + DiscoveredSqlModelFile, directory_snapshot_scope, discover_model_files, discover_python_module_files, ) -from sqlbuild.compiler.discovery.models import DiscoveredSqlModelFile from tests.integration.src.sqlbuild.compiler.discovery._test_types import ( NativeRuntimeTestCase, SharedSnapshotTestCase, diff --git a/tests/integration/src/sqlbuild/compiler/discovery/test_native_model_file_record.py b/tests/integration/src/sqlbuild/compiler/discovery/test_native_model_file_record.py index 439ca19d06..1132003328 100644 --- a/tests/integration/src/sqlbuild/compiler/discovery/test_native_model_file_record.py +++ b/tests/integration/src/sqlbuild/compiler/discovery/test_native_model_file_record.py @@ -7,7 +7,7 @@ import pytest -from sqlbuild.compiler.discovery.models import DiscoveredSqlModelFile +from sqlbuild._native import DiscoveredSqlModelFile from tests.integration.src.sqlbuild.compiler.discovery._test_types import RecordCopyTestCase from tests.integration.src.sqlbuild.compiler.discovery.helpers import model_file_record diff --git a/tests/integration/src/sqlbuild/compiler/model_config/helpers.py b/tests/integration/src/sqlbuild/compiler/model_config/helpers.py index 7fc4e4c664..34f74aa3be 100644 --- a/tests/integration/src/sqlbuild/compiler/model_config/helpers.py +++ b/tests/integration/src/sqlbuild/compiler/model_config/helpers.py @@ -14,6 +14,9 @@ import sqlbuild._native as _native from sqlbuild._native import ( CompileAdapterContext, + CompileInputError, + DiscoveredProjectInputs, + DiscoveredSqlModelFile, SchemaModelEntry, SourceLocation, build_compile_inputs, @@ -22,8 +25,6 @@ resolve_effective_collection_rendering, ) from sqlbuild.adapters.duckdb.classes.duckdb_adapter import DuckDbAdapter -from sqlbuild.compiler.compile.exceptions import CompileInputError -from sqlbuild.compiler.discovery.models import DiscoveredProjectInputs, DiscoveredSqlModelFile if TYPE_CHECKING: from _typeshed import DataclassInstance diff --git a/tests/integration/src/sqlbuild/compiler/pipeline/helpers.py b/tests/integration/src/sqlbuild/compiler/pipeline/helpers.py index 18dbd2ce24..9487e17ec9 100644 --- a/tests/integration/src/sqlbuild/compiler/pipeline/helpers.py +++ b/tests/integration/src/sqlbuild/compiler/pipeline/helpers.py @@ -14,6 +14,8 @@ CompileAdapterContext, CompiledLineageColumnFact, CompilePipelineOptions, + CompilePipelineResult, + DiscoveredProjectInputs, build_compile_inputs, build_compiled_project, build_manifest, @@ -26,8 +28,6 @@ from sqlbuild.adapters.duckdb.classes.duckdb_adapter import DuckDbAdapter from sqlbuild.adapters.postgres.classes.postgres_adapter import PostgresAdapter from sqlbuild.compiler.compile.types import CompiledModelView, CompiledProjectView -from sqlbuild.compiler.discovery.models import DiscoveredProjectInputs -from sqlbuild.compiler.pipeline.models import CompilePipelineResult from sqlbuild.runtime.contracts.models import ConnectionHooks if TYPE_CHECKING: diff --git a/tests/integration/src/sqlbuild/compiler/pipeline/test_attached_audit_gates.py b/tests/integration/src/sqlbuild/compiler/pipeline/test_attached_audit_gates.py index 0a3f70c70f..b73a779ad9 100644 --- a/tests/integration/src/sqlbuild/compiler/pipeline/test_attached_audit_gates.py +++ b/tests/integration/src/sqlbuild/compiler/pipeline/test_attached_audit_gates.py @@ -7,8 +7,8 @@ import pytest +from sqlbuild._native import CompilePipelineResult from sqlbuild.adapters.duckdb.classes.duckdb_adapter import DuckDbAdapter -from sqlbuild.compiler.pipeline.models import CompilePipelineResult from tests.integration.src.sqlbuild.compiler.pipeline._test_types import ( SingularAuditAttachmentIntegrationTestCase, ) diff --git a/tests/integration/src/sqlbuild/compiler/pipeline/test_diff_selectors.py b/tests/integration/src/sqlbuild/compiler/pipeline/test_diff_selectors.py index 259e866ba7..2716700155 100644 --- a/tests/integration/src/sqlbuild/compiler/pipeline/test_diff_selectors.py +++ b/tests/integration/src/sqlbuild/compiler/pipeline/test_diff_selectors.py @@ -6,13 +6,13 @@ import pytest from sqlbuild._native import ( + DiscoveredProjectInputs, build_compiled_project, discover_project_inputs, resolve_diff_model_names, ) from sqlbuild.adapters.duckdb.classes.duckdb_adapter import DuckDbAdapter from sqlbuild.compiler.compile.types import CompiledProjectView -from sqlbuild.compiler.discovery.models import DiscoveredProjectInputs from tests.integration.src.sqlbuild.compiler.pipeline._test_types import ( DiffSelectorIntegrationTestCase, ) diff --git a/tests/integration/src/sqlbuild/compiler/pipeline/test_macro_declaration_context.py b/tests/integration/src/sqlbuild/compiler/pipeline/test_macro_declaration_context.py index 3c250b11c2..379ec2345d 100644 --- a/tests/integration/src/sqlbuild/compiler/pipeline/test_macro_declaration_context.py +++ b/tests/integration/src/sqlbuild/compiler/pipeline/test_macro_declaration_context.py @@ -5,14 +5,17 @@ import pytest -from sqlbuild._native import build_compiled_project, discover_project_inputs +from sqlbuild._native import ( + CompileInputError, + CompilePipelineResult, + DiscoveredProjectInputs, + build_compiled_project, + discover_project_inputs, +) from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter from sqlbuild.adapters.duckdb.classes.duckdb_adapter import DuckDbAdapter from sqlbuild.adapters.snowflake.classes.snowflake_adapter import SnowflakeAdapter -from sqlbuild.compiler.compile.exceptions import CompileInputError from sqlbuild.compiler.compile.types import CompiledProjectView -from sqlbuild.compiler.discovery.models import DiscoveredProjectInputs -from sqlbuild.compiler.pipeline.models import CompilePipelineResult from tests.integration.src.sqlbuild.compiler.pipeline._test_types import ( GroupedDeclarationCompileTestCase, MacroDeclarationContextErrorTestCase, diff --git a/tests/integration/src/sqlbuild/compiler/pipeline/test_macro_load_count.py b/tests/integration/src/sqlbuild/compiler/pipeline/test_macro_load_count.py index 19136be0d5..3484fefca2 100644 --- a/tests/integration/src/sqlbuild/compiler/pipeline/test_macro_load_count.py +++ b/tests/integration/src/sqlbuild/compiler/pipeline/test_macro_load_count.py @@ -6,9 +6,8 @@ import pytest -from sqlbuild._native import DeclarationRecord +from sqlbuild._native import CompilePipelineResult, DeclarationRecord from sqlbuild.adapters.duckdb.classes.duckdb_adapter import DuckDbAdapter -from sqlbuild.compiler.pipeline.models import CompilePipelineResult from tests.integration.src.sqlbuild.compiler.pipeline._test_types import ( MacroCompositionIntegrationTestCase, MacroLoadCountIntegrationTestCase, diff --git a/tests/integration/src/sqlbuild/compiler/pipeline/test_main.py b/tests/integration/src/sqlbuild/compiler/pipeline/test_main.py index cfe37ac6eb..ee4de4919f 100644 --- a/tests/integration/src/sqlbuild/compiler/pipeline/test_main.py +++ b/tests/integration/src/sqlbuild/compiler/pipeline/test_main.py @@ -8,7 +8,10 @@ import pytest from sqlbuild._native import ( + CompilePipelineResult, + DiscoveredProjectInputs, ModelPlanEntry, + RulesError, build_compiled_project, build_project_graph, discover_project_inputs, @@ -18,10 +21,7 @@ from sqlbuild.adapters.snowflake.classes.snowflake_adapter import SnowflakeAdapter from sqlbuild.compiler.compile.models import InferredColumn from sqlbuild.compiler.compile.types import CompiledProjectView -from sqlbuild.compiler.discovery.models import DiscoveredProjectInputs from sqlbuild.compiler.lineage.types import InferredNullability -from sqlbuild.compiler.pipeline.models import CompilePipelineResult -from sqlbuild.rule_engine.exceptions import RulesError from tests.integration.src.sqlbuild.compiler.pipeline._test_types import ( AppendCursorPipelineIntegrationTestCase, CompileProgressIntegrationTestCase, diff --git a/tests/integration/src/sqlbuild/compiler/pipeline/test_measurement_audits.py b/tests/integration/src/sqlbuild/compiler/pipeline/test_measurement_audits.py index 847620ca96..3881d06fef 100644 --- a/tests/integration/src/sqlbuild/compiler/pipeline/test_measurement_audits.py +++ b/tests/integration/src/sqlbuild/compiler/pipeline/test_measurement_audits.py @@ -5,11 +5,10 @@ import pytest -from sqlbuild._native import AuditPlanEntry +from sqlbuild._native import AuditPlanEntry, CompilePipelineResult from sqlbuild.adapters.duckdb.classes.duckdb_adapter import DuckDbAdapter from sqlbuild.compiler.auditing.types import AuditEvaluationMode, ThresholdOperator from sqlbuild.compiler.compile.types import CompiledAuditView -from sqlbuild.compiler.pipeline.models import CompilePipelineResult from tests.integration.src.sqlbuild.compiler.pipeline._test_types import ( MeasurementCompileErrorIntegrationTestCase, MeasurementCompileIntegrationTestCase, diff --git a/tests/integration/src/sqlbuild/compiler/pipeline/test_python_source_reads.py b/tests/integration/src/sqlbuild/compiler/pipeline/test_python_source_reads.py index bc1e2f795d..ff5ca6cf19 100644 --- a/tests/integration/src/sqlbuild/compiler/pipeline/test_python_source_reads.py +++ b/tests/integration/src/sqlbuild/compiler/pipeline/test_python_source_reads.py @@ -7,9 +7,8 @@ import pytest -from sqlbuild._native import SourceEntry +from sqlbuild._native import CompilePipelineResult, SourceEntry from sqlbuild.adapters.duckdb.classes.duckdb_adapter import DuckDbAdapter -from sqlbuild.compiler.pipeline.models import CompilePipelineResult from tests.integration.src.sqlbuild.compiler.pipeline._test_types import ( PythonSourceReadPlanTestCase, ) diff --git a/tests/integration/src/sqlbuild/compiler/pipeline/test_selection_lineage.py b/tests/integration/src/sqlbuild/compiler/pipeline/test_selection_lineage.py index 9f6331740c..2d2b1984d1 100644 --- a/tests/integration/src/sqlbuild/compiler/pipeline/test_selection_lineage.py +++ b/tests/integration/src/sqlbuild/compiler/pipeline/test_selection_lineage.py @@ -5,8 +5,8 @@ import pytest +from sqlbuild._native import CompilePipelineResult from sqlbuild.adapters.duckdb.classes.duckdb_adapter import DuckDbAdapter -from sqlbuild.compiler.pipeline.models import CompilePipelineResult from tests.integration.src.sqlbuild.compiler.pipeline._test_types import ( SelectionLineageIntegrationTestCase, ) diff --git a/tests/integration/src/sqlbuild/compiler/planner/_helpers/helpers.py b/tests/integration/src/sqlbuild/compiler/planner/_helpers/helpers.py index ad82b737b5..8014e137ce 100644 --- a/tests/integration/src/sqlbuild/compiler/planner/_helpers/helpers.py +++ b/tests/integration/src/sqlbuild/compiler/planner/_helpers/helpers.py @@ -15,10 +15,12 @@ CompileModelConfig, DiscoveredSchemaFile, DiscoveredSeedFile, + DiscoveredSourceFile, SchemaColumn, SchemaModelEntry, SchemaSeedEntry, SourceEntry, + SqlReferenceKind, WarehouseSnapshot, compiled_model_record, gather_warehouse_snapshot, @@ -41,8 +43,6 @@ CompiledSeedView, CompiledSourceView, ) -from sqlbuild.compiler.discovery.models import DiscoveredSourceFile -from sqlbuild.compiler.references.types import SqlReferenceKind class RecordingDuckDbAdapter(DuckDbAdapter): diff --git a/tests/integration/src/sqlbuild/compiler/planner/_helpers/resolve/helpers.py b/tests/integration/src/sqlbuild/compiler/planner/_helpers/resolve/helpers.py index ed7e7658fb..bfccee908b 100644 --- a/tests/integration/src/sqlbuild/compiler/planner/_helpers/resolve/helpers.py +++ b/tests/integration/src/sqlbuild/compiler/planner/_helpers/resolve/helpers.py @@ -16,6 +16,7 @@ ModelCursorSnapshot, ModelPlanContext, SourceEntry, + SqlReferenceKind, WarehouseSnapshot, assemble_project, build_compile_inputs, @@ -35,7 +36,6 @@ CompiledProjectView, CompiledResourceType, ) -from sqlbuild.compiler.references.types import SqlReferenceKind class OrchestrationOverrideAdapter(DuckDbAdapter): diff --git a/tests/integration/src/sqlbuild/compiler/planner/_helpers/resolve/test_orchestration.py b/tests/integration/src/sqlbuild/compiler/planner/_helpers/resolve/test_orchestration.py index ef12ba2995..6d9d293942 100644 --- a/tests/integration/src/sqlbuild/compiler/planner/_helpers/resolve/test_orchestration.py +++ b/tests/integration/src/sqlbuild/compiler/planner/_helpers/resolve/test_orchestration.py @@ -11,6 +11,7 @@ BackfillResult, CursorOverridePair, ModelPlanContext, + SqlHookEntry, WarehouseSnapshot, build_function_locations, build_model_locations, @@ -20,7 +21,6 @@ resolve_model_sql, ) from sqlbuild.compiler.compile.types import CompiledModelView, CompiledProjectView -from sqlbuild.compiler.discovery.models import SqlHookEntry from tests.integration.src.sqlbuild.compiler.planner._helpers.resolve._test_types import ( OrchestrationTestCase, ) diff --git a/tests/integration/src/sqlbuild/compiler/planner/_helpers/sql_test_assembly/helpers.py b/tests/integration/src/sqlbuild/compiler/planner/_helpers/sql_test_assembly/helpers.py index 3b7cdc52ba..61687bce9d 100644 --- a/tests/integration/src/sqlbuild/compiler/planner/_helpers/sql_test_assembly/helpers.py +++ b/tests/integration/src/sqlbuild/compiler/planner/_helpers/sql_test_assembly/helpers.py @@ -9,6 +9,8 @@ CompileModelConfig, CompileSqlTestCte, DiscoveredSqlTestBlock, + DiscoveredSqlTestFile, + SqlReferenceKind, ) from sqlbuild.compiler.compile.models import ( CompiledModel, @@ -22,8 +24,6 @@ CompiledProjectView, CompiledResourceType, ) -from sqlbuild.compiler.discovery.models import DiscoveredSqlTestFile -from sqlbuild.compiler.references.types import SqlReferenceKind from tests.integration.src.sqlbuild.compiler.planner._helpers.sql_test_assembly._test_types import ( ExecuteChainTestCase, ) diff --git a/tests/integration/src/sqlbuild/compiler/planner/_helpers/sql_test_assembly/test_execute_chain.py b/tests/integration/src/sqlbuild/compiler/planner/_helpers/sql_test_assembly/test_execute_chain.py index 86b5f22be6..870d4dbdd2 100644 --- a/tests/integration/src/sqlbuild/compiler/planner/_helpers/sql_test_assembly/test_execute_chain.py +++ b/tests/integration/src/sqlbuild/compiler/planner/_helpers/sql_test_assembly/test_execute_chain.py @@ -11,6 +11,7 @@ CompiledDirectLogicSqlTestPayload, CompileSqlTestCte, DiscoveredSqlTestBlock, + DiscoveredSqlTestFile, SqlTestPlanEntry, build_compiled_project, discover_project_inputs, @@ -18,7 +19,6 @@ from sqlbuild.adapters.duckdb.classes.duckdb_adapter import DuckDbAdapter from sqlbuild.compiler.compile.models import CompiledObjectKey, CompiledProject, CompiledSqlTest from sqlbuild.compiler.compile.types import CompiledProjectView, CompiledResourceType, SqlTestMode -from sqlbuild.compiler.discovery.models import DiscoveredSqlTestFile from sqlbuild.executor.testing.main._execute import execute_sql_test from sqlbuild.executor.testing.models import SqlTestExecutionResult from sqlbuild.executor.testing.types import SqlTestOutcome diff --git a/tests/integration/src/sqlbuild/compiler/planner/_helpers/test_fixture_planning.py b/tests/integration/src/sqlbuild/compiler/planner/_helpers/test_fixture_planning.py index ee30f642a9..6cf18838c5 100644 --- a/tests/integration/src/sqlbuild/compiler/planner/_helpers/test_fixture_planning.py +++ b/tests/integration/src/sqlbuild/compiler/planner/_helpers/test_fixture_planning.py @@ -6,6 +6,7 @@ import pytest from sqlbuild._native import ( + DiscoveredProjectInputs, RelationFixtureCompletion, RelationFixturePlanningContext, SqlTestPlanEntry, @@ -18,7 +19,6 @@ ) from sqlbuild.adapters.duckdb.classes.duckdb_adapter import DuckDbAdapter from sqlbuild.compiler.compile.types import CompiledProjectView, CompiledResourceType -from sqlbuild.compiler.discovery.models import DiscoveredProjectInputs from tests.integration.src.sqlbuild.compiler.planner._helpers._test_types import ( FixturePlanningIntegrationTestCase, FixtureSchemaIntegrationTestCase, diff --git a/tests/integration/src/sqlbuild/compiler/planner/main/helpers.py b/tests/integration/src/sqlbuild/compiler/planner/main/helpers.py index ca2f0c1946..2e22794e93 100644 --- a/tests/integration/src/sqlbuild/compiler/planner/main/helpers.py +++ b/tests/integration/src/sqlbuild/compiler/planner/main/helpers.py @@ -18,9 +18,12 @@ CompiledSource, CompileModelConfig, CompilePipelineOptions, + CompilePipelineResult, DeferralInputs, + DiscoveredProjectInputs, DiscoveredSchemaFile, DiscoveredSeedFile, + DiscoveredSourceFile, Fingerprint, FunctionArgument, PlannerOverrides, @@ -30,6 +33,7 @@ SchemaSeedEntry, SettingsConfig, SourceEntry, + SqlReferenceKind, build_compiled_function_fingerprint_sql, build_execution_plan, compiled_model_record, @@ -57,9 +61,6 @@ CompiledSourceView, FunctionLanguage, ) -from sqlbuild.compiler.discovery.models import DiscoveredProjectInputs, DiscoveredSourceFile -from sqlbuild.compiler.pipeline.models import CompilePipelineResult -from sqlbuild.compiler.references.types import SqlReferenceKind from sqlbuild.runtime.contracts.models import ConnectionHooks from tests.integration.src.sqlbuild.compiler.planner.main._test_types import ( BuildExecutionPlanTestCase, diff --git a/tests/integration/src/sqlbuild/compiler/planner/main/test_snowflake_inspection_budget.py b/tests/integration/src/sqlbuild/compiler/planner/main/test_snowflake_inspection_budget.py index 3494144aac..67ce789fab 100644 --- a/tests/integration/src/sqlbuild/compiler/planner/main/test_snowflake_inspection_budget.py +++ b/tests/integration/src/sqlbuild/compiler/planner/main/test_snowflake_inspection_budget.py @@ -10,10 +10,9 @@ import pytest -from sqlbuild._native import PlanReason +from sqlbuild._native import CompilePipelineResult, PlanReason from sqlbuild.adapter.contract.exceptions import AdapterUserError from sqlbuild.adapter.relations.constants import INSPECTION_IN_LIST_LIMIT -from sqlbuild.compiler.pipeline.models import CompilePipelineResult from tests.integration.src.sqlbuild.compiler.planner.main._test_types import ( SnowflakeCursorBoundsBudgetTestCase, SnowflakeFreshTargetTestCase, diff --git a/tests/integration/src/sqlbuild/compiler/planner/main/test_source_table_probes.py b/tests/integration/src/sqlbuild/compiler/planner/main/test_source_table_probes.py index 3ac70e8357..9af062cc0b 100644 --- a/tests/integration/src/sqlbuild/compiler/planner/main/test_source_table_probes.py +++ b/tests/integration/src/sqlbuild/compiler/planner/main/test_source_table_probes.py @@ -7,7 +7,7 @@ import pytest -from sqlbuild.compiler.pipeline.models import CompilePipelineResult +from sqlbuild._native import CompilePipelineResult from tests.integration.src.sqlbuild.compiler.planner.main._test_types import ( SourceTableProbeBudgetTestCase, ) diff --git a/tests/integration/src/sqlbuild/compiler/project_assembly/test_native_project_assembly.py b/tests/integration/src/sqlbuild/compiler/project_assembly/test_native_project_assembly.py index e3bc480ef9..cbaa13dc99 100644 --- a/tests/integration/src/sqlbuild/compiler/project_assembly/test_native_project_assembly.py +++ b/tests/integration/src/sqlbuild/compiler/project_assembly/test_native_project_assembly.py @@ -11,9 +11,8 @@ import pytest import sqlbuild._native as native_module -from sqlbuild._native import assemble_project +from sqlbuild._native import CompileInputError, assemble_project from sqlbuild.adapter.contract.models import ExpressionInferenceProfile -from sqlbuild.compiler.compile.exceptions import CompileInputError from sqlbuild.compiler.compile.types import CompiledProjectView from tests.integration.src.sqlbuild.compiler.helpers import ( GoldenEntry, diff --git a/tests/integration/src/sqlbuild/compiler/scopes/helpers.py b/tests/integration/src/sqlbuild/compiler/scopes/helpers.py index 09cb40be63..3560a1468d 100644 --- a/tests/integration/src/sqlbuild/compiler/scopes/helpers.py +++ b/tests/integration/src/sqlbuild/compiler/scopes/helpers.py @@ -14,13 +14,17 @@ import sqlbuild._native as _native from sqlbuild._native import ( + CompileInputError, + ConstantDeclaration, DeclarationIdentity, DeclarationKind, DeclarationRecord, DeclarationScopeBuild, DeclarationScopeResolver, DeclarationVisibility, + DiscoveredProjectInputs, DiscoveryError, + EnumDeclaration, ResourceIdentity, ResourceKind, ScopeIndex, @@ -37,13 +41,7 @@ ) from sqlbuild.adapters.duckdb.classes.duckdb_adapter import DuckDbAdapter from sqlbuild.compiler.compile._helpers.render.user_macros import load_project_macros -from sqlbuild.compiler.compile.exceptions import CompileInputError from sqlbuild.compiler.compile.models import DeclarationResolutionContext, LoadedMacro -from sqlbuild.compiler.discovery.models import ( - ConstantDeclaration, - DiscoveredProjectInputs, - EnumDeclaration, -) from tests.integration.src.sqlbuild.compiler.helpers import GoldenEntry, golden_entry CURRENT_PATH_COMPONENT: str = "." diff --git a/tests/integration/src/sqlbuild/executor/build/concurrent/helpers.py b/tests/integration/src/sqlbuild/executor/build/concurrent/helpers.py index 5d706ab5c5..63bd1a4673 100644 --- a/tests/integration/src/sqlbuild/executor/build/concurrent/helpers.py +++ b/tests/integration/src/sqlbuild/executor/build/concurrent/helpers.py @@ -9,14 +9,14 @@ from sqlbuild._native import ( CompilePipelineOptions, + CompilePipelineResult, + DiscoveredProjectInputs, PlanOutput, discover_project_inputs, run_compile_pipeline, ) from sqlbuild.adapter.contract.types import TablePromotionMode from sqlbuild.adapters.duckdb.classes.duckdb_adapter import DuckDbAdapter -from sqlbuild.compiler.discovery.models import DiscoveredProjectInputs -from sqlbuild.compiler.pipeline.models import CompilePipelineResult from sqlbuild.executor.build.main._execute import execute_build_plan from sqlbuild.executor.build.models import ( BuildCustomizations, diff --git a/tests/integration/src/sqlbuild/executor/build/concurrent/test_ordering_invariant.py b/tests/integration/src/sqlbuild/executor/build/concurrent/test_ordering_invariant.py index e757c9d29f..a29f568635 100644 --- a/tests/integration/src/sqlbuild/executor/build/concurrent/test_ordering_invariant.py +++ b/tests/integration/src/sqlbuild/executor/build/concurrent/test_ordering_invariant.py @@ -11,14 +11,14 @@ from sqlbuild._native import ( CompilePipelineOptions, + CompilePipelineResult, + DiscoveredProjectInputs, PlanOutput, discover_project_inputs, run_compile_pipeline, ) from sqlbuild.adapter.contract.types import TablePromotionMode from sqlbuild.adapters.duckdb.classes.duckdb_adapter import DuckDbAdapter -from sqlbuild.compiler.discovery.models import DiscoveredProjectInputs -from sqlbuild.compiler.pipeline.models import CompilePipelineResult from sqlbuild.executor.build.main._execute import execute_build_plan from sqlbuild.executor.build.models import ( BuildCallbacks, diff --git a/tests/integration/src/sqlbuild/executor/build/custom/helpers.py b/tests/integration/src/sqlbuild/executor/build/custom/helpers.py index be4b320c00..7b6b004f0e 100644 --- a/tests/integration/src/sqlbuild/executor/build/custom/helpers.py +++ b/tests/integration/src/sqlbuild/executor/build/custom/helpers.py @@ -8,6 +8,7 @@ from sqlbuild._native import ( AuditPlanEntry, + DiscoveredHookFunction, MaterializationType, ModelPlanEntry, PlanAction, @@ -25,7 +26,6 @@ CompiledRelationLocation, ) from sqlbuild.compiler.compile.types import CompiledResourceType -from sqlbuild.compiler.discovery.models import DiscoveredHookFunction from sqlbuild.executor.auditing.models import AuditExecutionResult from sqlbuild.executor.custom.models import ( MaterializationContext, @@ -432,13 +432,13 @@ def run_scheduler_build( from sqlbuild._native import ( CompilePipelineOptions, + CompilePipelineResult, + DiscoveredProjectInputs, PlanOutput, discover_project_inputs, run_compile_pipeline, ) from sqlbuild.adapter.contract.types import TablePromotionMode - from sqlbuild.compiler.discovery.models import DiscoveredProjectInputs - from sqlbuild.compiler.pipeline.models import CompilePipelineResult from sqlbuild.executor.build.main._execute import execute_build_plan from sqlbuild.executor.build.models import ( BuildCustomizations, diff --git a/tests/integration/src/sqlbuild/executor/build/custom/test_custom_materialization.py b/tests/integration/src/sqlbuild/executor/build/custom/test_custom_materialization.py index 6717bfb894..c88578b9e0 100644 --- a/tests/integration/src/sqlbuild/executor/build/custom/test_custom_materialization.py +++ b/tests/integration/src/sqlbuild/executor/build/custom/test_custom_materialization.py @@ -9,11 +9,17 @@ import duckdb import pytest -from sqlbuild._native import AuditPlanEntry, ModelPlanEntry, PlanReason +from sqlbuild._native import ( + AuditPlanEntry, + DiscoveredHookFunction, + ModelPlanEntry, + PlanReason, + PythonHookEntry, + SqlHookEntry, +) from sqlbuild.adapters.duckdb.classes.duckdb_adapter import DuckDbAdapter from sqlbuild.compiler.auditing.types import AuditOutcome from sqlbuild.compiler.compile.models import CompiledRelationLocation -from sqlbuild.compiler.discovery.models import DiscoveredHookFunction, PythonHookEntry, SqlHookEntry from sqlbuild.executor.custom.models import ( MaterializationContext, MaterializationResult, diff --git a/tests/integration/src/sqlbuild/executor/build/helpers.py b/tests/integration/src/sqlbuild/executor/build/helpers.py index 8e6cae918d..3f84e5983c 100644 --- a/tests/integration/src/sqlbuild/executor/build/helpers.py +++ b/tests/integration/src/sqlbuild/executor/build/helpers.py @@ -12,6 +12,8 @@ from sqlbuild._native import ( CompilePipelineOptions, + CompilePipelineResult, + DiscoveredProjectInputs, MicrobatchLimitAction, PlanOutput, TableTypePlanEntry, @@ -21,8 +23,6 @@ from sqlbuild._native import run_cli as main from sqlbuild.adapter.contract.types import TablePromotionMode from sqlbuild.adapters.duckdb.classes.duckdb_adapter import DuckDbAdapter -from sqlbuild.compiler.discovery.models import DiscoveredProjectInputs -from sqlbuild.compiler.pipeline.models import CompilePipelineResult from sqlbuild.executor.build.main._execute import execute_build_plan from sqlbuild.executor.build.models import ( BuildExecutionResult, diff --git a/tests/integration/src/sqlbuild/executor/build/snowflake_metadata/helpers.py b/tests/integration/src/sqlbuild/executor/build/snowflake_metadata/helpers.py index 2f89e19bdc..cdce00d9fa 100644 --- a/tests/integration/src/sqlbuild/executor/build/snowflake_metadata/helpers.py +++ b/tests/integration/src/sqlbuild/executor/build/snowflake_metadata/helpers.py @@ -13,10 +13,14 @@ from pathlib import Path from typing import Any -from sqlbuild._native import CompilePipelineOptions, discover_project_inputs, run_compile_pipeline +from sqlbuild._native import ( + CompilePipelineOptions, + CompilePipelineResult, + DiscoveredProjectInputs, + discover_project_inputs, + run_compile_pipeline, +) from sqlbuild.adapter.contract.types import TablePromotionMode -from sqlbuild.compiler.discovery.models import DiscoveredProjectInputs -from sqlbuild.compiler.pipeline.models import CompilePipelineResult from sqlbuild.executor.build.models import BuildExecutionResult, BuildRuntimeParams from sqlbuild.executor.pipeline.main.run import run_build_pipeline from sqlbuild.executor.scheduling.types import ExecutionStatus diff --git a/tests/integration/src/sqlbuild/executor/build/test_capped_microbatch_dependencies.py b/tests/integration/src/sqlbuild/executor/build/test_capped_microbatch_dependencies.py index a0e4779470..984c8be714 100644 --- a/tests/integration/src/sqlbuild/executor/build/test_capped_microbatch_dependencies.py +++ b/tests/integration/src/sqlbuild/executor/build/test_capped_microbatch_dependencies.py @@ -4,10 +4,8 @@ import pytest -from sqlbuild._native import MicrobatchLimitAction +from sqlbuild._native import CompileInputError, CompilePipelineResult, MicrobatchLimitAction from sqlbuild.adapters.duckdb.classes.duckdb_adapter import DuckDbAdapter -from sqlbuild.compiler.compile.exceptions import CompileInputError -from sqlbuild.compiler.pipeline.models import CompilePipelineResult from tests.integration.src.sqlbuild.executor.build._test_types import ( CappedFilterConsumerTestCase, CappedIntermediateFilterTestCase, diff --git a/tests/integration/src/sqlbuild/executor/build/test_snapshot_execution.py b/tests/integration/src/sqlbuild/executor/build/test_snapshot_execution.py index c3924f4fbe..ab6df50036 100644 --- a/tests/integration/src/sqlbuild/executor/build/test_snapshot_execution.py +++ b/tests/integration/src/sqlbuild/executor/build/test_snapshot_execution.py @@ -10,14 +10,14 @@ from sqlbuild._native import ( CompilePipelineOptions, + CompilePipelineResult, + DiscoveredProjectInputs, ModelPlanEntry, PlanOutput, discover_project_inputs, run_compile_pipeline, ) from sqlbuild.adapters.duckdb.classes.duckdb_adapter import DuckDbAdapter -from sqlbuild.compiler.discovery.models import DiscoveredProjectInputs -from sqlbuild.compiler.pipeline.models import CompilePipelineResult from sqlbuild.executor.build.models import BuildExecutionResult from sqlbuild.executor.build.types import BuildStatus from sqlbuild.executor.scheduling.types import ExecutionStatus diff --git a/tests/integration/src/sqlbuild/executor/run/helpers.py b/tests/integration/src/sqlbuild/executor/run/helpers.py index ebb4d9253f..c1ce088ee5 100644 --- a/tests/integration/src/sqlbuild/executor/run/helpers.py +++ b/tests/integration/src/sqlbuild/executor/run/helpers.py @@ -9,6 +9,7 @@ from sqlbuild._native import ( AuditPlanEntry, + DiscoveredHookFunction, MaterializationType, ModelPlanEntry, PlanAction, @@ -26,7 +27,6 @@ CompiledRelationLocation, ) from sqlbuild.compiler.compile.types import CompiledResourceType -from sqlbuild.compiler.discovery.models import DiscoveredHookFunction from sqlbuild.executor.run.main._execute import execute_table_entry from sqlbuild.executor.run.models import ( HookContext, diff --git a/tests/integration/src/sqlbuild/executor/run/incremental/helpers.py b/tests/integration/src/sqlbuild/executor/run/incremental/helpers.py index 3e1ab53b3a..8c1d50fae5 100644 --- a/tests/integration/src/sqlbuild/executor/run/incremental/helpers.py +++ b/tests/integration/src/sqlbuild/executor/run/incremental/helpers.py @@ -11,6 +11,7 @@ AuditPlanEntry, CursorBounds, CursorInputRelation, + DiscoveredHookFunction, IncrementalStrategy, MaterializationType, ModelPlanEntry, @@ -30,7 +31,6 @@ CompiledRelationLocation, ) from sqlbuild.compiler.compile.types import CompiledResourceType -from sqlbuild.compiler.discovery.models import DiscoveredHookFunction from sqlbuild.executor.run._helpers.materializations.incremental import execute_incremental_entry from sqlbuild.executor.run.models import ( HookContext, diff --git a/tests/integration/src/sqlbuild/executor/run/incremental/test_incremental_execution.py b/tests/integration/src/sqlbuild/executor/run/incremental/test_incremental_execution.py index c8c150a114..d97ae0f5d8 100644 --- a/tests/integration/src/sqlbuild/executor/run/incremental/test_incremental_execution.py +++ b/tests/integration/src/sqlbuild/executor/run/incremental/test_incremental_execution.py @@ -8,10 +8,16 @@ import pytest -from sqlbuild._native import FutureCursorAction, FutureCursorsConfig, OnSchemaChange +from sqlbuild._native import ( + DiscoveredHookFunction, + FutureCursorAction, + FutureCursorsConfig, + OnSchemaChange, + PythonHookEntry, + SqlHookEntry, +) from sqlbuild.adapters.duckdb.classes.duckdb_adapter import DuckDbAdapter from sqlbuild.compiler.auditing.types import AuditRunScope -from sqlbuild.compiler.discovery.models import DiscoveredHookFunction, PythonHookEntry, SqlHookEntry from sqlbuild.compiler.planner.constants import MICROBATCH_END_SENTINEL, MICROBATCH_START_SENTINEL from sqlbuild.executor.run.models import ModelExecutionResult from sqlbuild.executor.run.types import ExecutionPhase diff --git a/tests/integration/src/sqlbuild/executor/run/microbatch/helpers.py b/tests/integration/src/sqlbuild/executor/run/microbatch/helpers.py index 22327065b0..067260ee38 100644 --- a/tests/integration/src/sqlbuild/executor/run/microbatch/helpers.py +++ b/tests/integration/src/sqlbuild/executor/run/microbatch/helpers.py @@ -11,6 +11,7 @@ AuditPlanEntry, CursorBounds, CursorInputRelation, + DiscoveredHookFunction, IncrementalMode, IncrementalStrategy, MaterializationType, @@ -28,7 +29,6 @@ CompiledRelationLocation, ) from sqlbuild.compiler.compile.types import CompiledResourceType -from sqlbuild.compiler.discovery.models import DiscoveredHookFunction from sqlbuild.compiler.planner.constants import ( MICROBATCH_END_SENTINEL, MICROBATCH_START_SENTINEL, diff --git a/tests/integration/src/sqlbuild/executor/run/microbatch/test_microbatch_execution.py b/tests/integration/src/sqlbuild/executor/run/microbatch/test_microbatch_execution.py index f98a5c0f54..2ff0714585 100644 --- a/tests/integration/src/sqlbuild/executor/run/microbatch/test_microbatch_execution.py +++ b/tests/integration/src/sqlbuild/executor/run/microbatch/test_microbatch_execution.py @@ -11,14 +11,16 @@ import pytest from sqlbuild._native import ( + DiscoveredHookFunction, FutureCursorAction, FutureCursorsConfig, MicrobatchLimitAction, OnSchemaChange, + PythonHookEntry, + SqlHookEntry, ) from sqlbuild.adapters.duckdb.classes.duckdb_adapter import DuckDbAdapter from sqlbuild.compiler.auditing.types import AuditRunScope -from sqlbuild.compiler.discovery.models import DiscoveredHookFunction, PythonHookEntry, SqlHookEntry from sqlbuild.compiler.planner.constants import ( MICROBATCH_END_SENTINEL, MICROBATCH_START_SENTINEL, diff --git a/tests/integration/src/sqlbuild/executor/run/test_table_execution.py b/tests/integration/src/sqlbuild/executor/run/test_table_execution.py index 876b861819..691687feb2 100644 --- a/tests/integration/src/sqlbuild/executor/run/test_table_execution.py +++ b/tests/integration/src/sqlbuild/executor/run/test_table_execution.py @@ -7,9 +7,9 @@ import pytest +from sqlbuild._native import DiscoveredHookFunction, PythonHookEntry, SqlHookEntry from sqlbuild.adapter.contract.types import TablePromotionMode from sqlbuild.adapters.duckdb.classes.duckdb_adapter import DuckDbAdapter -from sqlbuild.compiler.discovery.models import DiscoveredHookFunction, PythonHookEntry, SqlHookEntry from sqlbuild.executor.run.models import ModelExecutionResult from sqlbuild.executor.run.types import ExecutionPhase from tests.integration.src.sqlbuild.executor.run._test_types import ( diff --git a/tests/integration/src/sqlbuild/executor/run/view/helpers.py b/tests/integration/src/sqlbuild/executor/run/view/helpers.py index d5fb8b989b..703de1354c 100644 --- a/tests/integration/src/sqlbuild/executor/run/view/helpers.py +++ b/tests/integration/src/sqlbuild/executor/run/view/helpers.py @@ -8,6 +8,7 @@ from sqlbuild._native import ( AuditPlanEntry, + DiscoveredHookFunction, MaterializationType, ModelPlanEntry, PlanAction, @@ -24,7 +25,6 @@ CompiledRelationLocation, ) from sqlbuild.compiler.compile.types import CompiledResourceType -from sqlbuild.compiler.discovery.models import DiscoveredHookFunction from sqlbuild.executor.run._helpers.materializations.view import execute_view_entry from sqlbuild.executor.run.models import ( HookContext, diff --git a/tests/integration/src/sqlbuild/executor/run/view/test_view_execution.py b/tests/integration/src/sqlbuild/executor/run/view/test_view_execution.py index ae50c38012..20f5475d59 100644 --- a/tests/integration/src/sqlbuild/executor/run/view/test_view_execution.py +++ b/tests/integration/src/sqlbuild/executor/run/view/test_view_execution.py @@ -7,8 +7,8 @@ import pytest +from sqlbuild._native import DiscoveredHookFunction, PythonHookEntry, SqlHookEntry from sqlbuild.adapters.duckdb.classes.duckdb_adapter import DuckDbAdapter -from sqlbuild.compiler.discovery.models import DiscoveredHookFunction, PythonHookEntry, SqlHookEntry from sqlbuild.executor.run.models import ModelExecutionResult from sqlbuild.executor.run.types import ExecutionPhase from tests.integration.src.sqlbuild.executor.run.view._test_types import ( diff --git a/tests/integration/src/sqlbuild/executor/scenario/helpers.py b/tests/integration/src/sqlbuild/executor/scenario/helpers.py index ffcb8e18f1..3e8aae2e24 100644 --- a/tests/integration/src/sqlbuild/executor/scenario/helpers.py +++ b/tests/integration/src/sqlbuild/executor/scenario/helpers.py @@ -4,6 +4,7 @@ from typing import Any from sqlbuild._native import ( + DiscoveredHookFunction, MaterializationType, ModelPlanEntry, PlanAction, @@ -22,7 +23,6 @@ CompiledRelationLocation, ) from sqlbuild.compiler.compile.types import CompiledResourceType -from sqlbuild.compiler.discovery.models import DiscoveredHookFunction from sqlbuild.executor.run.models import HookContext SCENARIO_NAME: str = "revenue__customer_refund" diff --git a/tests/integration/src/sqlbuild/executor/scenario/test_fixture_lifecycle.py b/tests/integration/src/sqlbuild/executor/scenario/test_fixture_lifecycle.py index d4aa3492c7..07fe247849 100644 --- a/tests/integration/src/sqlbuild/executor/scenario/test_fixture_lifecycle.py +++ b/tests/integration/src/sqlbuild/executor/scenario/test_fixture_lifecycle.py @@ -5,7 +5,13 @@ import pytest -from sqlbuild._native import ScenarioExecutionPlan, ScenarioFixturePlan, SeedPlanEntry +from sqlbuild._native import ( + DiscoveredHookFunction, + PythonHookEntry, + ScenarioExecutionPlan, + ScenarioFixturePlan, + SeedPlanEntry, +) from sqlbuild.adapter.contract.models import ColumnInfo from sqlbuild.adapter.contract.types import TablePromotionMode from sqlbuild.adapters.duckdb.classes.duckdb_adapter import DuckDbAdapter @@ -14,7 +20,6 @@ CompiledRelationLocation, ) from sqlbuild.compiler.compile.types import CompiledResourceType -from sqlbuild.compiler.discovery.models import DiscoveredHookFunction, PythonHookEntry from sqlbuild.executor.build.models import SeedExecutionResult from sqlbuild.executor.run.models import ModelExecutionResult from sqlbuild.executor.scenario._helpers.execution.model_execution import execute_scenario_models diff --git a/tests/integration/src/sqlbuild/executor/testing/helpers.py b/tests/integration/src/sqlbuild/executor/testing/helpers.py index ba5c2de129..86553af302 100644 --- a/tests/integration/src/sqlbuild/executor/testing/helpers.py +++ b/tests/integration/src/sqlbuild/executor/testing/helpers.py @@ -6,11 +6,15 @@ from pathlib import Path from typing import Any -from sqlbuild._native import ChainStep, SqlTestPlanEntry, render_sql_test_comparison_sql +from sqlbuild._native import ( + ChainStep, + CompilePipelineResult, + SqlTestPlanEntry, + render_sql_test_comparison_sql, +) from sqlbuild.adapters.duckdb.classes.duckdb_adapter import DuckDbAdapter from sqlbuild.compiler.compile.models import CompiledObjectKey from sqlbuild.compiler.compile.types import CompiledResourceType -from sqlbuild.compiler.pipeline.models import CompilePipelineResult from sqlbuild.executor.testing.main._execute import execute_sql_test from sqlbuild.executor.testing.models import SqlTestExecutionResult from sqlbuild.executor.testing.types import SqlTestOutcome diff --git a/tests/integration/src/sqlbuild/integrations/dbt/helpers.py b/tests/integration/src/sqlbuild/integrations/dbt/helpers.py index 97893a3edc..9fcebecf61 100644 --- a/tests/integration/src/sqlbuild/integrations/dbt/helpers.py +++ b/tests/integration/src/sqlbuild/integrations/dbt/helpers.py @@ -8,7 +8,7 @@ build_compile_reference_resolver, ) from sqlbuild.adapters.duckdb.classes.duckdb_adapter import DuckDbAdapter -from sqlbuild.compiler.references.types import ExternalSqlReferenceResolver +from sqlbuild.compiler.compile.types import ExternalSqlReferenceResolver DUCKDB_COMPILE_ADAPTER_CONTEXT: CompileAdapterContext = CompileAdapterContext( value_renderer=DuckDbAdapter(), diff --git a/tests/integration/src/sqlbuild/integrations/dbt/test_graph.py b/tests/integration/src/sqlbuild/integrations/dbt/test_graph.py index 4284a9bbf6..e459462a3d 100644 --- a/tests/integration/src/sqlbuild/integrations/dbt/test_graph.py +++ b/tests/integration/src/sqlbuild/integrations/dbt/test_graph.py @@ -12,6 +12,7 @@ DbtCommandResult, DbtManifestIndex, DbtRunner, + DiscoveredProjectInputs, assemble_project, build_compile_inputs, build_dbt_combined_graph, @@ -19,7 +20,6 @@ discover_project_inputs, ) from sqlbuild.compiler.compile.types import CompiledProjectView -from sqlbuild.compiler.discovery.models import DiscoveredProjectInputs from tests.integration.src.sqlbuild.integrations.dbt._test_types import ( RealDbtCombinedGraphTestCase, ) diff --git a/tests/integration/src/sqlbuild/integrations/dbt/test_manifest.py b/tests/integration/src/sqlbuild/integrations/dbt/test_manifest.py index 1f527821db..d7e800f628 100644 --- a/tests/integration/src/sqlbuild/integrations/dbt/test_manifest.py +++ b/tests/integration/src/sqlbuild/integrations/dbt/test_manifest.py @@ -9,10 +9,10 @@ DbtCliOptions, DbtCommandResult, DbtRunner, + DiscoveredProjectInputs, build_compile_inputs, discover_project_inputs, ) -from sqlbuild.compiler.discovery.models import DiscoveredProjectInputs from tests.integration.src.sqlbuild.integrations.dbt._test_types import ( RealDbtManifestCompileTestCase, ) diff --git a/tests/unit/scripts/compiler_differential/_helpers/coverage/_test_types.py b/tests/unit/scripts/compiler_differential/_helpers/coverage/_test_types.py index e7ee8d4540..1486874e28 100644 --- a/tests/unit/scripts/compiler_differential/_helpers/coverage/_test_types.py +++ b/tests/unit/scripts/compiler_differential/_helpers/coverage/_test_types.py @@ -5,7 +5,7 @@ from dataclasses import dataclass from typing import TYPE_CHECKING -from sqlbuild.compiler.discovery.models import DiscoveredProjectInputs +from sqlbuild._native import DiscoveredProjectInputs if TYPE_CHECKING: from sqlbuild._native import CompileInputsView diff --git a/tests/unit/scripts/compiler_differential/_helpers/coverage/helpers.py b/tests/unit/scripts/compiler_differential/_helpers/coverage/helpers.py index 6798ba8333..62cb3cd70c 100644 --- a/tests/unit/scripts/compiler_differential/_helpers/coverage/helpers.py +++ b/tests/unit/scripts/compiler_differential/_helpers/coverage/helpers.py @@ -6,9 +6,12 @@ from itertools import filterfalse from typing import cast -from sqlbuild._native import STAGE_CAPTURE_OMITTED_ATTRIBUTES, CompileProjectInputs +from sqlbuild._native import ( + STAGE_CAPTURE_OMITTED_ATTRIBUTES, + CompileProjectInputs, + DiscoveredProjectInputs, +) from sqlbuild.compiler.compile.models import CompiledProject -from sqlbuild.compiler.discovery.models import DiscoveredProjectInputs _VALUE_KIND: str = "sqlbuild.sql_values.types:SqlValueKind" diff --git a/tests/unit/scripts/compiler_differential/_helpers/coverage/test_discovery.py b/tests/unit/scripts/compiler_differential/_helpers/coverage/test_discovery.py index 42ce02e13a..c9dc6fe253 100644 --- a/tests/unit/scripts/compiler_differential/_helpers/coverage/test_discovery.py +++ b/tests/unit/scripts/compiler_differential/_helpers/coverage/test_discovery.py @@ -15,8 +15,12 @@ required_discovery_kinds, ) from scripts.compiler_differential.constants import DISCOVERY_NON_COLLECTION_FIELDS -from sqlbuild._native import LocalConfig, ProjectConfig, render_stage_capture -from sqlbuild.compiler.discovery.models import DiscoveredProjectInputs +from sqlbuild._native import ( + DiscoveredProjectInputs, + LocalConfig, + ProjectConfig, + render_stage_capture, +) from tests.unit.scripts.compiler_differential._helpers.coverage._test_types import ( AuthoredByteKindsTestCase, CaptureProblemsTestCase, diff --git a/tests/unit/scripts/compiler_differential/_helpers/coverage/test_render.py b/tests/unit/scripts/compiler_differential/_helpers/coverage/test_render.py index a9bfb3280d..a8e553b9ee 100644 --- a/tests/unit/scripts/compiler_differential/_helpers/coverage/test_render.py +++ b/tests/unit/scripts/compiler_differential/_helpers/coverage/test_render.py @@ -17,12 +17,12 @@ from scripts.compiler_differential.constants import RENDER_NON_COLLECTION_FIELDS from sqlbuild._native import ( CompileProjectInputs, + DiscoveredProjectInputs, LocalConfig, ProjectConfig, SqlLexicalSyntax, render_stage_capture, ) -from sqlbuild.compiler.discovery.models import DiscoveredProjectInputs from tests.unit.scripts.compiler_differential._helpers.coverage._test_types import ( CaptureProblemsTestCase, RenderCaptureKindsTestCase, diff --git a/tests/unit/src/sqlbuild/adapter/contract/classes/base_adapter/test_statement_size.py b/tests/unit/src/sqlbuild/adapter/contract/classes/base_adapter/test_statement_size.py index a7d666cb56..29a0cde63e 100644 --- a/tests/unit/src/sqlbuild/adapter/contract/classes/base_adapter/test_statement_size.py +++ b/tests/unit/src/sqlbuild/adapter/contract/classes/base_adapter/test_statement_size.py @@ -4,6 +4,7 @@ import pytest +from sqlbuild._native import CompileInputError from sqlbuild.adapters.bigquery.classes.bigquery_adapter import BigQueryAdapter from sqlbuild.adapters.databricks.classes.databricks_adapter import DatabricksAdapter from sqlbuild.adapters.duckdb.classes.duckdb_adapter import DuckDbAdapter @@ -11,7 +12,6 @@ from sqlbuild.adapters.postgres.classes.postgres_adapter import PostgresAdapter from sqlbuild.adapters.snowflake.classes.snowflake_adapter import SnowflakeAdapter from sqlbuild.adapters.sqlserver.classes.sqlserver_adapter import SqlServerAdapter -from sqlbuild.compiler.compile.exceptions import CompileInputError from sqlbuild.executor.testing.main._sql_length import validate_unit_test_sql_length from tests.unit.src.sqlbuild.adapter.contract.classes.base_adapter._test_types import ( StatementSizeLimitTestCase, diff --git a/tests/unit/src/sqlbuild/cli/commands/_helpers/check/test_planning.py b/tests/unit/src/sqlbuild/cli/commands/_helpers/check/test_planning.py index 6149b9f353..08a3765a6f 100644 --- a/tests/unit/src/sqlbuild/cli/commands/_helpers/check/test_planning.py +++ b/tests/unit/src/sqlbuild/cli/commands/_helpers/check/test_planning.py @@ -8,6 +8,7 @@ from sqlbuild._native import ( CliUserError, + CompilePipelineResult, PlanOutput, PythonSqlRunLifecyclePlan, SourceEntry, @@ -16,7 +17,6 @@ from sqlbuild.adapter.contract.models import RelationLookup from sqlbuild.adapter.relations.main import relation_lookup as relation_lookup_module from sqlbuild.compiler.compile.models import CompiledProject, CompiledRelationLocation -from sqlbuild.compiler.pipeline.models import CompilePipelineResult from sqlbuild.executor.python_nodes.main import read_side as read_side_module from sqlbuild.executor.python_nodes.models import PythonNodeExecutionResult from sqlbuild.python_nodes.models import SqlResourceRef diff --git a/tests/unit/src/sqlbuild/cli/commands/_helpers/load_selection/helpers.py b/tests/unit/src/sqlbuild/cli/commands/_helpers/load_selection/helpers.py index 51c473ce3b..d0312a9858 100644 --- a/tests/unit/src/sqlbuild/cli/commands/_helpers/load_selection/helpers.py +++ b/tests/unit/src/sqlbuild/cli/commands/_helpers/load_selection/helpers.py @@ -4,11 +4,14 @@ from pathlib import Path -from sqlbuild._native import IntegrationLoaderConfig, LocalConfig, ProjectConfig, SourceEntry -from sqlbuild.compiler.discovery.models import ( +from sqlbuild._native import ( DiscoveredLoaderFunction, DiscoveredProjectInputs, DiscoveredSourceFile, + IntegrationLoaderConfig, + LocalConfig, + ProjectConfig, + SourceEntry, ) diff --git a/tests/unit/src/sqlbuild/cli/commands/_helpers/load_selection/test_selection.py b/tests/unit/src/sqlbuild/cli/commands/_helpers/load_selection/test_selection.py index 3cb53077ac..de12ddeff6 100644 --- a/tests/unit/src/sqlbuild/cli/commands/_helpers/load_selection/test_selection.py +++ b/tests/unit/src/sqlbuild/cli/commands/_helpers/load_selection/test_selection.py @@ -7,6 +7,8 @@ import pytest from sqlbuild._native import ( + DiscoveredLoaderFunction, + DiscoveredProjectInputs, LocalConfig, ProjectConfig, SourceEntry, @@ -15,7 +17,6 @@ select_load_entries, select_load_reference_entries, ) -from sqlbuild.compiler.discovery.models import DiscoveredLoaderFunction, DiscoveredProjectInputs from tests.unit.src.sqlbuild.cli.commands._helpers.load_selection._test_types import ( LoaderDefaultResolutionTestCase, LoaderDestinationSelectionTestCase, diff --git a/tests/unit/src/sqlbuild/cli/commands/_helpers/runtime/helpers.py b/tests/unit/src/sqlbuild/cli/commands/_helpers/runtime/helpers.py index 29760e4191..a707305d5c 100644 --- a/tests/unit/src/sqlbuild/cli/commands/_helpers/runtime/helpers.py +++ b/tests/unit/src/sqlbuild/cli/commands/_helpers/runtime/helpers.py @@ -4,8 +4,13 @@ from pathlib import Path -from sqlbuild._native import LocalConfig, ProjectConfig, TargetConfig, TargetWarehousesConfig -from sqlbuild.compiler.discovery.models import DiscoveredProjectInputs +from sqlbuild._native import ( + DiscoveredProjectInputs, + LocalConfig, + ProjectConfig, + TargetConfig, + TargetWarehousesConfig, +) from tests.unit.src.sqlbuild.cli.commands._helpers.runtime._test_types import ( CommandWarehouseResolutionTestCase, ) diff --git a/tests/unit/src/sqlbuild/cli/commands/_helpers/runtime/test_connection.py b/tests/unit/src/sqlbuild/cli/commands/_helpers/runtime/test_connection.py index cb0a7c8366..e6de32c867 100644 --- a/tests/unit/src/sqlbuild/cli/commands/_helpers/runtime/test_connection.py +++ b/tests/unit/src/sqlbuild/cli/commands/_helpers/runtime/test_connection.py @@ -6,6 +6,8 @@ from sqlbuild import _native as native from sqlbuild._native import ( + CompileInputError, + DiscoveredProjectInputs, LocalConfig, LocalTargetConfig, NormalizedDbtProfileConnection, @@ -17,8 +19,6 @@ resolve_target_connection_config, ) from sqlbuild.adapter.contract.types import BuiltinAdapter -from sqlbuild.compiler.compile.exceptions import CompileInputError -from sqlbuild.compiler.discovery.models import DiscoveredProjectInputs from tests.unit.src.sqlbuild.cli.commands._helpers.runtime._test_types import ( ExplicitMemoryConnectionTestCase, MissingConnectionTestCase, diff --git a/tests/unit/src/sqlbuild/cli/commands/_helpers/runtime/test_warehouses.py b/tests/unit/src/sqlbuild/cli/commands/_helpers/runtime/test_warehouses.py index 917bcc0c76..bf38d9db07 100644 --- a/tests/unit/src/sqlbuild/cli/commands/_helpers/runtime/test_warehouses.py +++ b/tests/unit/src/sqlbuild/cli/commands/_helpers/runtime/test_warehouses.py @@ -9,6 +9,7 @@ from sqlbuild._native import ( CliCommand, CliUserError, + DiscoveredProjectInputs, LocalTargetConfig, ParsedCliInvocation, ResolvedWarehouse, @@ -20,7 +21,6 @@ parse_cli_invocation, resolve_project_connection_config, ) -from sqlbuild.compiler.discovery.models import DiscoveredProjectInputs from tests.unit.src.sqlbuild.cli.commands._helpers.runtime._test_types import ( CommandWarehouseErrorTestCase, CommandWarehouseGroupTestCase, diff --git a/tests/unit/src/sqlbuild/cli/commands/main/compile/helpers.py b/tests/unit/src/sqlbuild/cli/commands/main/compile/helpers.py index a04f0cbcb6..8b0b01301f 100644 --- a/tests/unit/src/sqlbuild/cli/commands/main/compile/helpers.py +++ b/tests/unit/src/sqlbuild/cli/commands/main/compile/helpers.py @@ -18,6 +18,7 @@ PlanOutput, PlanReason, PreparedCompileArtifacts, + ProjectGraph, SeedCsvSettings, SeedPlanEntry, SqlTestPlanEntry, @@ -44,7 +45,6 @@ CompiledResourceType, FunctionLanguage, ) -from sqlbuild.compiler.pipeline.models import ProjectGraph from sqlbuild.executor.build.models import ( BuildExecutionResult, FunctionExecutionResult, diff --git a/tests/unit/src/sqlbuild/cli/commands/main/compile/test_compile_output.py b/tests/unit/src/sqlbuild/cli/commands/main/compile/test_compile_output.py index 52ece4d88d..b5447a3436 100644 --- a/tests/unit/src/sqlbuild/cli/commands/main/compile/test_compile_output.py +++ b/tests/unit/src/sqlbuild/cli/commands/main/compile/test_compile_output.py @@ -10,6 +10,8 @@ CompileLineageMode, CompileModelConfig, ProjectColumnLineage, + ProjectGraph, + SqlHookEntry, WrittenTarget, compiled_model_record, format_compile_json, @@ -18,8 +20,6 @@ ) from sqlbuild.compiler.compile.models import CompiledObjectKey from sqlbuild.compiler.compile.types import CompiledModelView, CompiledResourceType -from sqlbuild.compiler.discovery.models import SqlHookEntry -from sqlbuild.compiler.pipeline.models import ProjectGraph from tests.unit.src.sqlbuild.cli.commands.main.compile._test_types import ( CompileJsonExecutionLayersTestCase, CompileJsonLargeIntegerTestCase, diff --git a/tests/unit/src/sqlbuild/cli/commands/main/entry/_test_types.py b/tests/unit/src/sqlbuild/cli/commands/main/entry/_test_types.py index 334d3df385..1bcf26a480 100644 --- a/tests/unit/src/sqlbuild/cli/commands/main/entry/_test_types.py +++ b/tests/unit/src/sqlbuild/cli/commands/main/entry/_test_types.py @@ -5,11 +5,11 @@ from sqlbuild._native import ( CliUserError, ColumnLineageMode, + CompileInputError, CompileLineageMode, ProjectConfigError, SpecConfigError, ) -from sqlbuild.compiler.compile.exceptions import CompileInputError @dataclass(frozen=True) diff --git a/tests/unit/src/sqlbuild/cli/commands/main/entry/test_main.py b/tests/unit/src/sqlbuild/cli/commands/main/entry/test_main.py index b3fbacdb99..5e82c8d4d6 100644 --- a/tests/unit/src/sqlbuild/cli/commands/main/entry/test_main.py +++ b/tests/unit/src/sqlbuild/cli/commands/main/entry/test_main.py @@ -16,6 +16,7 @@ CloneCommandRequest, ColumnLineageMode, CompileCommandRequest, + CompileInputError, CompileLineageMode, CostCommandRequest, CursorOverrides, @@ -35,7 +36,6 @@ ) from sqlbuild._native import run_cli as _main_with_dependencies from sqlbuild._native import run_cli as main -from sqlbuild.compiler.compile.exceptions import CompileInputError from sqlbuild.executor.pipeline.exceptions import AuditExecutionError from sqlbuild.executor.pipeline.models import AuditExecutionFailure from sqlbuild.observability import EventDispatcher, ExecutionIdentity, current_execution_identity diff --git a/tests/unit/src/sqlbuild/cli/commands/main/execution/helpers.py b/tests/unit/src/sqlbuild/cli/commands/main/execution/helpers.py index ded745f743..5e8e51b653 100644 --- a/tests/unit/src/sqlbuild/cli/commands/main/execution/helpers.py +++ b/tests/unit/src/sqlbuild/cli/commands/main/execution/helpers.py @@ -10,11 +10,15 @@ import pytest from sqlbuild import _native as native -from sqlbuild._native import LocalConfig, ProjectConfig +from sqlbuild._native import ( + DiscoveredProjectInputs, + DiscoveredSqlScenarioFile, + LocalConfig, + ProjectConfig, +) from sqlbuild.adapter.discovery.main import resolve_adapter as adapters_module from sqlbuild.compiler.compile.models import CompiledObjectKey, CompiledSqlScenario from sqlbuild.compiler.compile.types import CompiledResourceType -from sqlbuild.compiler.discovery.models import DiscoveredProjectInputs, DiscoveredSqlScenarioFile from sqlbuild.diagnostics.main import process_resource_reporting as resource_reporting_module from sqlbuild.executor.pipeline.main import run as executor_run_module from tests.unit.src.sqlbuild.presentation.helpers import use_plain_terminal diff --git a/tests/unit/src/sqlbuild/cli/commands/main/execution/test_scenario_command.py b/tests/unit/src/sqlbuild/cli/commands/main/execution/test_scenario_command.py index b20328cb18..340918201b 100644 --- a/tests/unit/src/sqlbuild/cli/commands/main/execution/test_scenario_command.py +++ b/tests/unit/src/sqlbuild/cli/commands/main/execution/test_scenario_command.py @@ -8,11 +8,11 @@ import pytest from sqlbuild._native import ( + CompilePipelineResult, NativeProgressProjector, ScenarioTestCommandRequest, run_scenario_test_command, ) -from sqlbuild.compiler.pipeline.models import CompilePipelineResult from sqlbuild.observability import EventDispatcher, LifecycleEvent, dispatcher_scope from tests.unit.src.sqlbuild.cli.commands.main.execution._test_types import ( ScenarioCompilePresentationTestCase, diff --git a/tests/unit/src/sqlbuild/cli/output/_helpers/test_execution_result_document.py b/tests/unit/src/sqlbuild/cli/output/_helpers/test_execution_result_document.py index 36939cb23f..559a92b544 100644 --- a/tests/unit/src/sqlbuild/cli/output/_helpers/test_execution_result_document.py +++ b/tests/unit/src/sqlbuild/cli/output/_helpers/test_execution_result_document.py @@ -16,6 +16,7 @@ MeasurementThresholdBound, MeasurementThresholds, SqlLogicalType, + SqlTestParameterDeclaration, SqlValue, SqlValueKind, format_audit_execution_json, @@ -31,7 +32,6 @@ ThresholdOperator, ) from sqlbuild.compiler.compile.types import AttachedAuditTargetKind -from sqlbuild.compiler.discovery.models import SqlTestParameterDeclaration from sqlbuild.executor.auditing.models import AuditExecutionResult from sqlbuild.executor.run.models import ( MicrobatchAccountingInterval, diff --git a/tests/unit/src/sqlbuild/cli/output/main/plan/helpers.py b/tests/unit/src/sqlbuild/cli/output/main/plan/helpers.py index ee834131f9..4cf7b16f6d 100644 --- a/tests/unit/src/sqlbuild/cli/output/main/plan/helpers.py +++ b/tests/unit/src/sqlbuild/cli/output/main/plan/helpers.py @@ -10,6 +10,7 @@ ColumnRenameHint, CursorBounds, CursorInputRelation, + DiscoveredProviderUsage, FunctionPlanEntry, MaterializationType, MicrobatchLimitAction, @@ -34,7 +35,6 @@ CompiledRelationLocation, ) from sqlbuild.compiler.compile.types import CompiledResourceType, FunctionLanguage -from sqlbuild.compiler.discovery.models import DiscoveredProviderUsage def build_model_entry( diff --git a/tests/unit/src/sqlbuild/compiler/compile/_helpers/_test_types.py b/tests/unit/src/sqlbuild/compiler/compile/_helpers/_test_types.py index fa5ec544d3..d6b6f8ccd0 100644 --- a/tests/unit/src/sqlbuild/compiler/compile/_helpers/_test_types.py +++ b/tests/unit/src/sqlbuild/compiler/compile/_helpers/_test_types.py @@ -11,6 +11,7 @@ ResolvedTimeTravelRetention, SourceLocation, SqlLexicalSyntax, + SqlReferenceKind, SqlValue, UsageRecord, ) @@ -20,7 +21,6 @@ from sqlbuild.compiler.lineage.types import ( InferredNullability, ) -from sqlbuild.compiler.references.types import SqlReferenceKind if TYPE_CHECKING: from sqlbuild._native import CompileInputsView diff --git a/tests/unit/src/sqlbuild/compiler/compile/_helpers/config/test_table_type.py b/tests/unit/src/sqlbuild/compiler/compile/_helpers/config/test_table_type.py index 7b1e157a0c..e0dc676c2f 100644 --- a/tests/unit/src/sqlbuild/compiler/compile/_helpers/config/test_table_type.py +++ b/tests/unit/src/sqlbuild/compiler/compile/_helpers/config/test_table_type.py @@ -3,6 +3,7 @@ import pytest from sqlbuild._native import ( + CompileInputError, CompileModelConfig, DefaultsConfig, MaterializationDefaultsConfig, @@ -12,7 +13,6 @@ TableTypeSource, TargetConfig, ) -from sqlbuild.compiler.compile.exceptions import CompileInputError from tests.unit.src.sqlbuild.compiler.compile._helpers.config._test_types import ( TableTypeResolutionErrorTestCase, TableTypeResolutionTestCase, diff --git a/tests/unit/src/sqlbuild/compiler/compile/_helpers/helpers.py b/tests/unit/src/sqlbuild/compiler/compile/_helpers/helpers.py index 467fba143c..4807eb8696 100644 --- a/tests/unit/src/sqlbuild/compiler/compile/_helpers/helpers.py +++ b/tests/unit/src/sqlbuild/compiler/compile/_helpers/helpers.py @@ -21,6 +21,8 @@ DeclarationRecord, DeclarationRuntimeProjection, DeclarationScopeResolver, + DiscoveredMacroFile, + DiscoveredProjectInputs, MacroMetadata, NativeColumnQuery, NativeQueryColumns, @@ -68,10 +70,6 @@ CompiledProjectView, CompiledResourceType, ) -from sqlbuild.compiler.discovery.models import ( - DiscoveredMacroFile, - DiscoveredProjectInputs, -) from sqlbuild.compiler.lineage.types import InferredNullability if TYPE_CHECKING: diff --git a/tests/unit/src/sqlbuild/compiler/compile/_helpers/render/test_cursor_intrinsics.py b/tests/unit/src/sqlbuild/compiler/compile/_helpers/render/test_cursor_intrinsics.py index 24af81b145..c1264614e0 100644 --- a/tests/unit/src/sqlbuild/compiler/compile/_helpers/render/test_cursor_intrinsics.py +++ b/tests/unit/src/sqlbuild/compiler/compile/_helpers/render/test_cursor_intrinsics.py @@ -3,6 +3,7 @@ import pytest from sqlbuild._native import ( + CompileInputError, NativeColumnQuery, cursor_intrinsics_analysis_sql, infer_native_query_columns, @@ -10,7 +11,6 @@ validated_model_cursor_intrinsics, ) from sqlbuild.adapter.contract.models import ExpressionInferenceProfile -from sqlbuild.compiler.compile.exceptions import CompileInputError from sqlbuild.compiler.compile.models import InferredColumn from sqlbuild.compiler.lineage.types import InferredNullability from tests.unit.src.sqlbuild.compiler.compile._helpers.render._test_types import ( diff --git a/tests/unit/src/sqlbuild/compiler/compile/_helpers/test_assembly.py b/tests/unit/src/sqlbuild/compiler/compile/_helpers/test_assembly.py index 1c5b701ff9..9d06c7448f 100644 --- a/tests/unit/src/sqlbuild/compiler/compile/_helpers/test_assembly.py +++ b/tests/unit/src/sqlbuild/compiler/compile/_helpers/test_assembly.py @@ -6,7 +6,13 @@ import pytest -from sqlbuild._native import assemble_project, build_compile_inputs, discover_project_inputs +from sqlbuild._native import ( + DiscoveredProjectInputs, + SqlHookEntry, + assemble_project, + build_compile_inputs, + discover_project_inputs, +) from sqlbuild.adapter.contract.models import ExpressionInferenceProfile from sqlbuild.compiler.compile.models import ( CompiledObjectKey, @@ -17,7 +23,6 @@ CompiledResourceType, CompiledSeedView, ) -from sqlbuild.compiler.discovery.models import DiscoveredProjectInputs, SqlHookEntry from tests.unit.src.sqlbuild.compiler.compile._helpers._test_types import ( AssembleCompiledProjectEffectiveTargetTestCase, AssembleCompiledProjectTestCase, diff --git a/tests/unit/src/sqlbuild/compiler/compile/_helpers/test_audit_factories.py b/tests/unit/src/sqlbuild/compiler/compile/_helpers/test_audit_factories.py index 8f541715e3..6f9c5666b7 100644 --- a/tests/unit/src/sqlbuild/compiler/compile/_helpers/test_audit_factories.py +++ b/tests/unit/src/sqlbuild/compiler/compile/_helpers/test_audit_factories.py @@ -9,12 +9,12 @@ import pytest from sqlbuild._native import ( + CompileInputError, SchemaModelEntry, SourceLocation, build_compile_inputs, discover_project_inputs, ) -from sqlbuild.compiler.compile.exceptions import CompileInputError from tests.unit.src.sqlbuild.compiler.compile._helpers._test_types import ( AuditFactoryAttachmentTestCase, ) diff --git a/tests/unit/src/sqlbuild/compiler/compile/_helpers/test_declaration_scopes.py b/tests/unit/src/sqlbuild/compiler/compile/_helpers/test_declaration_scopes.py index 091c2703d9..7e2007f1a7 100644 --- a/tests/unit/src/sqlbuild/compiler/compile/_helpers/test_declaration_scopes.py +++ b/tests/unit/src/sqlbuild/compiler/compile/_helpers/test_declaration_scopes.py @@ -10,11 +10,15 @@ import pytest from sqlbuild._native import ( + CompileInputError, DeclarationIdentity, DeclarationKind, + DiscoveredProjectInputs, + DiscoveredSourceFile, ResourceIdentity, ResourceKind, SourceEntry, + SqlHookEntry, UsageKind, UsageRecord, assemble_project, @@ -22,13 +26,7 @@ discover_project_inputs, scope_metadata_projection, ) -from sqlbuild.compiler.compile.exceptions import CompileInputError from sqlbuild.compiler.compile.types import CompiledProjectView -from sqlbuild.compiler.discovery.models import ( - DiscoveredProjectInputs, - DiscoveredSourceFile, - SqlHookEntry, -) from tests.unit.src.sqlbuild.compiler.compile._helpers._test_types import ( ExpectedBooleanTestCase, ExpectedModelDeclarationGrantTestCase, diff --git a/tests/unit/src/sqlbuild/compiler/compile/_helpers/test_declarations.py b/tests/unit/src/sqlbuild/compiler/compile/_helpers/test_declarations.py index 9a08befc21..d629765405 100644 --- a/tests/unit/src/sqlbuild/compiler/compile/_helpers/test_declarations.py +++ b/tests/unit/src/sqlbuild/compiler/compile/_helpers/test_declarations.py @@ -8,17 +8,18 @@ import pytest from sqlbuild._native import ( + CompileInputError, DeclarationParseError, + DiscoveredProjectInputs, ResourceIdentityError, + SqlHookEntry, assemble_project, build_compile_inputs, build_model_version_identity_metadata_json, compute_query_hash, discover_project_inputs, ) -from sqlbuild.compiler.compile.exceptions import CompileInputError from sqlbuild.compiler.compile.types import CompiledModelView, CompiledProjectView -from sqlbuild.compiler.discovery.models import DiscoveredProjectInputs, SqlHookEntry from tests.unit.src.sqlbuild.compiler.compile._helpers._test_types import ( CompileDeclarationsErrorTestCase, CompileDeclarationsTestCase, diff --git a/tests/unit/src/sqlbuild/compiler/compile/_helpers/test_descriptions.py b/tests/unit/src/sqlbuild/compiler/compile/_helpers/test_descriptions.py index af860d16b1..acc1df5771 100644 --- a/tests/unit/src/sqlbuild/compiler/compile/_helpers/test_descriptions.py +++ b/tests/unit/src/sqlbuild/compiler/compile/_helpers/test_descriptions.py @@ -6,7 +6,7 @@ import pytest -from sqlbuild.compiler.discovery.models import DiscoveredProjectInputs +from sqlbuild._native import DiscoveredProjectInputs from tests.unit.src.sqlbuild.compiler.compile._helpers._test_types import ( DescriptionInputClassificationCase, ) diff --git a/tests/unit/src/sqlbuild/compiler/compile/_helpers/test_macros.py b/tests/unit/src/sqlbuild/compiler/compile/_helpers/test_macros.py index 0ce5692eec..f8032528dd 100644 --- a/tests/unit/src/sqlbuild/compiler/compile/_helpers/test_macros.py +++ b/tests/unit/src/sqlbuild/compiler/compile/_helpers/test_macros.py @@ -5,14 +5,18 @@ import pytest -from sqlbuild._native import ScopeKind, expand_sql_macros, find_macro_call_names +from sqlbuild._native import ( + DiscoveredMacroFile, + ScopeKind, + expand_sql_macros, + find_macro_call_names, +) from sqlbuild.compiler.compile._helpers.render import user_macros as macros_module from sqlbuild.compiler.compile._helpers.render.user_macros import load_project_macros from sqlbuild.compiler.compile.models import ( LoadedMacro, MacroContext, ) -from sqlbuild.compiler.discovery.models import DiscoveredMacroFile from tests.unit.src.sqlbuild.compiler.compile._helpers._test_types import ( ExpandSqlMacrosErrorTestCase, ExpandSqlMacrosTestCase, diff --git a/tests/unit/src/sqlbuild/compiler/compile/_helpers/test_microbatch_redesign.py b/tests/unit/src/sqlbuild/compiler/compile/_helpers/test_microbatch_redesign.py index ea02a03e59..e3a637ac74 100644 --- a/tests/unit/src/sqlbuild/compiler/compile/_helpers/test_microbatch_redesign.py +++ b/tests/unit/src/sqlbuild/compiler/compile/_helpers/test_microbatch_redesign.py @@ -5,6 +5,7 @@ import pytest from sqlbuild._native import ( + CompileInputError, CompileModelConfig, CursorBounds, CursorInputRelation, @@ -15,7 +16,6 @@ resolve_effective_timestamp_grain, watermark_type_is_compatible, ) -from sqlbuild.compiler.compile.exceptions import CompileInputError from sqlbuild.executor.run._helpers.validation.cursor_bounds import ( resolve_effective_timestamp_grain as resolve_runtime_effective_timestamp_grain, ) diff --git a/tests/unit/src/sqlbuild/compiler/compile/_helpers/test_model_config_validation.py b/tests/unit/src/sqlbuild/compiler/compile/_helpers/test_model_config_validation.py index 17b779ab5a..e72b1e8244 100644 --- a/tests/unit/src/sqlbuild/compiler/compile/_helpers/test_model_config_validation.py +++ b/tests/unit/src/sqlbuild/compiler/compile/_helpers/test_model_config_validation.py @@ -7,12 +7,12 @@ import pytest from sqlbuild._native import ( + CompileInputError, CompileModelConfig, ConfigValueTypeError, ResolvedTimeTravelRetention, TimeTravelRetentionSource, ) -from sqlbuild.compiler.compile.exceptions import CompileInputError from tests.unit.src.sqlbuild.compiler.compile._helpers._test_types import ( ChangePolicyHelpTestCase, ContractConfigErrorTestCase, diff --git a/tests/unit/src/sqlbuild/compiler/compile/_helpers/test_model_schemas.py b/tests/unit/src/sqlbuild/compiler/compile/_helpers/test_model_schemas.py index a341e5b8e3..3b1a67f4a6 100644 --- a/tests/unit/src/sqlbuild/compiler/compile/_helpers/test_model_schemas.py +++ b/tests/unit/src/sqlbuild/compiler/compile/_helpers/test_model_schemas.py @@ -8,6 +8,8 @@ import pytest from sqlbuild._native import ( + CompileInputError, + DiscoveredProjectInputs, SchemaAuditInstance, SchemaModelEntry, SourceLocation, @@ -18,12 +20,10 @@ evaluate_model_contracts, ) from sqlbuild.adapters.duckdb.classes.duckdb_adapter import DuckDbAdapter -from sqlbuild.compiler.compile.exceptions import CompileInputError from sqlbuild.compiler.compile.models import ( CompilerDiagnostic, ) from sqlbuild.compiler.compile.types import CompiledModelView, CompiledProjectView -from sqlbuild.compiler.discovery.models import DiscoveredProjectInputs from tests.unit.src.sqlbuild.compiler.compile._helpers._test_types import ( DynamicModelSchemaTestCase, ExpectedCountTestCase, diff --git a/tests/unit/src/sqlbuild/compiler/compile/_helpers/test_parameterized_sql_tests.py b/tests/unit/src/sqlbuild/compiler/compile/_helpers/test_parameterized_sql_tests.py index 02e5a47452..695580993a 100644 --- a/tests/unit/src/sqlbuild/compiler/compile/_helpers/test_parameterized_sql_tests.py +++ b/tests/unit/src/sqlbuild/compiler/compile/_helpers/test_parameterized_sql_tests.py @@ -8,6 +8,7 @@ import pytest from sqlbuild._native import ( + CompileInputError, SqlLogicalType, SqlValue, SqlValueKind, @@ -16,7 +17,6 @@ ) from sqlbuild.adapters.bigquery.classes.bigquery_adapter import BigQueryAdapter from sqlbuild.adapters.sqlserver.classes.sqlserver_adapter import SqlServerAdapter -from sqlbuild.compiler.compile.exceptions import CompileInputError from sqlbuild.compiler.compile.models import ( CompiledObjectKey, CompiledSqlTest, diff --git a/tests/unit/src/sqlbuild/compiler/compile/_helpers/test_rebind_declaration_scope.py b/tests/unit/src/sqlbuild/compiler/compile/_helpers/test_rebind_declaration_scope.py index 7bbf5d55e4..2f18c6002c 100644 --- a/tests/unit/src/sqlbuild/compiler/compile/_helpers/test_rebind_declaration_scope.py +++ b/tests/unit/src/sqlbuild/compiler/compile/_helpers/test_rebind_declaration_scope.py @@ -11,6 +11,7 @@ DeclarationIdentity, DeclarationKind, DeclarationScopeBuild, + DiscoveredProjectInputs, SqlLexicalSyntax, build_declaration_scope, discover_project_inputs, @@ -18,7 +19,6 @@ ) from sqlbuild.compiler.compile._helpers.render.user_macros import load_project_macros from sqlbuild.compiler.compile.models import LoadedMacro -from sqlbuild.compiler.discovery.models import DiscoveredProjectInputs from tests.unit.src.sqlbuild.compiler.compile._helpers._test_types import ( RebindDeclarationScopeTestCase, ) diff --git a/tests/unit/src/sqlbuild/compiler/compile/_helpers/test_required_columns.py b/tests/unit/src/sqlbuild/compiler/compile/_helpers/test_required_columns.py index f07166bac9..f782666a02 100644 --- a/tests/unit/src/sqlbuild/compiler/compile/_helpers/test_required_columns.py +++ b/tests/unit/src/sqlbuild/compiler/compile/_helpers/test_required_columns.py @@ -4,10 +4,9 @@ import pytest -from sqlbuild._native import analyze_resolved_column_reads +from sqlbuild._native import SqlReferenceKind, analyze_resolved_column_reads from sqlbuild.compiler.compile.models import CompiledLineageSourceFact, CompileSqlReference from sqlbuild.compiler.compile.types import CompiledResourceType -from sqlbuild.compiler.references.types import SqlReferenceKind from tests.unit.src.sqlbuild.compiler.compile._helpers._test_types import ( LongUnionRequiredColumnsTestCase, RequiredExternalColumnsTestCase, diff --git a/tests/unit/src/sqlbuild/compiler/compile/_helpers/test_sql_analysis_columns.py b/tests/unit/src/sqlbuild/compiler/compile/_helpers/test_sql_analysis_columns.py index 0f73e79bee..283ac59d53 100644 --- a/tests/unit/src/sqlbuild/compiler/compile/_helpers/test_sql_analysis_columns.py +++ b/tests/unit/src/sqlbuild/compiler/compile/_helpers/test_sql_analysis_columns.py @@ -8,6 +8,7 @@ CompiledLineageColumnFact, NativeColumnQuery, NativeQueryColumns, + SqlReferenceKind, infer_native_query_columns, substitute_placeholder_defaults, ) @@ -19,7 +20,6 @@ ) from sqlbuild.compiler.compile.types import CompiledResourceType from sqlbuild.compiler.lineage.types import InferredNullability -from sqlbuild.compiler.references.types import SqlReferenceKind from tests.unit.src.sqlbuild.compiler.compile._helpers._test_types import ( InferColumnsTestCase, PolyglotAnalysisTestCase, diff --git a/tests/unit/src/sqlbuild/compiler/compile/_helpers/test_sql_analysis_validation.py b/tests/unit/src/sqlbuild/compiler/compile/_helpers/test_sql_analysis_validation.py index ab3eaf9d83..9db54e9764 100644 --- a/tests/unit/src/sqlbuild/compiler/compile/_helpers/test_sql_analysis_validation.py +++ b/tests/unit/src/sqlbuild/compiler/compile/_helpers/test_sql_analysis_validation.py @@ -5,7 +5,7 @@ import pytest import sqlbuild._native as _native -from sqlbuild.compiler.compile.exceptions import CompileInputError +from sqlbuild._native import CompileInputError from tests.unit.src.sqlbuild.compiler.compile._helpers._test_types import ( ValidateHookSqlSyntaxTestCase, ValidateSqlSyntaxTestCase, diff --git a/tests/unit/src/sqlbuild/compiler/compile/_helpers/test_sql_references.py b/tests/unit/src/sqlbuild/compiler/compile/_helpers/test_sql_references.py index e182b40a38..0d14eaad84 100644 --- a/tests/unit/src/sqlbuild/compiler/compile/_helpers/test_sql_references.py +++ b/tests/unit/src/sqlbuild/compiler/compile/_helpers/test_sql_references.py @@ -6,15 +6,19 @@ import pytest -from sqlbuild._native import SqlLexicalSyntax, collect_compile_diagnostics, extract_sql_references +from sqlbuild._native import ( + CompileInputError, + SqlLexicalSyntax, + SqlReferenceKind, + collect_compile_diagnostics, + extract_sql_references, +) from sqlbuild.adapters.bigquery.classes.bigquery_adapter import BigQueryAdapter from sqlbuild.adapters.databricks.classes.databricks_adapter import DatabricksAdapter from sqlbuild.adapters.duckdb.classes.duckdb_adapter import DuckDbAdapter from sqlbuild.adapters.postgres.classes.postgres_adapter import PostgresAdapter from sqlbuild.adapters.snowflake.classes.snowflake_adapter import SnowflakeAdapter -from sqlbuild.compiler.compile.exceptions import CompileInputError from sqlbuild.compiler.compile.models import CompileSqlReference -from sqlbuild.compiler.references.types import SqlReferenceKind from tests.unit.src.sqlbuild.compiler.compile._helpers._test_types import ( CollectedReferenceCallSyntaxTestCase, DialectSqlScanTestCase, diff --git a/tests/unit/src/sqlbuild/compiler/compile/_helpers/test_sql_vars.py b/tests/unit/src/sqlbuild/compiler/compile/_helpers/test_sql_vars.py index d070fba082..06d1cc8571 100644 --- a/tests/unit/src/sqlbuild/compiler/compile/_helpers/test_sql_vars.py +++ b/tests/unit/src/sqlbuild/compiler/compile/_helpers/test_sql_vars.py @@ -6,8 +6,7 @@ import pytest -from sqlbuild._native import substitute_sql_vars_with_spans -from sqlbuild.compiler.compile.exceptions import CompileInputError +from sqlbuild._native import CompileInputError, substitute_sql_vars_with_spans from tests.unit.src.sqlbuild.compiler.compile._helpers._test_types import ( SubstituteSqlVarsErrorTestCase, SubstituteSqlVarsTestCase, diff --git a/tests/unit/src/sqlbuild/compiler/compile/_helpers/test_templating.py b/tests/unit/src/sqlbuild/compiler/compile/_helpers/test_templating.py index 8830420024..05f0680f65 100644 --- a/tests/unit/src/sqlbuild/compiler/compile/_helpers/test_templating.py +++ b/tests/unit/src/sqlbuild/compiler/compile/_helpers/test_templating.py @@ -3,11 +3,11 @@ import pytest from sqlbuild._native import ( + CompileInputError, UnicodeEnvironment, expand_effective_vars, expand_template_data, ) -from sqlbuild.compiler.compile.exceptions import CompileInputError from tests.unit.src.sqlbuild.compiler.compile._helpers._test_types import ( ExpandTemplateDataErrorTestCase, ExpandTemplateDataTestCase, diff --git a/tests/unit/src/sqlbuild/compiler/compile/_helpers/test_typed_macro_references.py b/tests/unit/src/sqlbuild/compiler/compile/_helpers/test_typed_macro_references.py index c828ed11da..d5927cab1a 100644 --- a/tests/unit/src/sqlbuild/compiler/compile/_helpers/test_typed_macro_references.py +++ b/tests/unit/src/sqlbuild/compiler/compile/_helpers/test_typed_macro_references.py @@ -6,7 +6,7 @@ import pytest -from sqlbuild.compiler.compile.exceptions import CompileInputError +from sqlbuild._native import CompileInputError from sqlbuild.compiler.compile.models import CompilerDiagnostic from sqlbuild.compiler.compile.types import CompiledModelView, CompiledProjectView from tests.unit.src.sqlbuild.compiler.compile._helpers._test_types import ( diff --git a/tests/unit/src/sqlbuild/compiler/compile/_test_helpers.py b/tests/unit/src/sqlbuild/compiler/compile/_test_helpers.py index 20a9a56759..c0cc5cd6a7 100644 --- a/tests/unit/src/sqlbuild/compiler/compile/_test_helpers.py +++ b/tests/unit/src/sqlbuild/compiler/compile/_test_helpers.py @@ -9,7 +9,7 @@ CompileSqlTestInput, build_compile_reference_resolver, ) -from sqlbuild.compiler.references.types import ExternalSqlReferenceResolver +from sqlbuild.compiler.compile.types import ExternalSqlReferenceResolver def base_repo_files() -> dict[str, str]: diff --git a/tests/unit/src/sqlbuild/compiler/compile/test_cursor_start.py b/tests/unit/src/sqlbuild/compiler/compile/test_cursor_start.py index 00c1a2ab70..0cdc703b1f 100644 --- a/tests/unit/src/sqlbuild/compiler/compile/test_cursor_start.py +++ b/tests/unit/src/sqlbuild/compiler/compile/test_cursor_start.py @@ -6,8 +6,7 @@ import pytest -from sqlbuild._native import build_compile_inputs, discover_project_inputs -from sqlbuild.compiler.discovery.models import DiscoveredProjectInputs +from sqlbuild._native import DiscoveredProjectInputs, build_compile_inputs, discover_project_inputs from tests.unit.src.sqlbuild.compiler.compile._helpers.helpers import ( DUCKDB_COMPILE_ADAPTER_CONTEXT, ) diff --git a/tests/unit/src/sqlbuild/compiler/compile/test_effective_runtime.py b/tests/unit/src/sqlbuild/compiler/compile/test_effective_runtime.py index bb8bc01f11..120fa04c5e 100644 --- a/tests/unit/src/sqlbuild/compiler/compile/test_effective_runtime.py +++ b/tests/unit/src/sqlbuild/compiler/compile/test_effective_runtime.py @@ -5,6 +5,7 @@ import pytest from sqlbuild._native import ( + DiscoveredProjectInputs, LocalConfig, LocalTargetConfig, ProjectConfig, @@ -12,7 +13,6 @@ build_effective_runtime_config, resolve_run_id, ) -from sqlbuild.compiler.discovery.models import DiscoveredProjectInputs from tests.unit.src.sqlbuild.compiler.compile._test_types import ( BuildEffectiveRuntimeConfigTestCase, RunIdGenerationTestCase, diff --git a/tests/unit/src/sqlbuild/compiler/compile/test_main.py b/tests/unit/src/sqlbuild/compiler/compile/test_main.py index 48712b6902..b182f19296 100644 --- a/tests/unit/src/sqlbuild/compiler/compile/test_main.py +++ b/tests/unit/src/sqlbuild/compiler/compile/test_main.py @@ -13,26 +13,24 @@ ClonePolicy, CompileModelConfig, DefaultsConfig, + DiscoveredProjectInputs, LocalClonePolicy, LocalConfig, LocalTargetConfig, MaterializationDefaultsConfig, MaterializationRetentionDefaults, ProjectConfig, + PythonHookEntry, ResolvedTimeTravelRetention, ResourceIdentityError, SpecConfigError, + SqlHookEntry, TargetConfig, TimeTravelRetentionSource, build_compile_inputs, discover_project_inputs, resolve_target_config, ) -from sqlbuild.compiler.discovery.models import ( - DiscoveredProjectInputs, - PythonHookEntry, - SqlHookEntry, -) from tests.unit.src.sqlbuild.compiler.compile._helpers.config.helpers import build_config_natively from tests.unit.src.sqlbuild.compiler.compile._helpers.helpers import ( DUCKDB_COMPILE_ADAPTER_CONTEXT, diff --git a/tests/unit/src/sqlbuild/compiler/compile/test_resolved_connection.py b/tests/unit/src/sqlbuild/compiler/compile/test_resolved_connection.py index a5c718b2d3..aacbaf32ff 100644 --- a/tests/unit/src/sqlbuild/compiler/compile/test_resolved_connection.py +++ b/tests/unit/src/sqlbuild/compiler/compile/test_resolved_connection.py @@ -6,8 +6,7 @@ import pytest -from sqlbuild._native import build_compile_inputs, discover_project_inputs -from sqlbuild.compiler.discovery.models import DiscoveredProjectInputs +from sqlbuild._native import DiscoveredProjectInputs, build_compile_inputs, discover_project_inputs from tests.unit.src.sqlbuild.compiler.compile._helpers.helpers import ( DUCKDB_COMPILE_ADAPTER_CONTEXT, ) diff --git a/tests/unit/src/sqlbuild/compiler/dag/main/helpers.py b/tests/unit/src/sqlbuild/compiler/dag/main/helpers.py index e3d2acdf3f..d68ba67363 100644 --- a/tests/unit/src/sqlbuild/compiler/dag/main/helpers.py +++ b/tests/unit/src/sqlbuild/compiler/dag/main/helpers.py @@ -9,10 +9,15 @@ CompileModelConfig, DiscoveredAuditBlock, DiscoveredAuditFile, + DiscoveredLoaderFunction, DiscoveredSchemaFile, DiscoveredSeedFile, + DiscoveredSourceFile, + DiscoveredSqlScenarioFile, DiscoveredSqlTestBlock, + DiscoveredSqlTestFile, FunctionArgument, + ProjectGraph, SchemaColumn, SchemaModelEntry, SchemaSeedEntry, @@ -35,13 +40,6 @@ CompiledResourceType, FunctionLanguage, ) -from sqlbuild.compiler.discovery.models import ( - DiscoveredLoaderFunction, - DiscoveredSourceFile, - DiscoveredSqlScenarioFile, - DiscoveredSqlTestFile, -) -from sqlbuild.compiler.pipeline.models import ProjectGraph from tests.unit.src.sqlbuild.compiler.graph.helpers import project_graph_from_indexes diff --git a/tests/unit/src/sqlbuild/compiler/dag/main/test_build.py b/tests/unit/src/sqlbuild/compiler/dag/main/test_build.py index ba96505a1a..d0c62a82e5 100644 --- a/tests/unit/src/sqlbuild/compiler/dag/main/test_build.py +++ b/tests/unit/src/sqlbuild/compiler/dag/main/test_build.py @@ -9,6 +9,8 @@ import pytest from sqlbuild._native import ( + DiscoveredProjectInputs, + ProjectGraph, assemble_project, build_compile_inputs, build_dag_json, @@ -21,8 +23,6 @@ ) from sqlbuild.compiler.compile.types import CompiledProjectView from sqlbuild.compiler.dag.types import NodeKind -from sqlbuild.compiler.discovery.models import DiscoveredProjectInputs -from sqlbuild.compiler.pipeline.models import ProjectGraph from tests.unit.src.sqlbuild.compiler.compile._helpers.helpers import ( DUCKDB_COMPILE_ADAPTER_CONTEXT, ) diff --git a/tests/unit/src/sqlbuild/compiler/discovery/_helpers/_test_helpers.py b/tests/unit/src/sqlbuild/compiler/discovery/_helpers/_test_helpers.py index db5f550b6e..6c95e35e99 100644 --- a/tests/unit/src/sqlbuild/compiler/discovery/_helpers/_test_helpers.py +++ b/tests/unit/src/sqlbuild/compiler/discovery/_helpers/_test_helpers.py @@ -12,6 +12,7 @@ import sqlbuild._native as _native from sqlbuild._native import ( + DiscoveredProjectInputs, discover_audit_files, discover_constant_files, discover_enum_files, @@ -24,7 +25,6 @@ discover_sql_hook_files, discover_test_files, ) -from sqlbuild.compiler.discovery.models import DiscoveredProjectInputs from tests.unit.src.sqlbuild.compiler.discovery._helpers.helpers import ( load_local_config, load_project_config, diff --git a/tests/unit/src/sqlbuild/compiler/discovery/_helpers/helpers.py b/tests/unit/src/sqlbuild/compiler/discovery/_helpers/helpers.py index 9dc0bcbff9..8c714f268f 100644 --- a/tests/unit/src/sqlbuild/compiler/discovery/_helpers/helpers.py +++ b/tests/unit/src/sqlbuild/compiler/discovery/_helpers/helpers.py @@ -6,11 +6,13 @@ from sqlbuild._native import ( DiscoveredConstantFile, DiscoveredEnumFile, + DiscoveredMacroFile, DiscoveredPythonFunctionFile, DiscoveredSqlTestBlock, DiscoveredSqlTestCase, LocalConfig, ProjectConfig, + SqlTestParameterDeclaration, discover_audit_files, discover_constant_files, discover_enum_files, @@ -20,7 +22,6 @@ discover_sql_function_files, discover_sql_hook_files, ) -from sqlbuild.compiler.discovery.models import DiscoveredMacroFile, SqlTestParameterDeclaration from tests.unit.src.sqlbuild.compiler.discovery._helpers._test_types import ( LoadProjectConfigTestCase, ) diff --git a/tests/unit/src/sqlbuild/compiler/discovery/_helpers/test_filesystem_hooks.py b/tests/unit/src/sqlbuild/compiler/discovery/_helpers/test_filesystem_hooks.py index 071c411e05..02f1f92e7e 100644 --- a/tests/unit/src/sqlbuild/compiler/discovery/_helpers/test_filesystem_hooks.py +++ b/tests/unit/src/sqlbuild/compiler/discovery/_helpers/test_filesystem_hooks.py @@ -14,6 +14,11 @@ DiscoveredHookFunction, DiscoveredProvider, DiscoveredProviderUsage, + PythonNodeDiscoveryError, +) +from sqlbuild.compiler.discovery._helpers.filesystem.core import ( + discover_hook_functions, + discover_provider_classes, ) from tests.unit.src.sqlbuild.compiler.discovery._helpers._test_types import ( DiscoverHookFunctionsErrorTestCase, diff --git a/tests/unit/src/sqlbuild/compiler/discovery/_helpers/test_filesystem_macros.py b/tests/unit/src/sqlbuild/compiler/discovery/_helpers/test_filesystem_macros.py index b73f54fb5d..afeda43c52 100644 --- a/tests/unit/src/sqlbuild/compiler/discovery/_helpers/test_filesystem_macros.py +++ b/tests/unit/src/sqlbuild/compiler/discovery/_helpers/test_filesystem_macros.py @@ -7,9 +7,9 @@ import pytest from sqlbuild._native import ( + DiscoveredMacroFile, discover_macro_files, ) -from sqlbuild.compiler.discovery.models import DiscoveredMacroFile from tests.unit.src.sqlbuild.compiler.discovery._helpers._test_types import ( DiscoverMacroFilesTestCase, ) diff --git a/tests/unit/src/sqlbuild/compiler/discovery/_helpers/test_filesystem_materializations.py b/tests/unit/src/sqlbuild/compiler/discovery/_helpers/test_filesystem_materializations.py index 6e031cbed8..110165c40a 100644 --- a/tests/unit/src/sqlbuild/compiler/discovery/_helpers/test_filesystem_materializations.py +++ b/tests/unit/src/sqlbuild/compiler/discovery/_helpers/test_filesystem_materializations.py @@ -15,6 +15,10 @@ DiscoveredProvider, DiscoveredProviderUsage, ) +from sqlbuild.compiler.discovery._helpers.filesystem.core import ( + discover_materialization_files, + discover_provider_classes, +) from tests.unit.src.sqlbuild.compiler.discovery._helpers._test_types import ( DiscoverMaterializationFilesTestCase, ) diff --git a/tests/unit/src/sqlbuild/compiler/discovery/_helpers/test_filesystem_python_nodes.py b/tests/unit/src/sqlbuild/compiler/discovery/_helpers/test_filesystem_python_nodes.py index e571870469..05bd85a827 100644 --- a/tests/unit/src/sqlbuild/compiler/discovery/_helpers/test_filesystem_python_nodes.py +++ b/tests/unit/src/sqlbuild/compiler/discovery/_helpers/test_filesystem_python_nodes.py @@ -19,6 +19,11 @@ DiscoveredCheckFunction, DiscoveredPythonNodeFunctions, DiscoveredTaskFunction, + PythonNodeDiscoveryError, + PythonNodeIdentity, +) +from sqlbuild.compiler.discovery._helpers.filesystem.core import ( + discover_python_node_functions, ) from sqlbuild.compiler.python_nodes.main.identity import build_python_node_identity from tests.unit.src.sqlbuild.compiler.discovery._helpers._test_types import ( diff --git a/tests/unit/src/sqlbuild/compiler/discovery/_helpers/test_filesystem_scoped_declarations.py b/tests/unit/src/sqlbuild/compiler/discovery/_helpers/test_filesystem_scoped_declarations.py index 86778b6f9f..7e94edd8b4 100644 --- a/tests/unit/src/sqlbuild/compiler/discovery/_helpers/test_filesystem_scoped_declarations.py +++ b/tests/unit/src/sqlbuild/compiler/discovery/_helpers/test_filesystem_scoped_declarations.py @@ -12,6 +12,8 @@ DeclarationParseError, DiscoveredConstantFile, DiscoveredEnumFile, + DiscoveredMacroFile, + DiscoveredProjectInputs, discover_audit_files, discover_hook_functions, discover_macro_files, diff --git a/tests/unit/src/sqlbuild/compiler/discovery/_helpers/test_sql_declarations.py b/tests/unit/src/sqlbuild/compiler/discovery/_helpers/test_sql_declarations.py index c466332472..a4d077b83a 100644 --- a/tests/unit/src/sqlbuild/compiler/discovery/_helpers/test_sql_declarations.py +++ b/tests/unit/src/sqlbuild/compiler/discovery/_helpers/test_sql_declarations.py @@ -9,15 +9,16 @@ import sqlbuild._native as _native from sqlbuild._native import ( AuthoredSqlValueCall, + ConstantDeclaration, DeclarationParseError, DiscoveredConstantFile, DiscoveredEnumFile, DiscoveredModelSchemaFile, + EnumDeclaration, ModelSchemaDeclaration, ResourceIdentityError, SqlValue, ) -from sqlbuild.compiler.discovery.models import ConstantDeclaration, EnumDeclaration from tests.unit.src.sqlbuild.compiler.discovery._helpers._test_types import ( DeclarationResourceIdentityErrorTestCase, ParseDeclarationFileErrorTestCase, diff --git a/tests/unit/src/sqlbuild/compiler/discovery/_helpers/test_sql_models.py b/tests/unit/src/sqlbuild/compiler/discovery/_helpers/test_sql_models.py index 9905d83633..bf6107ed94 100644 --- a/tests/unit/src/sqlbuild/compiler/discovery/_helpers/test_sql_models.py +++ b/tests/unit/src/sqlbuild/compiler/discovery/_helpers/test_sql_models.py @@ -8,13 +8,15 @@ from sqlbuild._native import ( AuthoredSqlSet, AuthoredSqlValueCall, + DiscoveredSqlModelFile, DiscoveryFileFault, NamedSqlHookEntry, + PythonHookEntry, SourceLocation, + SqlHookEntry, discover_model_files, model_output_column_locations, ) -from sqlbuild.compiler.discovery.models import DiscoveredSqlModelFile, PythonHookEntry, SqlHookEntry from tests.unit.src.sqlbuild.compiler.discovery._helpers._test_types import ( DeferredModelOutputLocationTestCase, ExpectedBooleanTestCase, diff --git a/tests/unit/src/sqlbuild/compiler/discovery/_helpers/test_sql_scenarios.py b/tests/unit/src/sqlbuild/compiler/discovery/_helpers/test_sql_scenarios.py index 9431ff703c..ac8b2b784a 100644 --- a/tests/unit/src/sqlbuild/compiler/discovery/_helpers/test_sql_scenarios.py +++ b/tests/unit/src/sqlbuild/compiler/discovery/_helpers/test_sql_scenarios.py @@ -4,8 +4,7 @@ import pytest -from sqlbuild._native import SqlScenarioParseError -from sqlbuild.compiler.discovery.models import DiscoveredSqlScenarioFile +from sqlbuild._native import DiscoveredSqlScenarioFile, SqlScenarioParseError from tests.unit.src.sqlbuild.compiler.discovery._helpers._test_types import ( ParseSqlScenarioFileErrorTestCase, ParseSqlScenarioFileTestCase, diff --git a/tests/unit/src/sqlbuild/compiler/discovery/_helpers/test_sql_tests.py b/tests/unit/src/sqlbuild/compiler/discovery/_helpers/test_sql_tests.py index 051cce0841..6aaaf361d3 100644 --- a/tests/unit/src/sqlbuild/compiler/discovery/_helpers/test_sql_tests.py +++ b/tests/unit/src/sqlbuild/compiler/discovery/_helpers/test_sql_tests.py @@ -9,13 +9,14 @@ from sqlbuild._native import ( DiscoveredSqlTestBlock, DiscoveredSqlTestCase, + DiscoveredSqlTestFile, DiscoveryFileFault, SqlLexicalSyntax, + SqlTestParameterDeclaration, SqlTestParseError, discover_test_files, omitted_ceremonial_select, ) -from sqlbuild.compiler.discovery.models import DiscoveredSqlTestFile, SqlTestParameterDeclaration from tests.unit.src.sqlbuild.compiler.discovery._helpers._test_types import ( ExpectedCountTestCase, OmittedSelectPerformanceTestCase, diff --git a/tests/unit/src/sqlbuild/compiler/discovery/_helpers/test_target_warehouses.py b/tests/unit/src/sqlbuild/compiler/discovery/_helpers/test_target_warehouses.py index 6a8407e8f8..413b040910 100644 --- a/tests/unit/src/sqlbuild/compiler/discovery/_helpers/test_target_warehouses.py +++ b/tests/unit/src/sqlbuild/compiler/discovery/_helpers/test_target_warehouses.py @@ -6,8 +6,12 @@ import pytest -from sqlbuild._native import ProjectConfigError, TargetWarehousesConfig, discover_project_inputs -from sqlbuild.compiler.discovery.models import DiscoveredProjectInputs +from sqlbuild._native import ( + DiscoveredProjectInputs, + ProjectConfigError, + TargetWarehousesConfig, + discover_project_inputs, +) from tests.unit.src.sqlbuild.compiler.discovery._helpers._test_types import ( TargetWarehousesConfigErrorTestCase, TargetWarehousesConfigTestCase, diff --git a/tests/unit/src/sqlbuild/compiler/discovery/helpers.py b/tests/unit/src/sqlbuild/compiler/discovery/helpers.py index edf8456e4b..a35a152be8 100644 --- a/tests/unit/src/sqlbuild/compiler/discovery/helpers.py +++ b/tests/unit/src/sqlbuild/compiler/discovery/helpers.py @@ -2,13 +2,14 @@ import sqlbuild._native as _native from sqlbuild._native import ( + DiscoveredSqlModelFile, + DiscoveredSqlScenarioFile, DiscoveredSqlTestBlock, SchemaModelEntry, SchemaSeedEntry, SourceEntry, SourceLocation, ) -from sqlbuild.compiler.discovery.models import DiscoveredSqlModelFile, DiscoveredSqlScenarioFile REMOVED_SQL_MODEL_HEADER_KEYS: frozenset[str] = frozenset({"run_despite_unchanged"}) SQL_MODEL_HEADER_KEYS: frozenset[str] = frozenset( diff --git a/tests/unit/src/sqlbuild/compiler/discovery/main/test_serialize_hook_entries.py b/tests/unit/src/sqlbuild/compiler/discovery/main/test_serialize_hook_entries.py index 853a33d646..4304bec241 100644 --- a/tests/unit/src/sqlbuild/compiler/discovery/main/test_serialize_hook_entries.py +++ b/tests/unit/src/sqlbuild/compiler/discovery/main/test_serialize_hook_entries.py @@ -4,8 +4,7 @@ import pytest -from sqlbuild._native import serialize_hook_entries -from sqlbuild.compiler.discovery.models import PythonHookEntry, SqlHookEntry +from sqlbuild._native import PythonHookEntry, SqlHookEntry, serialize_hook_entries from tests.unit.src.sqlbuild.compiler.discovery.main._test_types import ( SerializeHookEntriesTestCase, ) diff --git a/tests/unit/src/sqlbuild/compiler/discovery/test_main.py b/tests/unit/src/sqlbuild/compiler/discovery/test_main.py index 1ba30ebcf2..57fdddbf9e 100644 --- a/tests/unit/src/sqlbuild/compiler/discovery/test_main.py +++ b/tests/unit/src/sqlbuild/compiler/discovery/test_main.py @@ -7,12 +7,12 @@ import pytest from sqlbuild._native import ( + DiscoveredProjectInputs, DiscoveredSqlTestBlock, ProjectConfigError, ProjectPythonPathError, discover_project_inputs, ) -from sqlbuild.compiler.discovery.models import DiscoveredProjectInputs from sqlbuild.observability import EventDispatcher, LifecycleEvent, dispatcher_scope from tests.unit.src.sqlbuild.compiler.discovery._test_helpers import ( base_repo_files, diff --git a/tests/unit/src/sqlbuild/compiler/discovery/test_runtime_event_exporter_config.py b/tests/unit/src/sqlbuild/compiler/discovery/test_runtime_event_exporter_config.py index b1f5e23222..97cd46cb36 100644 --- a/tests/unit/src/sqlbuild/compiler/discovery/test_runtime_event_exporter_config.py +++ b/tests/unit/src/sqlbuild/compiler/discovery/test_runtime_event_exporter_config.py @@ -2,12 +2,12 @@ import pytest -from sqlbuild._native import ProjectConfigError -from sqlbuild.compiler.discovery.main.runtime_extensions import discover_runtime_extensions -from sqlbuild.compiler.discovery.models import ( +from sqlbuild._native import ( DiscoveredEventExporter, DiscoveredRuntimeExtensions, + ProjectConfigError, ) +from sqlbuild.compiler.discovery.main.runtime_extensions import discover_runtime_extensions from tests.unit.src.sqlbuild.compiler.discovery._test_types import ( EventExporterConfigTestCase, InvalidEventExporterConfigTestCase, diff --git a/tests/unit/src/sqlbuild/compiler/graph/helpers.py b/tests/unit/src/sqlbuild/compiler/graph/helpers.py index 55ae3cb760..25ac641513 100644 --- a/tests/unit/src/sqlbuild/compiler/graph/helpers.py +++ b/tests/unit/src/sqlbuild/compiler/graph/helpers.py @@ -4,10 +4,9 @@ from collections.abc import Iterable, Mapping -from sqlbuild._native import native_graph_from_views +from sqlbuild._native import ProjectGraph, native_graph_from_views from sqlbuild.compiler.compile.models import CompiledObjectKey from sqlbuild.compiler.compile.types import CompiledProjectView -from sqlbuild.compiler.pipeline.models import ProjectGraph def project_graph_from_indexes( diff --git a/tests/unit/src/sqlbuild/compiler/macro_bridge/_helpers/helpers.py b/tests/unit/src/sqlbuild/compiler/macro_bridge/_helpers/helpers.py index fbbcb1722b..2e798912c4 100644 --- a/tests/unit/src/sqlbuild/compiler/macro_bridge/_helpers/helpers.py +++ b/tests/unit/src/sqlbuild/compiler/macro_bridge/_helpers/helpers.py @@ -14,9 +14,8 @@ from pathlib import Path import sqlbuild._native as _native -from sqlbuild._native import SqlLogicalType, SqlValue, SqlValueKind +from sqlbuild._native import ConstantDeclaration, SqlLogicalType, SqlValue, SqlValueKind from sqlbuild.compiler.compile.models import DeclarationResolutionContext, LoadedMacro, MacroContext -from sqlbuild.compiler.discovery.models import ConstantDeclaration from sqlbuild.compiler.macro_bridge._helpers.store_environment import ( project_fingerprint, store_environment, diff --git a/tests/unit/src/sqlbuild/compiler/macro_bridge/main/helpers.py b/tests/unit/src/sqlbuild/compiler/macro_bridge/main/helpers.py index f51152461b..5d10440e96 100644 --- a/tests/unit/src/sqlbuild/compiler/macro_bridge/main/helpers.py +++ b/tests/unit/src/sqlbuild/compiler/macro_bridge/main/helpers.py @@ -5,8 +5,7 @@ from collections.abc import Callable from typing import cast -from sqlbuild._native import NativeStageMismatchError -from sqlbuild.compiler.compile.exceptions import CompileInputError +from sqlbuild._native import CompileInputError, NativeStageMismatchError from sqlbuild.compiler.macro_bridge.classes.macro_bridge import MacroBridge from sqlbuild.compiler.macro_bridge.main.active_macro_bridge import active_macro_bridge diff --git a/tests/unit/src/sqlbuild/compiler/manifest/helpers.py b/tests/unit/src/sqlbuild/compiler/manifest/helpers.py index 5264fb5851..9dd29ea9ec 100644 --- a/tests/unit/src/sqlbuild/compiler/manifest/helpers.py +++ b/tests/unit/src/sqlbuild/compiler/manifest/helpers.py @@ -14,6 +14,7 @@ CompileModelConfig, DiscoveredSchemaFile, DiscoveredSeedFile, + DiscoveredSourceFile, DiscoveredSqlHookFile, MaterializationType, ModelPlanEntry, @@ -48,7 +49,6 @@ CompiledSeedView, CompiledSourceView, ) -from sqlbuild.compiler.discovery.models import DiscoveredSourceFile def model_key(name: str) -> CompiledObjectKey: diff --git a/tests/unit/src/sqlbuild/compiler/manifest/test_main.py b/tests/unit/src/sqlbuild/compiler/manifest/test_main.py index dfa53b03e7..3a096c978d 100644 --- a/tests/unit/src/sqlbuild/compiler/manifest/test_main.py +++ b/tests/unit/src/sqlbuild/compiler/manifest/test_main.py @@ -23,6 +23,7 @@ ScopeIndex, ScopeKind, SourceColumnEntry, + SqlHookEntry, SqlTestPlanEntry, build_manifest, project_with, @@ -37,7 +38,6 @@ CompiledSeedView, CompiledSourceView, ) -from sqlbuild.compiler.discovery.models import SqlHookEntry from tests.unit.src.sqlbuild.compiler.manifest._test_types import ( FqnTestCase, ManifestAuditNodeTestCase, diff --git a/tests/unit/src/sqlbuild/compiler/pipeline/_helpers/target_validation/test_main.py b/tests/unit/src/sqlbuild/compiler/pipeline/_helpers/target_validation/test_main.py index d416cf0605..60a2e667ee 100644 --- a/tests/unit/src/sqlbuild/compiler/pipeline/_helpers/target_validation/test_main.py +++ b/tests/unit/src/sqlbuild/compiler/pipeline/_helpers/target_validation/test_main.py @@ -5,6 +5,8 @@ import pytest from sqlbuild._native import ( + DiscoveredProjectInputs, + DiscoveredSourceFile, LocalConfig, LocalTargetConfig, ProjectConfig, @@ -27,7 +29,6 @@ CompiledRelationLocation, ) from sqlbuild.compiler.compile.types import CompiledProjectView -from sqlbuild.compiler.discovery.models import DiscoveredProjectInputs, DiscoveredSourceFile from tests.unit.src.sqlbuild.compiler.pipeline._helpers.target_validation._test_types import ( ValidateManagedLoaderConnectionTestCase, ValidateManagedWriteSchemaTestCase, diff --git a/tests/unit/src/sqlbuild/compiler/pipeline/_helpers/test_clone.py b/tests/unit/src/sqlbuild/compiler/pipeline/_helpers/test_clone.py index 903b611bc9..e6c36f232e 100644 --- a/tests/unit/src/sqlbuild/compiler/pipeline/_helpers/test_clone.py +++ b/tests/unit/src/sqlbuild/compiler/pipeline/_helpers/test_clone.py @@ -6,13 +6,13 @@ ClonePipelineConnection, ClonePipelineOptions, ClonePipelineResult, + DiscoveredProjectInputs, LocalConfig, ProjectConfig, TargetConfig, run_clone_pipeline_with_options, ) from sqlbuild.adapters.postgres.classes.postgres_adapter import PostgresAdapter -from sqlbuild.compiler.discovery.models import DiscoveredProjectInputs from tests.unit.src.sqlbuild.compiler.pipeline._helpers._test_types import ( CloneConnectionInjectionTestCase, CloneTargetSchemaTestCase, diff --git a/tests/unit/src/sqlbuild/compiler/pipeline/main/helpers.py b/tests/unit/src/sqlbuild/compiler/pipeline/main/helpers.py index 2efd3e6ace..fb08e8786b 100644 --- a/tests/unit/src/sqlbuild/compiler/pipeline/main/helpers.py +++ b/tests/unit/src/sqlbuild/compiler/pipeline/main/helpers.py @@ -5,6 +5,7 @@ from sqlbuild._native import ( CompiledSource, + DiscoveredSourceFile, MaterializationType, ModelPlanEntry, PlanAction, @@ -19,7 +20,6 @@ CompiledRelationLocation, ) from sqlbuild.compiler.compile.types import CompiledProjectView, CompiledResourceType -from sqlbuild.compiler.discovery.models import DiscoveredSourceFile class RelationTargetTestAdapter(BaseAdapter): diff --git a/tests/unit/src/sqlbuild/compiler/pipeline/main/test_compile_phase_timings.py b/tests/unit/src/sqlbuild/compiler/pipeline/main/test_compile_phase_timings.py index 361c760458..4663874013 100644 --- a/tests/unit/src/sqlbuild/compiler/pipeline/main/test_compile_phase_timings.py +++ b/tests/unit/src/sqlbuild/compiler/pipeline/main/test_compile_phase_timings.py @@ -12,6 +12,7 @@ from sqlbuild._native import ( BuildPhaseTimingTracker, CompilePipelineOptions, + CompilePipelineResult, PartialBuildPhaseTimings, PlanOutput, RulesRunResult, @@ -19,7 +20,6 @@ from sqlbuild.compiler.compile.models import CompiledProject from sqlbuild.compiler.compile.types import CompiledProjectView from sqlbuild.compiler.pipeline._helpers import materializations -from sqlbuild.compiler.pipeline.models import CompilePipelineResult from sqlbuild.runtime.contracts.models import ConnectionHooks from tests.unit.src.sqlbuild.compiler.pipeline.main._test_types import PipelinePhaseTimingTestCase diff --git a/tests/unit/src/sqlbuild/compiler/planner/_helpers/_test_types.py b/tests/unit/src/sqlbuild/compiler/planner/_helpers/_test_types.py index 95f6312495..68862d8592 100644 --- a/tests/unit/src/sqlbuild/compiler/planner/_helpers/_test_types.py +++ b/tests/unit/src/sqlbuild/compiler/planner/_helpers/_test_types.py @@ -23,6 +23,7 @@ SourceEntry, SourceLoadPlanEntry, SqlLexicalSyntax, + SqlReferenceKind, WarningSeverity, ) from sqlbuild.compiler.auditing.types import ( @@ -35,7 +36,6 @@ InferredColumn, ) from sqlbuild.compiler.compile.types import AttachedAuditTargetKind, CompiledProjectView -from sqlbuild.compiler.references.types import SqlReferenceKind from tests.unit.src.sqlbuild.compiler.planner._helpers.sql_test_assembly._test_types import ( PlanTestChainTestCase as PlanTestChainTestCase, ) diff --git a/tests/unit/src/sqlbuild/compiler/planner/_helpers/changes/test_detect.py b/tests/unit/src/sqlbuild/compiler/planner/_helpers/changes/test_detect.py index 5263aa3ea9..9c2bde4352 100644 --- a/tests/unit/src/sqlbuild/compiler/planner/_helpers/changes/test_detect.py +++ b/tests/unit/src/sqlbuild/compiler/planner/_helpers/changes/test_detect.py @@ -16,6 +16,7 @@ PlannerScope, PlanReason, PlanWarning, + SqlHookEntry, WarehouseFingerprints, WarehouseSnapshot, WarningSeverity, @@ -29,7 +30,6 @@ ) from sqlbuild.compiler.compile.models import InferredColumn from sqlbuild.compiler.compile.types import CompiledModelView -from sqlbuild.compiler.discovery.models import SqlHookEntry from tests.unit.src.sqlbuild.compiler.planner._helpers.changes._test_helpers import ( build_function_caller_metadata_case, build_metadata_json_with_audit_gate, diff --git a/tests/unit/src/sqlbuild/compiler/planner/_helpers/helpers.py b/tests/unit/src/sqlbuild/compiler/planner/_helpers/helpers.py index 027529c245..7cedd317a1 100644 --- a/tests/unit/src/sqlbuild/compiler/planner/_helpers/helpers.py +++ b/tests/unit/src/sqlbuild/compiler/planner/_helpers/helpers.py @@ -22,14 +22,19 @@ CompiledSeed, CompiledSource, CompileModelConfig, + CompilePipelineResult, CompileSqlScenarioCte, CompileSqlTestCte, DefaultsConfig, DiscoveredAuditBlock, DiscoveredAuditFile, + DiscoveredLoaderFunction, DiscoveredSchemaFile, DiscoveredSeedFile, + DiscoveredSourceFile, + DiscoveredSqlScenarioFile, DiscoveredSqlTestBlock, + DiscoveredSqlTestFile, Fingerprint, FunctionArgument, PlannerChangeResults, @@ -45,6 +50,7 @@ SourceEntry, SourceWriteStrategy, SqlLexicalSyntax, + SqlReferenceKind, WarehouseSnapshot, build_downstream_deps, build_scenario_relation_map, @@ -78,14 +84,6 @@ CompiledSourceView, SqlTestMode, ) -from sqlbuild.compiler.discovery.models import ( - DiscoveredLoaderFunction, - DiscoveredSourceFile, - DiscoveredSqlScenarioFile, - DiscoveredSqlTestFile, -) -from sqlbuild.compiler.pipeline.models import CompilePipelineResult -from sqlbuild.compiler.references.types import SqlReferenceKind from tests.unit.src.sqlbuild.compiler.planner._helpers._test_types import ( BuildModelWarningsTestCase, IncrementalStrategyErrorTestCase, diff --git a/tests/unit/src/sqlbuild/compiler/planner/_helpers/resolve/test_hooks.py b/tests/unit/src/sqlbuild/compiler/planner/_helpers/resolve/test_hooks.py index 4e4e38a8d7..c00449ebb0 100644 --- a/tests/unit/src/sqlbuild/compiler/planner/_helpers/resolve/test_hooks.py +++ b/tests/unit/src/sqlbuild/compiler/planner/_helpers/resolve/test_hooks.py @@ -8,12 +8,13 @@ from sqlbuild._native import ( ModelPlanContext, + PythonHookEntry, SourceEntry, + SqlHookEntry, resolve_model_hook_entries, ) from sqlbuild.adapters.duckdb.classes.duckdb_adapter import DuckDbAdapter from sqlbuild.compiler.compile.types import CompiledModelView -from sqlbuild.compiler.discovery.models import PythonHookEntry, SqlHookEntry from tests.unit.src.sqlbuild.compiler.planner._helpers.resolve._test_types import ( HookSqlResolutionTestCase, ) diff --git a/tests/unit/src/sqlbuild/compiler/planner/_helpers/selection/helpers.py b/tests/unit/src/sqlbuild/compiler/planner/_helpers/selection/helpers.py index 1858c9a743..e7f3904331 100644 --- a/tests/unit/src/sqlbuild/compiler/planner/_helpers/selection/helpers.py +++ b/tests/unit/src/sqlbuild/compiler/planner/_helpers/selection/helpers.py @@ -2,7 +2,7 @@ from types import SimpleNamespace from typing import cast -from sqlbuild.compiler.discovery.models import DiscoveredProjectInputs +from sqlbuild._native import DiscoveredProjectInputs def build_selector_discovered_inputs( diff --git a/tests/unit/src/sqlbuild/compiler/planner/_helpers/selection/test_unit_test_selectors.py b/tests/unit/src/sqlbuild/compiler/planner/_helpers/selection/test_unit_test_selectors.py index 0bfda38ba4..5fc6d33ba1 100644 --- a/tests/unit/src/sqlbuild/compiler/planner/_helpers/selection/test_unit_test_selectors.py +++ b/tests/unit/src/sqlbuild/compiler/planner/_helpers/selection/test_unit_test_selectors.py @@ -5,12 +5,12 @@ import pytest from sqlbuild._native import ( + DiscoveredProjectInputs, PlannerInputError, SqlTestSelection, UnitTestSelectorSplit, split_project_unit_test_selectors, ) -from sqlbuild.compiler.discovery.models import DiscoveredProjectInputs from tests.unit.src.sqlbuild.compiler.planner._helpers.selection._test_types import ( UnitTestSelectorErrorTestCase, UnitTestSelectorSplitTestCase, diff --git a/tests/unit/src/sqlbuild/compiler/planner/_helpers/sql_test_assembly/helpers.py b/tests/unit/src/sqlbuild/compiler/planner/_helpers/sql_test_assembly/helpers.py index 0cb86900ee..e750b392f4 100644 --- a/tests/unit/src/sqlbuild/compiler/planner/_helpers/sql_test_assembly/helpers.py +++ b/tests/unit/src/sqlbuild/compiler/planner/_helpers/sql_test_assembly/helpers.py @@ -13,9 +13,11 @@ CompileModelConfig, CompileSqlTestCte, DiscoveredSqlTestBlock, + DiscoveredSqlTestFile, NativeSqlTestPlan, PlanWarning, SqlLexicalSyntax, + SqlReferenceKind, SqlTestPlanEntry, SqlTestPlanResult, expand_sql_macros, @@ -44,8 +46,6 @@ CompiledProjectView, CompiledResourceType, ) -from sqlbuild.compiler.discovery.models import DiscoveredSqlTestFile -from sqlbuild.compiler.references.types import SqlReferenceKind from sqlbuild.compiler.sql_analysis.main.import_polyglot_sql import import_polyglot_sql from tests.unit.src.sqlbuild.compiler.planner._helpers.sql_test_assembly._test_types import ( PlanTestChainTestCase, diff --git a/tests/unit/src/sqlbuild/compiler/planner/_helpers/sql_test_assembly/test_plan_test.py b/tests/unit/src/sqlbuild/compiler/planner/_helpers/sql_test_assembly/test_plan_test.py index 905098b82f..a450ff53b0 100644 --- a/tests/unit/src/sqlbuild/compiler/planner/_helpers/sql_test_assembly/test_plan_test.py +++ b/tests/unit/src/sqlbuild/compiler/planner/_helpers/sql_test_assembly/test_plan_test.py @@ -10,8 +10,10 @@ CompiledDirectLogicSqlTestPayload, CompiledFunction, CompiledModelSqlTestPayload, + CompileInputError, CompileSqlTestCte, DiscoveredSqlTestBlock, + DiscoveredSqlTestFile, NativeSqlTestPlan, PlannerInputError, PlannerScope, @@ -20,6 +22,7 @@ PlanWarning, SqlTestPlanEntry, SqlTestPlanResult, + SqlTestReferenceError, WarningSeverity, build_test_command_plan, plan_compiled_sql_test_artifacts, @@ -32,7 +35,6 @@ from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter from sqlbuild.adapters.duckdb.classes.duckdb_adapter import DuckDbAdapter from sqlbuild.adapters.sqlserver.classes.sqlserver_adapter import SqlServerAdapter -from sqlbuild.compiler.compile.exceptions import CompileInputError, SqlTestReferenceError from sqlbuild.compiler.compile.models import ( CompiledObjectKey, CompiledProject, @@ -40,7 +42,6 @@ CompiledSqlTest, ) from sqlbuild.compiler.compile.types import CompiledProjectView, CompiledResourceType, SqlTestMode -from sqlbuild.compiler.discovery.models import DiscoveredSqlTestFile from tests.unit.src.sqlbuild.compiler.planner._helpers.sql_test_assembly._test_types import ( AssertionChainCteErrorTestCase, NativePlanningDifferentialTestCase, diff --git a/tests/unit/src/sqlbuild/compiler/planner/_helpers/test_audit_entry.py b/tests/unit/src/sqlbuild/compiler/planner/_helpers/test_audit_entry.py index 7694e02859..7f7a371ac7 100644 --- a/tests/unit/src/sqlbuild/compiler/planner/_helpers/test_audit_entry.py +++ b/tests/unit/src/sqlbuild/compiler/planner/_helpers/test_audit_entry.py @@ -2,7 +2,7 @@ import pytest -from sqlbuild._native import AuditPlanEntry, SourceEntry, plan_audit +from sqlbuild._native import AuditPlanEntry, SourceEntry, SqlReferenceKind, plan_audit from sqlbuild.compiler.auditing.types import AuditAttachmentKind, AuditRunScope from sqlbuild.compiler.compile.models import ( CompiledObjectKey, @@ -10,7 +10,6 @@ CompileSqlReference, ) from sqlbuild.compiler.compile.types import AttachedAuditTargetKind, CompiledAuditView -from sqlbuild.compiler.references.types import SqlReferenceKind from tests.unit.src.sqlbuild.compiler.planner._helpers._test_types import ( PlanAttachedAuditTestCase, PlanAuditTestCase, diff --git a/tests/unit/src/sqlbuild/compiler/planner/_helpers/test_audit_scheduling.py b/tests/unit/src/sqlbuild/compiler/planner/_helpers/test_audit_scheduling.py index 4c3a44ea67..5c1a2b9f75 100644 --- a/tests/unit/src/sqlbuild/compiler/planner/_helpers/test_audit_scheduling.py +++ b/tests/unit/src/sqlbuild/compiler/planner/_helpers/test_audit_scheduling.py @@ -6,6 +6,7 @@ from sqlbuild._native import ( MaterializationType, + SqlReferenceKind, reads_outside_target_lineage, resolve_attachment_kind, resolve_effective_run_scope, @@ -22,7 +23,6 @@ AttachedAuditTargetKind, CompiledAuditView, ) -from sqlbuild.compiler.references.types import SqlReferenceKind from tests.unit.src.sqlbuild.compiler.planner._helpers._test_types import ( ResolveAttachmentErrorTestCase, ResolveAttachmentTestCase, diff --git a/tests/unit/src/sqlbuild/compiler/planner/_helpers/test_fixture_planning_work.py b/tests/unit/src/sqlbuild/compiler/planner/_helpers/test_fixture_planning_work.py index 1aa809c355..c13b233337 100644 --- a/tests/unit/src/sqlbuild/compiler/planner/_helpers/test_fixture_planning_work.py +++ b/tests/unit/src/sqlbuild/compiler/planner/_helpers/test_fixture_planning_work.py @@ -14,6 +14,7 @@ RelationFixtureCompletion, RelationFixturePlanningContext, SettingsConfig, + SqlReferenceKind, SqlTestPlanEntry, SqlTestSelection, build_selected_test_entries, @@ -30,7 +31,6 @@ CompiledProjectView, CompiledResourceType, ) -from sqlbuild.compiler.references.types import SqlReferenceKind from tests.unit.src.sqlbuild.compiler.planner._helpers._test_types import ( CompleteFixturePlanningTestCase, FixturePlanningWorkTestCase, diff --git a/tests/unit/src/sqlbuild/compiler/planner/_helpers/test_microbatch_limit.py b/tests/unit/src/sqlbuild/compiler/planner/_helpers/test_microbatch_limit.py index 522b35b6e0..667c2526f5 100644 --- a/tests/unit/src/sqlbuild/compiler/planner/_helpers/test_microbatch_limit.py +++ b/tests/unit/src/sqlbuild/compiler/planner/_helpers/test_microbatch_limit.py @@ -5,6 +5,7 @@ import pytest from sqlbuild._native import ( + CompileInputError, CursorBounds, CursorInputRelation, IncrementalMode, @@ -19,7 +20,6 @@ apply_microbatch_limit, ) from sqlbuild._native import cursor_sentinel_to_token as sentinel_to_token -from sqlbuild.compiler.compile.exceptions import CompileInputError from sqlbuild.compiler.compile.models import ( CompiledObjectKey, CompiledRelationLocation, diff --git a/tests/unit/src/sqlbuild/compiler/planner/_helpers/test_scenario_cli.py b/tests/unit/src/sqlbuild/compiler/planner/_helpers/test_scenario_cli.py index 5ba18dc8dd..c42aac231c 100644 --- a/tests/unit/src/sqlbuild/compiler/planner/_helpers/test_scenario_cli.py +++ b/tests/unit/src/sqlbuild/compiler/planner/_helpers/test_scenario_cli.py @@ -6,13 +6,13 @@ import pytest from sqlbuild._native import ( + CompilePipelineResult, ScenarioArtifactName, ScenarioExecutionPlan, SqlLexicalSyntax, build_cli_scenario_plan, ) from sqlbuild.compiler.compile.models import CompiledSqlScenario -from sqlbuild.compiler.pipeline.models import CompilePipelineResult from tests.unit.src.sqlbuild.compiler.planner._helpers._test_types import ( ScenarioCliPlanIdentifierLimitTestCase, ) diff --git a/tests/unit/src/sqlbuild/compiler/planner/_helpers/test_scenario_relations.py b/tests/unit/src/sqlbuild/compiler/planner/_helpers/test_scenario_relations.py index fa88a184cb..16ad72384b 100644 --- a/tests/unit/src/sqlbuild/compiler/planner/_helpers/test_scenario_relations.py +++ b/tests/unit/src/sqlbuild/compiler/planner/_helpers/test_scenario_relations.py @@ -8,6 +8,7 @@ from sqlbuild._native import ( CompileSqlScenarioCte, + DiscoveredHookFunction, PlannerInputError, ScenarioExecutionPlan, ScenarioFixturePlan, @@ -28,7 +29,6 @@ from sqlbuild.adapters.sqlserver.classes.sqlserver_adapter import SqlServerAdapter from sqlbuild.compiler.compile.models import CompiledSqlScenario from sqlbuild.compiler.compile.types import CompiledProjectView -from sqlbuild.compiler.discovery.models import DiscoveredHookFunction from tests.unit.src.sqlbuild.compiler.planner._helpers._test_types import ( ScenarioCheckSqlResolutionErrorTestCase, ScenarioCheckSqlResolutionTestCase, diff --git a/tests/unit/src/sqlbuild/compiler/planner/_helpers/test_source_columns.py b/tests/unit/src/sqlbuild/compiler/planner/_helpers/test_source_columns.py index 34e55fdd08..7549f16ccb 100644 --- a/tests/unit/src/sqlbuild/compiler/planner/_helpers/test_source_columns.py +++ b/tests/unit/src/sqlbuild/compiler/planner/_helpers/test_source_columns.py @@ -12,6 +12,7 @@ PlannerRelationsContext, PlannerScope, SourceEntry, + SqlReferenceKind, build_cursor_input_relations, build_planner_relations_context, gather_source_columns, @@ -31,7 +32,6 @@ CompiledProjectView, CompiledResourceType, ) -from sqlbuild.compiler.references.types import SqlReferenceKind from tests.unit.src.sqlbuild.compiler.planner._helpers._test_types import ( CursorRuntimeOwnershipTestCase, KnownSourceColumnsReuseTestCase, diff --git a/tests/unit/src/sqlbuild/compiler/planner/_helpers/test_source_load_nodes.py b/tests/unit/src/sqlbuild/compiler/planner/_helpers/test_source_load_nodes.py index f2d27e4e57..46bb990b65 100644 --- a/tests/unit/src/sqlbuild/compiler/planner/_helpers/test_source_load_nodes.py +++ b/tests/unit/src/sqlbuild/compiler/planner/_helpers/test_source_load_nodes.py @@ -9,6 +9,7 @@ from sqlbuild._native import ( CompileModelConfig, + DiscoveredLoaderFunction, SourceEntry, SourceLoadPlanEntry, SourceWriteStrategy, @@ -31,7 +32,6 @@ CompiledProjectView, CompiledResourceType, ) -from sqlbuild.compiler.discovery.models import DiscoveredLoaderFunction from sqlbuild.runtime.contracts.types import ExecutionResourceKind from tests.unit.src.sqlbuild.compiler.planner._helpers._test_types import ( LoaderDagExpansionTestCase, diff --git a/tests/unit/src/sqlbuild/compiler/planner/_helpers/warehouse/helpers.py b/tests/unit/src/sqlbuild/compiler/planner/_helpers/warehouse/helpers.py index 5bbfaec0a1..c23d5c4d3d 100644 --- a/tests/unit/src/sqlbuild/compiler/planner/_helpers/warehouse/helpers.py +++ b/tests/unit/src/sqlbuild/compiler/planner/_helpers/warehouse/helpers.py @@ -8,6 +8,7 @@ from sqlbuild._native import ( CompiledSource, + DiscoveredSourceFile, PlannerRelationsContext, PlannerScope, SourceEntry, @@ -17,7 +18,6 @@ from sqlbuild.adapter.contract.models import ColumnInfo from sqlbuild.compiler.compile.models import CompiledObjectKey, CompiledProject from sqlbuild.compiler.compile.types import CompiledResourceType, CompiledSourceView -from sqlbuild.compiler.discovery.models import DiscoveredSourceFile def column_names(columns: tuple[ColumnInfo, ...]) -> tuple[str, ...]: diff --git a/tests/unit/src/sqlbuild/compiler/planner/main/identity/test_version_identity_hooks.py b/tests/unit/src/sqlbuild/compiler/planner/main/identity/test_version_identity_hooks.py index b84f7ff474..0b107bd78a 100644 --- a/tests/unit/src/sqlbuild/compiler/planner/main/identity/test_version_identity_hooks.py +++ b/tests/unit/src/sqlbuild/compiler/planner/main/identity/test_version_identity_hooks.py @@ -5,7 +5,7 @@ import pytest -from sqlbuild.compiler.discovery.models import PythonHookEntry, SqlHookEntry +from sqlbuild._native import PythonHookEntry, SqlHookEntry from tests.unit.src.sqlbuild.compiler.planner.main.identity._test_types import ( HookIdentityPayloadTestCase, HookIdentityTestCase, diff --git a/tests/unit/src/sqlbuild/compiler/planner/main/test_execution.py b/tests/unit/src/sqlbuild/compiler/planner/main/test_execution.py index 5d5ac90cc1..af817c7f81 100644 --- a/tests/unit/src/sqlbuild/compiler/planner/main/test_execution.py +++ b/tests/unit/src/sqlbuild/compiler/planner/main/test_execution.py @@ -6,6 +6,7 @@ import pytest from sqlbuild._native import ( + DiscoveredHookFunction, ModelPlanEntry, PlanAction, PlannerInputError, @@ -16,7 +17,6 @@ from sqlbuild.adapters.duckdb.classes.duckdb_adapter import DuckDbAdapter from sqlbuild.compiler.compile.models import CompiledProject from sqlbuild.compiler.compile.types import CompiledProjectView -from sqlbuild.compiler.discovery.models import DiscoveredHookFunction from tests.unit.src.sqlbuild.compiler.planner.main._test_types import ( DirectSourceFreshnessPlanOutputTestCase, ExternalBlockedPlanOutputTestCase, diff --git a/tests/unit/src/sqlbuild/compiler/python_nodes/_helpers/helpers.py b/tests/unit/src/sqlbuild/compiler/python_nodes/_helpers/helpers.py index 87a9179fe7..7966f84e3e 100644 --- a/tests/unit/src/sqlbuild/compiler/python_nodes/_helpers/helpers.py +++ b/tests/unit/src/sqlbuild/compiler/python_nodes/_helpers/helpers.py @@ -13,8 +13,16 @@ from sqlbuild._native import ( CompiledSource, CompileModelConfig, + DiscoveredAssetFunction, + DiscoveredCheckFunction, + DiscoveredLoaderFunction, + DiscoveredProjectInputs, + DiscoveredSourceFile, + DiscoveredTaskFunction, + LoaderConnectionMode, LocalConfig, ProjectConfig, + ProjectGraph, PythonNodeGraph, PythonSqlRunSelection, SourceColumnEntry, @@ -35,16 +43,6 @@ CompiledResourceType, CompiledSourceView, ) -from sqlbuild.compiler.discovery.models import ( - DiscoveredAssetFunction, - DiscoveredCheckFunction, - DiscoveredLoaderFunction, - DiscoveredProjectInputs, - DiscoveredSourceFile, - DiscoveredTaskFunction, -) -from sqlbuild.compiler.discovery.types import LoaderConnectionMode -from sqlbuild.compiler.pipeline.models import ProjectGraph from sqlbuild.providers import Provider from sqlbuild.python_nodes.models import SqlResourceRef from sqlbuild.python_nodes.types import PythonCheckSeverity diff --git a/tests/unit/src/sqlbuild/compiler/python_nodes/_helpers/test_inventory.py b/tests/unit/src/sqlbuild/compiler/python_nodes/_helpers/test_inventory.py index 55e0862df4..4d6878bf34 100644 --- a/tests/unit/src/sqlbuild/compiler/python_nodes/_helpers/test_inventory.py +++ b/tests/unit/src/sqlbuild/compiler/python_nodes/_helpers/test_inventory.py @@ -4,13 +4,21 @@ import json from collections import defaultdict +from dataclasses import asdict from pathlib import Path from typing import cast import pytest from sqlbuild._native import ( + DiscoveredAssetFunction, + DiscoveredCheckFunction, + DiscoveredLoaderFunction, + DiscoveredProjectInputs, + DiscoveredProvider, DiscoveredPythonNode, + DiscoveredTaskFunction, + LoaderConnectionMode, LocalConfig, ProjectConfig, PythonNodeGraph, @@ -18,15 +26,6 @@ SourceColumnEntry, build_python_node_graph, ) -from sqlbuild.compiler.discovery.models import ( - DiscoveredAssetFunction, - DiscoveredCheckFunction, - DiscoveredLoaderFunction, - DiscoveredProjectInputs, - DiscoveredProvider, - DiscoveredTaskFunction, -) -from sqlbuild.compiler.discovery.types import LoaderConnectionMode from sqlbuild.compiler.python_nodes.types import PythonNodeKind from sqlbuild.python_nodes.models import RetryPolicy from sqlbuild.python_nodes.types import PythonCheckSeverity @@ -215,7 +214,7 @@ def test_given_discovered_python_functions_when_building_graph_then_indexes_node assert tuple(usage.annotation_module for usage in notify_node.provider_usages) == ( test_case.expected_provider_usage_annotation_modules ) - assert tuple(vars(usage) for usage in notify_node.provider_usages) == ( + assert tuple(asdict(usage) for usage in notify_node.provider_usages) == ( { "provider_name": "slack_provider", "parameter_name": "slack_provider", diff --git a/tests/unit/src/sqlbuild/compiler/python_nodes/_helpers/test_run_selectors.py b/tests/unit/src/sqlbuild/compiler/python_nodes/_helpers/test_run_selectors.py index b77e06aa77..c36f6956be 100644 --- a/tests/unit/src/sqlbuild/compiler/python_nodes/_helpers/test_run_selectors.py +++ b/tests/unit/src/sqlbuild/compiler/python_nodes/_helpers/test_run_selectors.py @@ -4,8 +4,7 @@ import pytest -from sqlbuild._native import PythonNodeGraph, PythonSqlRunSelection -from sqlbuild.compiler.pipeline.models import ProjectGraph +from sqlbuild._native import ProjectGraph, PythonNodeGraph, PythonSqlRunSelection from tests.unit.src.sqlbuild.compiler.python_nodes._helpers._test_types import ( PythonSqlSelectorErrorTestCase, PythonSqlSelectorTestCase, diff --git a/tests/unit/src/sqlbuild/compiler/python_nodes/_helpers/test_unified_selectors.py b/tests/unit/src/sqlbuild/compiler/python_nodes/_helpers/test_unified_selectors.py index ca2d6ae1a0..4f3f0dad32 100644 --- a/tests/unit/src/sqlbuild/compiler/python_nodes/_helpers/test_unified_selectors.py +++ b/tests/unit/src/sqlbuild/compiler/python_nodes/_helpers/test_unified_selectors.py @@ -4,9 +4,13 @@ import pytest -from sqlbuild._native import PythonNodeGraph, PythonSqlRunSelection, resolve_project_selectors +from sqlbuild._native import ( + ProjectGraph, + PythonNodeGraph, + PythonSqlRunSelection, + resolve_project_selectors, +) from sqlbuild.compiler.compile.models import CompiledObjectKey -from sqlbuild.compiler.pipeline.models import ProjectGraph from sqlbuild.refs import seed from tests.unit.src.sqlbuild.compiler.python_nodes._helpers._test_types import ( PythonSqlSelectorErrorTestCase, diff --git a/tests/unit/src/sqlbuild/compiler/references/_test_types.py b/tests/unit/src/sqlbuild/compiler/references/_test_types.py index 30174e00c8..f5ce399907 100644 --- a/tests/unit/src/sqlbuild/compiler/references/_test_types.py +++ b/tests/unit/src/sqlbuild/compiler/references/_test_types.py @@ -5,7 +5,7 @@ from dataclasses import dataclass from typing import Literal -from sqlbuild.compiler.references.types import SqlReferenceKind +from sqlbuild._native import SqlReferenceKind @dataclass(frozen=True) diff --git a/tests/unit/src/sqlbuild/compiler/references/test_types.py b/tests/unit/src/sqlbuild/compiler/references/test_types.py index 71421fc6f6..9980caed45 100644 --- a/tests/unit/src/sqlbuild/compiler/references/test_types.py +++ b/tests/unit/src/sqlbuild/compiler/references/test_types.py @@ -4,7 +4,7 @@ import pytest -from sqlbuild.compiler.references.types import SqlReferenceKind +from sqlbuild._native import SqlReferenceKind from tests.unit.src.sqlbuild.compiler.references._test_types import ( SqlReferenceKindExampleCallTestCase, ) diff --git a/tests/unit/src/sqlbuild/compiler/scopes/helpers.py b/tests/unit/src/sqlbuild/compiler/scopes/helpers.py index c25736e064..f5fc0720a4 100644 --- a/tests/unit/src/sqlbuild/compiler/scopes/helpers.py +++ b/tests/unit/src/sqlbuild/compiler/scopes/helpers.py @@ -12,6 +12,9 @@ DeclarationIdentity, DeclarationKind, DeclarationRecord, + DiscoveredProjectInputs, + DiscoveredSqlModelFile, + EnumDeclaration, EnumMemberMetadata, EnumMetadata, GrantRecord, @@ -31,11 +34,6 @@ build_scope_lookup, get_placement_validated_scope_index, ) -from sqlbuild.compiler.discovery.models import ( - DiscoveredProjectInputs, - DiscoveredSqlModelFile, - EnumDeclaration, -) SCOPE_CACHE_PROJECT: str = 'name = "demo"\nadapter = "duckdb"\n' SCOPE_CACHE_MODEL: str = "MODEL(description 'Test model.');\nSELECT 1 AS id\n" diff --git a/tests/unit/src/sqlbuild/compiler/scopes/test_builder_and_visibility.py b/tests/unit/src/sqlbuild/compiler/scopes/test_builder_and_visibility.py index fd4fe7233a..e52bbf58f7 100644 --- a/tests/unit/src/sqlbuild/compiler/scopes/test_builder_and_visibility.py +++ b/tests/unit/src/sqlbuild/compiler/scopes/test_builder_and_visibility.py @@ -7,6 +7,7 @@ import pytest from sqlbuild._native import ( + ConstantDeclaration, DeclarationIdentity, DeclarationKind, DeclarationRecord, @@ -14,9 +15,14 @@ DiscoveredAuditFile, DiscoveredConstantFile, DiscoveredEnumFile, + DiscoveredMacroFile, + DiscoveredSourceFile, DiscoveredSqlFunctionFile, DiscoveredSqlHookFile, + DiscoveredSqlScenarioFile, DiscoveredSqlTestBlock, + DiscoveredSqlTestFile, + EnumDeclaration, EnumMember, InaccessibleReason, OwnershipRoot, @@ -40,14 +46,6 @@ validate_scope_index, ) from sqlbuild.compiler.compile.models import LoadedMacro -from sqlbuild.compiler.discovery.models import ( - ConstantDeclaration, - DiscoveredMacroFile, - DiscoveredSourceFile, - DiscoveredSqlScenarioFile, - DiscoveredSqlTestFile, - EnumDeclaration, -) from tests.unit.src.sqlbuild.compiler.scopes._test_types import ( ExpectedBooleanCase, ResourceExpectation, diff --git a/tests/unit/src/sqlbuild/compiler/sql_analysis/_helpers/test_scanning.py b/tests/unit/src/sqlbuild/compiler/sql_analysis/_helpers/test_scanning.py index dbd5725a51..426131f98c 100644 --- a/tests/unit/src/sqlbuild/compiler/sql_analysis/_helpers/test_scanning.py +++ b/tests/unit/src/sqlbuild/compiler/sql_analysis/_helpers/test_scanning.py @@ -3,8 +3,7 @@ import pytest import sqlbuild._native as _native -from sqlbuild._native import SqlLexicalSyntax -from sqlbuild.compiler.compile.exceptions import CompileInputError +from sqlbuild._native import CompileInputError, SqlLexicalSyntax from tests.unit.src.sqlbuild.compiler.sql_analysis._helpers._test_types import ( DialectMatchingParenTestCase, DialectNonCodeErrorTestCase, diff --git a/tests/unit/src/sqlbuild/executor/build/_helpers/_test_types.py b/tests/unit/src/sqlbuild/executor/build/_helpers/_test_types.py index 9e942d49a5..e30da1baae 100644 --- a/tests/unit/src/sqlbuild/executor/build/_helpers/_test_types.py +++ b/tests/unit/src/sqlbuild/executor/build/_helpers/_test_types.py @@ -1,10 +1,15 @@ from collections.abc import Callable from dataclasses import dataclass -from sqlbuild._native import MaterializationType, PlanAction, RetentionDirection, RetentionPlanPhase +from sqlbuild._native import ( + DiscoveredLoaderFunction, + MaterializationType, + PlanAction, + RetentionDirection, + RetentionPlanPhase, +) from sqlbuild.adapter.contract.models import RelationInfo from sqlbuild.adapter.contract.types import RetentionChangePhase -from sqlbuild.compiler.discovery.models import DiscoveredLoaderFunction from sqlbuild.executor.build.types import BuildStatus from sqlbuild.executor.scheduling.types import ExecutionStatus from sqlbuild.runtime.contracts.types import ExecutionResourceKind diff --git a/tests/unit/src/sqlbuild/executor/build/_helpers/helpers.py b/tests/unit/src/sqlbuild/executor/build/_helpers/helpers.py index d46eaefea6..eb4d31cf74 100644 --- a/tests/unit/src/sqlbuild/executor/build/_helpers/helpers.py +++ b/tests/unit/src/sqlbuild/executor/build/_helpers/helpers.py @@ -7,6 +7,7 @@ from unittest.mock import Mock from sqlbuild._native import ( + DiscoveredLoaderFunction, MaterializationType, ModelPlanEntry, PlanAction, @@ -31,7 +32,6 @@ CompiledRelationLocation, ) from sqlbuild.compiler.compile.types import CompiledResourceType -from sqlbuild.compiler.discovery.models import DiscoveredLoaderFunction from sqlbuild.executor.auditing.models import AuditExecutionResult diff --git a/tests/unit/src/sqlbuild/executor/build/_helpers/test_external_source_loads.py b/tests/unit/src/sqlbuild/executor/build/_helpers/test_external_source_loads.py index 13e24c96f7..f5f7a729b8 100644 --- a/tests/unit/src/sqlbuild/executor/build/_helpers/test_external_source_loads.py +++ b/tests/unit/src/sqlbuild/executor/build/_helpers/test_external_source_loads.py @@ -6,11 +6,15 @@ import pytest -from sqlbuild._native import PlanOutput, SourceEntry, SourceLoadPlanEntry +from sqlbuild._native import ( + DiscoveredLoaderFunction, + LoaderConnectionMode, + PlanOutput, + SourceEntry, + SourceLoadPlanEntry, +) from sqlbuild.compiler.compile.models import CompiledObjectKey from sqlbuild.compiler.compile.types import CompiledResourceType -from sqlbuild.compiler.discovery.models import DiscoveredLoaderFunction -from sqlbuild.compiler.discovery.types import LoaderConnectionMode from sqlbuild.executor.build._helpers.external_source_loads import ( run_external_source_loads_before_connections, ) diff --git a/tests/unit/src/sqlbuild/executor/build/_helpers/test_scheduler.py b/tests/unit/src/sqlbuild/executor/build/_helpers/test_scheduler.py index dd87ba00c6..ff91e03455 100644 --- a/tests/unit/src/sqlbuild/executor/build/_helpers/test_scheduler.py +++ b/tests/unit/src/sqlbuild/executor/build/_helpers/test_scheduler.py @@ -10,11 +10,14 @@ import pytest from sqlbuild._native import ( + DiscoveredHookFunction, + DiscoveredLoaderFunction, ModelPlanEntry, NativeProgressProjector, PlanAction, PlanOutput, PlanReason, + PythonHookEntry, SourceEntry, SourceLoadPlanEntry, SourceWriteStrategy, @@ -23,11 +26,6 @@ from sqlbuild.adapters.duckdb.classes.duckdb_adapter import DuckDbAdapter from sqlbuild.compiler.compile.models import CompiledObjectKey from sqlbuild.compiler.compile.types import CompiledResourceType -from sqlbuild.compiler.discovery.models import ( - DiscoveredHookFunction, - DiscoveredLoaderFunction, - PythonHookEntry, -) from sqlbuild.executor.build.main._execute import execute_build_plan from sqlbuild.executor.build.models import ( BuildCallbacks, diff --git a/tests/unit/src/sqlbuild/executor/load/_test_types.py b/tests/unit/src/sqlbuild/executor/load/_test_types.py index bc458caba8..20f6a9fc0a 100644 --- a/tests/unit/src/sqlbuild/executor/load/_test_types.py +++ b/tests/unit/src/sqlbuild/executor/load/_test_types.py @@ -6,9 +6,8 @@ from datetime import date, datetime from typing import Any -from sqlbuild._native import SourceColumnEntry +from sqlbuild._native import LoaderConnectionMode, SourceColumnEntry from sqlbuild.adapter.contract.types import CursorKind -from sqlbuild.compiler.discovery.types import LoaderConnectionMode from sqlbuild.compiler.python_nodes.types import SkipMode from sqlbuild.cursor_algebra.types import CursorScalar from sqlbuild.executor.scheduling.types import ExecutionStatus diff --git a/tests/unit/src/sqlbuild/executor/load/test_execute.py b/tests/unit/src/sqlbuild/executor/load/test_execute.py index c7c9ebc22f..4b277f089a 100644 --- a/tests/unit/src/sqlbuild/executor/load/test_execute.py +++ b/tests/unit/src/sqlbuild/executor/load/test_execute.py @@ -9,10 +9,14 @@ import pytest -from sqlbuild._native import CostContext, SourceEntry, SourceWriteStrategy +from sqlbuild._native import ( + CostContext, + DiscoveredLoaderFunction, + LoaderConnectionMode, + SourceEntry, + SourceWriteStrategy, +) from sqlbuild.adapter.contract.classes.statement_recorder import StatementRecorder -from sqlbuild.compiler.discovery.models import DiscoveredLoaderFunction -from sqlbuild.compiler.discovery.types import LoaderConnectionMode from sqlbuild.cost.models import CostResourceContext from sqlbuild.executor.load.main._execute import execute_source_load from sqlbuild.executor.load.models import ( diff --git a/tests/unit/src/sqlbuild/executor/load/test_run.py b/tests/unit/src/sqlbuild/executor/load/test_run.py index 3158a8ae0a..1d5bc8a8e9 100644 --- a/tests/unit/src/sqlbuild/executor/load/test_run.py +++ b/tests/unit/src/sqlbuild/executor/load/test_run.py @@ -8,9 +8,13 @@ import pytest -from sqlbuild._native import LoadProgressReporter, NativeProgressProjector, SourceEntry -from sqlbuild.compiler.discovery.models import DiscoveredLoaderFunction -from sqlbuild.compiler.discovery.types import LoaderConnectionMode +from sqlbuild._native import ( + DiscoveredLoaderFunction, + LoaderConnectionMode, + LoadProgressReporter, + NativeProgressProjector, + SourceEntry, +) from sqlbuild.compiler.python_nodes.types import SkipMode from sqlbuild.executor.load.main.run import run_load_pipeline from sqlbuild.executor.load.models import ( diff --git a/tests/unit/src/sqlbuild/executor/pipeline/_helpers/helpers.py b/tests/unit/src/sqlbuild/executor/pipeline/_helpers/helpers.py index 33de311c7d..b2bb95334a 100644 --- a/tests/unit/src/sqlbuild/executor/pipeline/_helpers/helpers.py +++ b/tests/unit/src/sqlbuild/executor/pipeline/_helpers/helpers.py @@ -10,6 +10,8 @@ from sqlbuild._native import ( AuditPlanEntry, + CompilePipelineResult, + DiscoveredSqlScenarioFile, PlanOutput, ScenarioExecutionPlan, ScenarioGraphPlan, @@ -35,8 +37,6 @@ CompiledSqlScenario, ) from sqlbuild.compiler.compile.types import CompiledResourceType -from sqlbuild.compiler.discovery.models import DiscoveredSqlScenarioFile -from sqlbuild.compiler.pipeline.models import CompilePipelineResult from sqlbuild.errors.contracts.exceptions import ExecutorInputError from sqlbuild.executor.auditing.models import AuditExecutionResult from sqlbuild.executor.scenario.constants import SCENARIO_LOCAL_JSONL_INVALID diff --git a/tests/unit/src/sqlbuild/executor/pipeline/_helpers/test_scenario.py b/tests/unit/src/sqlbuild/executor/pipeline/_helpers/test_scenario.py index 3aa9db6548..c9362d38f9 100644 --- a/tests/unit/src/sqlbuild/executor/pipeline/_helpers/test_scenario.py +++ b/tests/unit/src/sqlbuild/executor/pipeline/_helpers/test_scenario.py @@ -7,9 +7,8 @@ import pytest -from sqlbuild._native import ScenarioExecutionPlan, SqlLexicalSyntax +from sqlbuild._native import CompilePipelineResult, ScenarioExecutionPlan, SqlLexicalSyntax from sqlbuild.compiler.compile.models import CompiledSqlScenario -from sqlbuild.compiler.pipeline.models import CompilePipelineResult from sqlbuild.executor.pipeline._helpers import scenario as scenario_pipeline from sqlbuild.executor.scenario._helpers.local import execution as local_scenario from sqlbuild.executor.scenario.constants import ( diff --git a/tests/unit/src/sqlbuild/executor/python_nodes/_helpers/helpers.py b/tests/unit/src/sqlbuild/executor/python_nodes/_helpers/helpers.py index 03901f89a4..d775f4e8ce 100644 --- a/tests/unit/src/sqlbuild/executor/python_nodes/_helpers/helpers.py +++ b/tests/unit/src/sqlbuild/executor/python_nodes/_helpers/helpers.py @@ -13,7 +13,12 @@ from sqlbuild._native import ( CostContext, + DiscoveredAssetFunction, + DiscoveredCheckFunction, + DiscoveredLoaderFunction, + DiscoveredProjectInputs, DiscoveredPythonNode, + DiscoveredTaskFunction, LocalConfig, ProjectConfig, PythonNodeGraph, @@ -27,13 +32,6 @@ from sqlbuild.adapter.contract.classes.statement_recorder import StatementRecorder from sqlbuild.assets import AssetContext, asset from sqlbuild.checks import CheckContext -from sqlbuild.compiler.discovery.models import ( - DiscoveredAssetFunction, - DiscoveredCheckFunction, - DiscoveredLoaderFunction, - DiscoveredProjectInputs, - DiscoveredTaskFunction, -) from sqlbuild.compiler.python_nodes.types import SkipMode from sqlbuild.cost.models import CostResourceContext from sqlbuild.errors.contracts.exceptions import ExecutorInputError diff --git a/tests/unit/src/sqlbuild/executor/python_nodes/_helpers/test_execution.py b/tests/unit/src/sqlbuild/executor/python_nodes/_helpers/test_execution.py index e50de69cc8..41439907dd 100644 --- a/tests/unit/src/sqlbuild/executor/python_nodes/_helpers/test_execution.py +++ b/tests/unit/src/sqlbuild/executor/python_nodes/_helpers/test_execution.py @@ -7,9 +7,8 @@ import pytest -from sqlbuild._native import CostContext +from sqlbuild._native import CostContext, DiscoveredAssetFunction, DiscoveredTaskFunction from sqlbuild.adapter.contract.classes.statement_recorder import StatementRecorder -from sqlbuild.compiler.discovery.models import DiscoveredAssetFunction, DiscoveredTaskFunction from sqlbuild.compiler.python_nodes.types import PythonNodeStatus from sqlbuild.cost.models import CostResourceContext from sqlbuild.executor.node_results.classes.direct_store import DirectNodeResultStore diff --git a/tests/unit/src/sqlbuild/executor/python_nodes/_helpers/test_python_checks.py b/tests/unit/src/sqlbuild/executor/python_nodes/_helpers/test_python_checks.py index c94552e596..c735ce8329 100644 --- a/tests/unit/src/sqlbuild/executor/python_nodes/_helpers/test_python_checks.py +++ b/tests/unit/src/sqlbuild/executor/python_nodes/_helpers/test_python_checks.py @@ -10,13 +10,13 @@ from sqlbuild._native import ( CostContext, + DiscoveredCheckFunction, ExecutionEventWriter, PythonNodeGraph, TerminalEventIndex, format_check_json, terminal_event_index_scope, ) -from sqlbuild.compiler.discovery.models import DiscoveredCheckFunction from sqlbuild.compiler.python_nodes.types import PythonNodeKind, PythonNodeStatus, SkipMode from sqlbuild.cost.models import CostResourceContext from sqlbuild.executor.node_results.models import NodeResultEnvelope diff --git a/tests/unit/src/sqlbuild/executor/python_nodes/classes/_test_types.py b/tests/unit/src/sqlbuild/executor/python_nodes/classes/_test_types.py index 89b16ada8b..a6f0961b84 100644 --- a/tests/unit/src/sqlbuild/executor/python_nodes/classes/_test_types.py +++ b/tests/unit/src/sqlbuild/executor/python_nodes/classes/_test_types.py @@ -2,7 +2,7 @@ from dataclasses import dataclass, field -from sqlbuild.compiler.references.types import HardCodedRelationOwnerKind +from sqlbuild._native import HardCodedRelationOwnerKind from sqlbuild.python_nodes.models import SqlResourceRef diff --git a/tests/unit/src/sqlbuild/executor/python_nodes/classes/test_runtime_relation_guard.py b/tests/unit/src/sqlbuild/executor/python_nodes/classes/test_runtime_relation_guard.py index fb6847b0d3..d178f0f5bd 100644 --- a/tests/unit/src/sqlbuild/executor/python_nodes/classes/test_runtime_relation_guard.py +++ b/tests/unit/src/sqlbuild/executor/python_nodes/classes/test_runtime_relation_guard.py @@ -4,7 +4,7 @@ import pytest -from sqlbuild.compiler.references.types import HardCodedRelationOwnerKind +from sqlbuild._native import HardCodedRelationOwnerKind from sqlbuild.executor.python_nodes.classes.runtime_relation_guard import RuntimeRelationGuard from sqlbuild.python_nodes.models import SqlResourceRef from sqlbuild.refs import model, source diff --git a/tests/unit/src/sqlbuild/executor/run/_helpers/helpers.py b/tests/unit/src/sqlbuild/executor/run/_helpers/helpers.py index c21ee40fad..cb3008e1c8 100644 --- a/tests/unit/src/sqlbuild/executor/run/_helpers/helpers.py +++ b/tests/unit/src/sqlbuild/executor/run/_helpers/helpers.py @@ -4,10 +4,12 @@ from sqlbuild._native import ( AuditPlanEntry, + DiscoveredHookFunction, MaterializationType, ModelPlanEntry, PlanAction, PlanReason, + PythonHookEntry, SchemaDynamicColumnFamily, SourceEntry, ) @@ -29,7 +31,6 @@ CompiledRelationLocation, ) from sqlbuild.compiler.compile.types import CompiledResourceType -from sqlbuild.compiler.discovery.models import DiscoveredHookFunction, PythonHookEntry from sqlbuild.executor.auditing.models import AuditExecutionResult from sqlbuild.executor.run._helpers.execution.hooks import execute_hooks from sqlbuild.executor.run.models import HookContext, HookRelationLookup, HookRunContext diff --git a/tests/unit/src/sqlbuild/executor/run/_helpers/test_hooks.py b/tests/unit/src/sqlbuild/executor/run/_helpers/test_hooks.py index 1775657443..9a37f1cbf1 100644 --- a/tests/unit/src/sqlbuild/executor/run/_helpers/test_hooks.py +++ b/tests/unit/src/sqlbuild/executor/run/_helpers/test_hooks.py @@ -7,11 +7,16 @@ import pytest -from sqlbuild._native import CostContext, ModelPlanEntry +from sqlbuild._native import ( + CostContext, + DiscoveredHookFunction, + ModelPlanEntry, + PythonHookEntry, + SqlHookEntry, +) from sqlbuild.adapter.contract.classes.statement_recorder import StatementRecorder from sqlbuild.adapters.duckdb.classes.duckdb_adapter import DuckDbAdapter from sqlbuild.compiler.compile.models import CompiledRelationLocation -from sqlbuild.compiler.discovery.models import DiscoveredHookFunction, PythonHookEntry, SqlHookEntry from sqlbuild.cost.models import CostResourceContext from sqlbuild.errors.contracts.exceptions import ExecutorInputError from sqlbuild.executor.run._helpers.execution.hooks import execute_hooks, render_hooks diff --git a/tests/unit/src/sqlbuild/executor/run/_helpers/test_snapshot_lifecycle.py b/tests/unit/src/sqlbuild/executor/run/_helpers/test_snapshot_lifecycle.py index e12176d150..41dc1b31d1 100644 --- a/tests/unit/src/sqlbuild/executor/run/_helpers/test_snapshot_lifecycle.py +++ b/tests/unit/src/sqlbuild/executor/run/_helpers/test_snapshot_lifecycle.py @@ -7,12 +7,17 @@ import pytest -from sqlbuild._native import ModelPlanEntry, SnapshotsConfig +from sqlbuild._native import ( + DiscoveredHookFunction, + ModelPlanEntry, + PythonHookEntry, + SnapshotsConfig, + SqlHookEntry, +) from sqlbuild.adapter.contract.classes.statement_recorder import StatementRecorder from sqlbuild.adapter.contract.models import ColumnInfo from sqlbuild.adapters.duckdb.classes.duckdb_adapter import DuckDbAdapter from sqlbuild.adapters.snowflake.classes.snowflake_adapter import SnowflakeAdapter -from sqlbuild.compiler.discovery.models import DiscoveredHookFunction, PythonHookEntry, SqlHookEntry from sqlbuild.compiler.fingerprints.constants import FINGERPRINT_TABLE_NAME, NODE_TYPE_MODEL from sqlbuild.errors.contracts.exceptions import ExecutorInputError from sqlbuild.executor.run._helpers.materializations.snapshot import ( diff --git a/tests/unit/src/sqlbuild/integrations/dbt/helpers.py b/tests/unit/src/sqlbuild/integrations/dbt/helpers.py index 7eef80fe3e..42c0c11e9f 100644 --- a/tests/unit/src/sqlbuild/integrations/dbt/helpers.py +++ b/tests/unit/src/sqlbuild/integrations/dbt/helpers.py @@ -20,6 +20,8 @@ DbtEventStream, DbtEventStreamUpdate, DbtRunner, + DiscoveredProjectInputs, + DiscoveredSqlModelFile, LocalConfig, MaterializationType, ModelPlanEntry, @@ -40,7 +42,6 @@ CompileModelInput, ) from sqlbuild.compiler.compile.types import CompiledProjectView, CompiledResourceType -from sqlbuild.compiler.discovery.models import DiscoveredProjectInputs, DiscoveredSqlModelFile _GENERIC_SQL_SYNTAX: SqlLexicalSyntax = SqlLexicalSyntax() diff --git a/tests/unit/src/sqlbuild/integrations/dbt/test_execute.py b/tests/unit/src/sqlbuild/integrations/dbt/test_execute.py index 6396fd1f88..0709a6e627 100644 --- a/tests/unit/src/sqlbuild/integrations/dbt/test_execute.py +++ b/tests/unit/src/sqlbuild/integrations/dbt/test_execute.py @@ -28,6 +28,7 @@ DbtManifestIndex, DbtNodeExecutionResult, DbtRunner, + DiscoveredProjectInputs, build_dbt_interop_plan, build_failed_sqlbuild_model_names, build_merged_dbt_execution_argv, @@ -35,7 +36,6 @@ ) from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter from sqlbuild.compiler.compile.models import CompiledProject -from sqlbuild.compiler.discovery.models import DiscoveredProjectInputs from tests.unit.src.sqlbuild.integrations.dbt._test_types import ( DbtCompileFullRefreshPipelineTestCase, DbtExecutionSelectionStatusTestCase, diff --git a/tests/unit/src/sqlbuild/integrations/dbt/test_manifest.py b/tests/unit/src/sqlbuild/integrations/dbt/test_manifest.py index 588e28708b..ea6f0d786f 100644 --- a/tests/unit/src/sqlbuild/integrations/dbt/test_manifest.py +++ b/tests/unit/src/sqlbuild/integrations/dbt/test_manifest.py @@ -3,13 +3,13 @@ import pytest from sqlbuild._native import ( + CompileInputError, DbtManifestIndex, DbtManifestModel, DbtManifestSource, build_dbt_manifest_index, resolve_dbt_manifest_model, ) -from sqlbuild.compiler.compile.exceptions import CompileInputError from tests.unit.src.sqlbuild.integrations.dbt._test_types import ( DbtManifestIndexErrorTestCase, DbtManifestResolutionErrorTestCase, diff --git a/tests/unit/src/sqlbuild/lint/_helpers/test_project_files.py b/tests/unit/src/sqlbuild/lint/_helpers/test_project_files.py index 111ccd092b..ea414169a9 100644 --- a/tests/unit/src/sqlbuild/lint/_helpers/test_project_files.py +++ b/tests/unit/src/sqlbuild/lint/_helpers/test_project_files.py @@ -7,13 +7,13 @@ import pytest from sqlbuild._native import ( + DiscoveredProjectInputs, LintConfig, LintRunResult, collect_lint_project_files, discover_project_inputs, run_lint, ) -from sqlbuild.compiler.discovery.models import DiscoveredProjectInputs from tests.unit.src.sqlbuild.lint._helpers._test_types import ( CollectProjectFilesTestCase, SelectedProjectFilesTestCase, diff --git a/tests/unit/src/sqlbuild/lint/test_adapter_lexical_syntax.py b/tests/unit/src/sqlbuild/lint/test_adapter_lexical_syntax.py index d174e5775e..c723e451f2 100644 --- a/tests/unit/src/sqlbuild/lint/test_adapter_lexical_syntax.py +++ b/tests/unit/src/sqlbuild/lint/test_adapter_lexical_syntax.py @@ -10,9 +10,8 @@ import sqlbuild._native as fixture_formatting_module import sqlbuild._native as native_module -from sqlbuild._native import SqlLexicalSyntax, discover_project_inputs +from sqlbuild._native import DiscoveredProjectInputs, SqlLexicalSyntax, discover_project_inputs from sqlbuild.adapters.duckdb.classes.duckdb_adapter import DuckDbAdapter -from sqlbuild.compiler.discovery.models import DiscoveredProjectInputs from tests.unit.src.sqlbuild.lint._test_types import PassedAdapterSyntaxTestCase from tests.unit.src.sqlbuild.lint.helpers import write_fixture_format_project diff --git a/tests/unit/src/sqlbuild/lint/test_expanded_lint.py b/tests/unit/src/sqlbuild/lint/test_expanded_lint.py index 74b0cb46d6..d9f90d1c8a 100644 --- a/tests/unit/src/sqlbuild/lint/test_expanded_lint.py +++ b/tests/unit/src/sqlbuild/lint/test_expanded_lint.py @@ -9,6 +9,7 @@ import sqlbuild._native as native_module from sqlbuild._native import ( + DiscoveredProjectInputs, LintBody, LintConfig, LintRunResult, @@ -22,7 +23,6 @@ prepare_lint_body, run_lint, ) -from sqlbuild.compiler.discovery.models import DiscoveredProjectInputs from tests.unit.src.sqlbuild.lint._test_types import ( ExpandedLintTestCase, ExpandedTypedConstantTestCase, diff --git a/tests/unit/src/sqlbuild/provider/_helpers/test_build_session.py b/tests/unit/src/sqlbuild/provider/_helpers/test_build_session.py index 991087bbb9..04fa7ff424 100644 --- a/tests/unit/src/sqlbuild/provider/_helpers/test_build_session.py +++ b/tests/unit/src/sqlbuild/provider/_helpers/test_build_session.py @@ -5,7 +5,7 @@ import pytest -from sqlbuild.compiler.discovery.models import DiscoveredProvider +from sqlbuild._native import DiscoveredProvider from sqlbuild.provider._helpers.session import build_provider_session from sqlbuild.provider.classes.session import ProviderSession from sqlbuild.providers import Provider diff --git a/tests/unit/src/sqlbuild/rule_engine/_helpers/engine/helpers.py b/tests/unit/src/sqlbuild/rule_engine/_helpers/engine/helpers.py index a97386c269..031847235e 100644 --- a/tests/unit/src/sqlbuild/rule_engine/_helpers/engine/helpers.py +++ b/tests/unit/src/sqlbuild/rule_engine/_helpers/engine/helpers.py @@ -17,6 +17,7 @@ CompileSqlTestCte, CustomRulesOutcome, DiscoveredSqlTestBlock, + DiscoveredSqlTestFile, NativeRulesEvaluation, NativeRulesRequest, RuleExemption, @@ -31,7 +32,6 @@ from sqlbuild._native import evaluate_project_rules as evaluate from sqlbuild.compiler.compile.models import CompiledObjectKey, CompiledSqlTest from sqlbuild.compiler.compile.types import CompiledProjectView, CompiledResourceType, SqlTestMode -from sqlbuild.compiler.discovery.models import DiscoveredSqlTestFile from sqlbuild.rule_engine.constants import MIN_CUSTOM_RULE_TEST_CASES from sqlbuild.rule_engine.models import Finding, Rule, RulesConfig, RulesResult from tests.unit.src.sqlbuild.rule_engine._helpers.engine._test_types import CustomRuleTestCase diff --git a/tests/unit/src/sqlbuild/rule_engine/_helpers/engine/test_config.py b/tests/unit/src/sqlbuild/rule_engine/_helpers/engine/test_config.py index 38255e913c..2a1482d858 100644 --- a/tests/unit/src/sqlbuild/rule_engine/_helpers/engine/test_config.py +++ b/tests/unit/src/sqlbuild/rule_engine/_helpers/engine/test_config.py @@ -4,8 +4,7 @@ import pytest -from sqlbuild._native import SqlTestRulesConfig, load_rules_config -from sqlbuild.rule_engine.exceptions import RulesConfigError, RulesError +from sqlbuild._native import RulesConfigError, RulesError, SqlTestRulesConfig, load_rules_config from sqlbuild.rule_engine.models import LayoutConfig, RulesConfig from tests.unit.src.sqlbuild.rule_engine._helpers.engine._test_types import ( AllowExceptionsConfigTestCase, diff --git a/tests/unit/src/sqlbuild/rule_engine/_helpers/engine/test_custom_rule_evidence.py b/tests/unit/src/sqlbuild/rule_engine/_helpers/engine/test_custom_rule_evidence.py index e8bd7e90fe..360da45a7f 100644 --- a/tests/unit/src/sqlbuild/rule_engine/_helpers/engine/test_custom_rule_evidence.py +++ b/tests/unit/src/sqlbuild/rule_engine/_helpers/engine/test_custom_rule_evidence.py @@ -5,13 +5,13 @@ import pytest from sqlbuild._native import ( + RulesError, build_rule_catalogue, custom_rule_implementation_fingerprint, custom_rule_test_evidence, ) from sqlbuild._native import evaluate_project_rules as evaluate from sqlbuild.compiler.compile.types import CompiledProjectView -from sqlbuild.rule_engine.exceptions import RulesError from sqlbuild.rule_engine.models import Rule, RulesConfig, RulesResult from tests.unit.src.sqlbuild.rule_engine._helpers.engine._test_types import ( CustomRuleEvidenceTestCase, diff --git a/tests/unit/src/sqlbuild/rule_engine/_helpers/engine/test_custom_rules.py b/tests/unit/src/sqlbuild/rule_engine/_helpers/engine/test_custom_rules.py index 4f65dae842..4a6aafef9b 100644 --- a/tests/unit/src/sqlbuild/rule_engine/_helpers/engine/test_custom_rules.py +++ b/tests/unit/src/sqlbuild/rule_engine/_helpers/engine/test_custom_rules.py @@ -8,13 +8,13 @@ from sqlbuild._native import ( RuleExemption, RuleIgnore, + RulesError, compiled_model_record, project_with, ) from sqlbuild._native import evaluate_project_rules as evaluate from sqlbuild.compiler.compile.models import CompactLineageFacts from sqlbuild.compiler.compile.types import CompiledProjectView -from sqlbuild.rule_engine.exceptions import RulesError from sqlbuild.rule_engine.models import RulesConfig, RulesResult from tests.unit.src.sqlbuild.rule_engine._helpers.engine._test_types import ( CustomRuleCacheDependencyTestCase, diff --git a/tests/unit/src/sqlbuild/rule_engine/_helpers/engine/test_definition.py b/tests/unit/src/sqlbuild/rule_engine/_helpers/engine/test_definition.py index bedcf29e3e..6cebaf7c89 100644 --- a/tests/unit/src/sqlbuild/rule_engine/_helpers/engine/test_definition.py +++ b/tests/unit/src/sqlbuild/rule_engine/_helpers/engine/test_definition.py @@ -4,11 +4,11 @@ import pytest +from sqlbuild._native import RulesError from sqlbuild.rule_engine._helpers.engine.definition import ( resolve_rule_signature, rule_from_value, ) -from sqlbuild.rule_engine.exceptions import RulesError from sqlbuild.rule_engine.models import Rule from sqlbuild.rules import Finding, Model, Project, RuleContext, rule from tests.unit.src.sqlbuild.rule_engine._helpers.engine._test_types import ( diff --git a/tests/unit/src/sqlbuild/rule_engine/_helpers/engine/test_hermeticity.py b/tests/unit/src/sqlbuild/rule_engine/_helpers/engine/test_hermeticity.py index 35ccfe213f..bce9564f69 100644 --- a/tests/unit/src/sqlbuild/rule_engine/_helpers/engine/test_hermeticity.py +++ b/tests/unit/src/sqlbuild/rule_engine/_helpers/engine/test_hermeticity.py @@ -4,8 +4,8 @@ import pytest +from sqlbuild._native import NonHermeticRuleError from sqlbuild.rule_engine._helpers.engine.hermeticity import verify_custom_rules -from sqlbuild.rule_engine.exceptions import NonHermeticRuleError from sqlbuild.rule_engine.models import Rule from tests.unit.src.sqlbuild.rule_engine._helpers.engine._test_types import ( CustomRuleImportTestCase, diff --git a/tests/unit/src/sqlbuild/rule_engine/_helpers/engine/test_native_facts.py b/tests/unit/src/sqlbuild/rule_engine/_helpers/engine/test_native_facts.py index fab374960c..fd679a27f9 100644 --- a/tests/unit/src/sqlbuild/rule_engine/_helpers/engine/test_native_facts.py +++ b/tests/unit/src/sqlbuild/rule_engine/_helpers/engine/test_native_facts.py @@ -4,6 +4,7 @@ import pytest from sqlbuild._native import ( + DiscoveredSqlScenarioFile, project_with, ) from sqlbuild.compiler.compile.models import ( @@ -12,9 +13,6 @@ CompiledSqlTest, ) from sqlbuild.compiler.compile.types import CompiledProjectView, CompiledResourceType, SqlTestMode -from sqlbuild.compiler.discovery.models import ( - DiscoveredSqlScenarioFile, -) from tests.unit.src.sqlbuild.rule_engine._helpers.engine._test_types import ( NativeFactPayloadTestCase, ) diff --git a/tests/unit/src/sqlbuild/rule_engine/_helpers/engine/test_typed_constants.py b/tests/unit/src/sqlbuild/rule_engine/_helpers/engine/test_typed_constants.py index 8719db8a69..28a64c9b6b 100644 --- a/tests/unit/src/sqlbuild/rule_engine/_helpers/engine/test_typed_constants.py +++ b/tests/unit/src/sqlbuild/rule_engine/_helpers/engine/test_typed_constants.py @@ -4,10 +4,10 @@ import pytest from sqlbuild._native import ( + ConstantDeclaration, declaration_rows, normalize_sql_value, ) -from sqlbuild.compiler.discovery.models import ConstantDeclaration from tests.unit.src.sqlbuild.rule_engine._helpers.engine._test_types import ( TypedConstantPayloadTestCase, ) diff --git a/tests/unit/src/sqlbuild/rule_engine/_helpers/host/helpers.py b/tests/unit/src/sqlbuild/rule_engine/_helpers/host/helpers.py index bbdb0ebc62..7d785aac72 100644 --- a/tests/unit/src/sqlbuild/rule_engine/_helpers/host/helpers.py +++ b/tests/unit/src/sqlbuild/rule_engine/_helpers/host/helpers.py @@ -18,6 +18,7 @@ from sqlbuild._native import ( CustomHostObserver, CustomRulesOutcome, + OpaqueModuleStateError, RulesCacheConfig, build_rule_catalogue, project_with, @@ -26,7 +27,6 @@ from sqlbuild.compiler.compile.types import CompiledProjectView from sqlbuild.rule_engine._helpers.host.module_state import module_state_token, rule_namespaces from sqlbuild.rule_engine.classes.module_state_monitor import ModuleStateMonitor -from sqlbuild.rule_engine.exceptions import OpaqueModuleStateError from sqlbuild.rule_engine.models import Rule, RulesConfig from tests.unit.src.sqlbuild.rule_engine.main.evaluate.helpers import build_project diff --git a/tests/unit/src/sqlbuild/rule_engine/_helpers/host/test_custom_host_pool.py b/tests/unit/src/sqlbuild/rule_engine/_helpers/host/test_custom_host_pool.py index 528da9bb3f..ec91915212 100644 --- a/tests/unit/src/sqlbuild/rule_engine/_helpers/host/test_custom_host_pool.py +++ b/tests/unit/src/sqlbuild/rule_engine/_helpers/host/test_custom_host_pool.py @@ -5,11 +5,16 @@ import pytest -from sqlbuild._native import CustomHostObserver, CustomRulesOutcome, rules_bulk_cache_path +from sqlbuild._native import ( + CustomHostObserver, + CustomRulesOutcome, + HostCancelledError, + RulesError, + rules_bulk_cache_path, +) from sqlbuild.compiler.compile.types import CompiledProjectView from sqlbuild.rule_engine._helpers.host import custom_host from sqlbuild.rule_engine.constants import CUSTOM_RULES_CACHE_FILE -from sqlbuild.rule_engine.exceptions import HostCancelledError, RulesError from tests.unit.src.sqlbuild.rule_engine._helpers.host._test_types import ( AwaitInputsTestCase, AwaitPublishedInputsTestCase, diff --git a/tests/unit/src/sqlbuild/rule_engine/_helpers/run/test_native_rows.py b/tests/unit/src/sqlbuild/rule_engine/_helpers/run/test_native_rows.py index 03ec9b4be7..79b54483ec 100644 --- a/tests/unit/src/sqlbuild/rule_engine/_helpers/run/test_native_rows.py +++ b/tests/unit/src/sqlbuild/rule_engine/_helpers/run/test_native_rows.py @@ -5,9 +5,8 @@ import pytest import sqlbuild._native as native -from sqlbuild._native import RulesCacheConfig +from sqlbuild._native import RulesCacheConfig, RulesError from sqlbuild.compiler.compile.types import CompiledProjectView -from sqlbuild.rule_engine.exceptions import RulesError from sqlbuild.rule_engine.models import RulesConfig, RulesResult from tests.unit.src.sqlbuild.rule_engine._helpers.engine.helpers import evaluate_contract_rule from tests.unit.src.sqlbuild.rule_engine._helpers.run._test_types import ( diff --git a/tests/unit/src/sqlbuild/rule_engine/classes/project_tree/test_read_text.py b/tests/unit/src/sqlbuild/rule_engine/classes/project_tree/test_read_text.py index addaa83265..df5918535e 100644 --- a/tests/unit/src/sqlbuild/rule_engine/classes/project_tree/test_read_text.py +++ b/tests/unit/src/sqlbuild/rule_engine/classes/project_tree/test_read_text.py @@ -4,8 +4,8 @@ import pytest +from sqlbuild._native import RuleUsageError from sqlbuild.rule_engine.classes.project_tree import ProjectTree -from sqlbuild.rule_engine.exceptions import RuleUsageError from tests.unit.src.sqlbuild.rule_engine.classes.project_tree._test_types import ReadTextTestCase from tests.unit.src.sqlbuild.rule_engine.classes.project_tree.helpers import ( LIMITS_YAML, diff --git a/tests/unit/src/sqlbuild/rule_engine/classes/shared_fact_encoder/helpers.py b/tests/unit/src/sqlbuild/rule_engine/classes/shared_fact_encoder/helpers.py index cb01e5fb78..f43d38b810 100644 --- a/tests/unit/src/sqlbuild/rule_engine/classes/shared_fact_encoder/helpers.py +++ b/tests/unit/src/sqlbuild/rule_engine/classes/shared_fact_encoder/helpers.py @@ -2,6 +2,7 @@ from sqlbuild._native import ( CompiledSource, + DiscoveredSourceFile, SourceEntry, project_with, ) @@ -11,7 +12,6 @@ CompiledResourceType, CompiledSourceView, ) -from sqlbuild.compiler.discovery.models import DiscoveredSourceFile from sqlbuild.rule_engine.classes.rule_context import RuleFactViews, build_rule_fact_views from tests.unit.src.sqlbuild.rule_engine.main.evaluate.helpers import build_project diff --git a/tests/unit/src/sqlbuild/rule_engine/main/evaluate/helpers.py b/tests/unit/src/sqlbuild/rule_engine/main/evaluate/helpers.py index 2d882a3330..49fc43cf74 100644 --- a/tests/unit/src/sqlbuild/rule_engine/main/evaluate/helpers.py +++ b/tests/unit/src/sqlbuild/rule_engine/main/evaluate/helpers.py @@ -2,7 +2,7 @@ from pathlib import Path -from sqlbuild._native import CompileModelConfig, EnumMember +from sqlbuild._native import CompileModelConfig, EnumDeclaration, EnumMember from sqlbuild.compiler.compile.models import ( CompiledModel, CompiledObjectKey, @@ -15,7 +15,6 @@ CompiledProjectView, CompiledResourceType, ) -from sqlbuild.compiler.discovery.models import EnumDeclaration _ENUM_DECLARATION: EnumDeclaration = EnumDeclaration( name="status", diff --git a/tests/unit/src/sqlbuild/rule_engine/main/evaluate/test_cache_and_suppressions.py b/tests/unit/src/sqlbuild/rule_engine/main/evaluate/test_cache_and_suppressions.py index 350169e5c8..0d1df9fc33 100644 --- a/tests/unit/src/sqlbuild/rule_engine/main/evaluate/test_cache_and_suppressions.py +++ b/tests/unit/src/sqlbuild/rule_engine/main/evaluate/test_cache_and_suppressions.py @@ -8,12 +8,12 @@ from sqlbuild._native import ( RuleExemption, RuleIgnore, + RulesError, compiled_model_record, project_with, ) from sqlbuild._native import evaluate_project_rules as evaluate from sqlbuild.compiler.compile.types import CompiledProjectView -from sqlbuild.rule_engine.exceptions import RulesError from sqlbuild.rule_engine.models import RulesConfig, RulesResult from tests.unit.src.sqlbuild.rule_engine.main.evaluate._test_types import PolicyBehaviorTestCase from tests.unit.src.sqlbuild.rule_engine.main.evaluate.helpers import build_project diff --git a/tests/unit/src/sqlbuild/rule_engine/main/evaluate_rule/test_evaluate_rule.py b/tests/unit/src/sqlbuild/rule_engine/main/evaluate_rule/test_evaluate_rule.py index 897f0cf4cc..a6ad12eae4 100644 --- a/tests/unit/src/sqlbuild/rule_engine/main/evaluate_rule/test_evaluate_rule.py +++ b/tests/unit/src/sqlbuild/rule_engine/main/evaluate_rule/test_evaluate_rule.py @@ -6,11 +6,10 @@ import pytest -from sqlbuild._native import RulesCacheConfig +from sqlbuild._native import RulesCacheConfig, RulesError from sqlbuild._native import evaluate_project_rules as evaluate from sqlbuild.compiler.compile.types import CompiledProjectView from sqlbuild.rule_engine.constants import MIN_CUSTOM_RULE_TEST_CASES -from sqlbuild.rule_engine.exceptions import RulesError from sqlbuild.rule_engine.models import RulesConfig, RulesResult from sqlbuild.rule_engine.types import RuleOptionValue from sqlbuild.rules.testing import RuleCase, RuleResult, evaluate_rule diff --git a/tests/unit/src/sqlbuild/rule_engine/main/run_rules/test_run_rules.py b/tests/unit/src/sqlbuild/rule_engine/main/run_rules/test_run_rules.py index 090b25dd4f..12b78fb7ac 100644 --- a/tests/unit/src/sqlbuild/rule_engine/main/run_rules/test_run_rules.py +++ b/tests/unit/src/sqlbuild/rule_engine/main/run_rules/test_run_rules.py @@ -5,6 +5,8 @@ import pytest from sqlbuild._native import ( + DiscoveredProjectInputs, + ProjectGraph, RulesRunResult, build_project_graph, discover_project_inputs, @@ -13,8 +15,6 @@ run_rules, ) from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter -from sqlbuild.compiler.discovery.models import DiscoveredProjectInputs -from sqlbuild.compiler.pipeline.models import ProjectGraph from tests.unit.src.sqlbuild.rule_engine.main.run_rules._test_types import ( TypeProofRuleSkipTestCase, ) diff --git a/tests/unit/src/sqlbuild/rule_engine/main/skills/test_skills.py b/tests/unit/src/sqlbuild/rule_engine/main/skills/test_skills.py index 33f78a0237..4274b49b5a 100644 --- a/tests/unit/src/sqlbuild/rule_engine/main/skills/test_skills.py +++ b/tests/unit/src/sqlbuild/rule_engine/main/skills/test_skills.py @@ -5,9 +5,8 @@ import pytest -from sqlbuild._native import RuleExemption +from sqlbuild._native import RuleExemption, RulesError from sqlbuild._native import install_rules_skills as install_skills -from sqlbuild.rule_engine.exceptions import RulesError from sqlbuild.rule_engine.models import RulesConfig from tests.unit.src.sqlbuild.rule_engine.main.skills._test_types import ( SkillDivergenceTestCase, diff --git a/tests/unit/src/sqlbuild/runtime/event_exporting/classes/helpers.py b/tests/unit/src/sqlbuild/runtime/event_exporting/classes/helpers.py index 6ce4b02087..9ef062f5d0 100644 --- a/tests/unit/src/sqlbuild/runtime/event_exporting/classes/helpers.py +++ b/tests/unit/src/sqlbuild/runtime/event_exporting/classes/helpers.py @@ -8,6 +8,7 @@ DiscoveredEventExporter, DiscoveredProvider, DiscoveredProviderUsage, + EventExportSeverity, ) from sqlbuild.observability import LifecycleEvent from sqlbuild.providers import Provider diff --git a/tests/unit/src/sqlbuild/spec/contracts/main/test_resolve_effective_scenario_config.py b/tests/unit/src/sqlbuild/spec/contracts/main/test_resolve_effective_scenario_config.py index 9c55b71425..1fc5a111ee 100644 --- a/tests/unit/src/sqlbuild/spec/contracts/main/test_resolve_effective_scenario_config.py +++ b/tests/unit/src/sqlbuild/spec/contracts/main/test_resolve_effective_scenario_config.py @@ -7,11 +7,11 @@ import pytest from sqlbuild._native import ( + DiscoveredProjectInputs, ProjectConfigError, discover_project_inputs, resolve_effective_scenario_config, ) -from sqlbuild.compiler.discovery.models import DiscoveredProjectInputs from tests.unit.src.sqlbuild.spec.contracts.main._test_types import ( InvalidScenarioNamespaceConfigTestCase, ScenarioNamespaceConfigTestCase, From d65a427c27052cb0824b2ee870d07d2e3ef037b3 Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Thu, 15 Oct 2026 17:28:14 +0100 Subject: [PATCH 05/48] refactor: host compile output, macro and lineage-fact shapes natively --- .../src/hosted_shapes/constants.rs | 82 +- .../_helpers/analysis/explicit_references.rs | 2 +- .../_helpers/analysis/required_columns.rs | 2 +- .../analysis_session/fixture_columns.rs | 2 +- .../analysis_session/outcome_objects.rs | 10 +- .../analysis_session/stage_functions.rs | 2 +- .../_helpers/build_command/lifecycle.rs | 2 +- .../bindings/_helpers/check/check_command.rs | 2 +- .../_helpers/clone/clone_execution.rs | 2 +- .../compile_attachment/model_inputs.rs | 2 +- .../sql_expansion_context.rs | 2 +- .../_helpers/compile_command/report.rs | 2 +- .../_helpers/compile_command/target_writer.rs | 2 +- .../_helpers/compile_driver/analysis.rs | 2 +- .../compile_driver/assembly_checks.rs | 2 +- .../compile_driver/assembly_project.rs | 2 +- .../_helpers/compile_driver/deferral.rs | 2 +- .../_helpers/compile_driver/inputs.rs | 2 +- .../_helpers/compile_driver/phases.rs | 2 +- .../compile_driver/target_defaults.rs | 2 +- .../_helpers/compile_records/constants.rs | 444 ++++++++++ .../_helpers/compiled_project/attachment.rs | 2 +- .../compiled_model_analysis.rs | 11 +- .../_helpers/compiled_project/loop_classes.rs | 2 +- .../_helpers/compiled_project/records.rs | 2 +- .../contracts/contract_diagnostics.rs | 2 +- .../dbt_interop/reference_resolver.rs | 2 +- .../bindings/_helpers/freshness/planning.rs | 2 +- .../_helpers/graph/execution_gates.rs | 2 +- .../src/bindings/_helpers/graph/keys.rs | 2 +- .../_helpers/hand_built/compiled_model.rs | 2 +- .../_helpers/lineage/column_lineage_graph.rs | 26 +- .../_helpers/lineage/lineage_index.rs | 2 +- .../_helpers/lineage/lineage_objects.rs | 2 +- .../_helpers/planner_entries/values.rs | 2 +- .../planner_migrations/old_name_objects.rs | 6 +- .../planner_migrations/planning_origin.rs | 6 +- .../_helpers/planner_phases/objects.rs | 2 +- .../_helpers/planner_scenario/relations.rs | 6 +- .../_helpers/planner_strategy/audits.rs | 2 +- .../_helpers/planner_strategy/constants.rs | 2 +- .../_helpers/record_behaviours/compiled.rs | 384 +++++++++ .../_helpers/record_behaviours/constants.rs | 83 +- .../record_behaviours/lineage_facts.rs | 190 +++++ .../_helpers/record_behaviours/macros.rs | 197 +++++ .../_helpers/record_behaviours/mod.rs | 3 + .../bindings/_helpers/records/constants.rs | 4 +- .../bindings/_helpers/records/registration.rs | 8 +- .../_helpers/refactor/offline_compile.rs | 2 +- .../bindings/_helpers/render/authored_sql.rs | 2 +- .../_helpers/render/declaration_values.rs | 4 +- .../_helpers/render/macro_expansion_result.rs | 2 +- .../bindings/_helpers/rules/custom_hosts.rs | 2 +- .../bindings/_helpers/rules/relation_keys.rs | 2 +- .../_helpers/scopes/compile_usages.rs | 2 +- .../bindings/_helpers/scopes/scope_errors.rs | 2 +- .../_helpers/semantic_checks/resource_sql.rs | 2 +- .../_helpers/semantic_checks/stage_objects.rs | 2 +- .../semantic_shapes/source_bindings.rs | 2 +- .../_helpers/sql_test_glue/compiled_tests.rs | 2 +- .../_helpers/sqltext/sql_references.rs | 2 +- python-boundary.toml | 2 +- .../_helpers/coverage/analysis.py | 3 +- src/sqlbuild/_native.pyi | 683 ++++++++++++++- .../compile/_helpers/render/user_macros.py | 12 +- src/sqlbuild/compiler/compile/constants.py | 8 - src/sqlbuild/compiler/compile/models.py | 787 ------------------ src/sqlbuild/compiler/compile/types.py | 18 +- .../executor/auditing/_helpers/execution.py | 2 +- .../auditing/_helpers/result_projection.py | 2 +- .../executor/auditing/main/_execute.py | 3 +- .../main/_execute_reporting_errors.py | 3 +- src/sqlbuild/executor/auditing/models.py | 3 +- .../executor/build/_helpers/blocking.py | 3 +- .../executor/build/_helpers/end_audits.py | 8 +- .../build/_helpers/external_source_loads.py | 2 +- .../executor/build/_helpers/scheduler.py | 2 +- .../executor/build/_helpers/source_audits.py | 3 +- .../executor/build/_helpers/source_node.py | 2 +- .../executor/build/classes/build_scheduler.py | 2 +- .../build/main/_external_source_loads.py | 3 +- src/sqlbuild/executor/build/models.py | 3 +- .../clone/_helpers/execution_items.py | 2 +- src/sqlbuild/executor/clone/main/execute.py | 9 +- src/sqlbuild/executor/clone/models.py | 10 +- .../migrations/_helpers/old_name_views.py | 1 - .../_helpers/relation_for_location.py | 3 +- .../main/_bound_old_name_view_guard.py | 3 +- .../main/_compatibility_view_sql.py | 3 +- .../main/_complete_old_name_views.py | 3 +- src/sqlbuild/executor/migrations/models.py | 2 +- .../executor/pipeline/_helpers/scenario.py | 2 +- .../executor/pipeline/_helpers/testing.py | 13 +- .../run/_helpers/execution/final_audits.py | 3 +- .../executor/run/_helpers/execution/hooks.py | 2 +- .../run/_helpers/materializations/custom.py | 9 +- .../run/_helpers/materializations/view.py | 9 +- src/sqlbuild/executor/run/models.py | 2 +- .../scenario/_helpers/lifecycle/cleanup.py | 2 +- .../scenario/_helpers/local/execution.py | 2 +- .../classes/prepared_scenario_schemas.py | 3 +- src/sqlbuild/executor/scenario/models.py | 2 +- .../microbatches/classes/direct_store.py | 3 +- .../rule_engine/_helpers/host/custom_host.py | 3 +- .../_helpers/host/tracked_facts.py | 9 +- .../rule_engine/classes/audit_facts.py | 2 +- .../rule_engine/classes/column_facts.py | 2 +- .../rule_engine/classes/model_test_facts.py | 2 +- .../rule_engine/classes/project_tree.py | 2 +- .../waffle_shop/models/marts/_macros/time.py | 2 +- .../snowflake/test_table_type_conversion.py | 8 +- .../sqlbuild/cli/commands/main/test_rules.py | 2 +- .../main/test_startup_import_footprint.py | 2 - .../compiler/analysis_session/helpers.py | 3 +- .../compiler/auditing/main/test_rendering.py | 3 +- .../src/sqlbuild/compiler/compile/helpers.py | 3 +- .../compile/native_model_config/helpers.py | 2 +- .../compiler/compiled_project/helpers.py | 6 +- .../test_native_compiled_project.py | 2 +- .../sqlbuild/compiler/contracts/helpers.py | 3 +- .../compiler/explicit_references/helpers.py | 3 +- .../sqlbuild/compiler/pipeline/test_main.py | 4 +- .../pipeline/test_native_binding_catalog.py | 3 +- .../compiler/pipeline/test_semantic_triage.py | 2 +- .../compiler/planner/_helpers/_test_types.py | 3 +- .../compiler/planner/_helpers/helpers.py | 12 +- .../planner/_helpers/resolve/helpers.py | 10 +- .../planner/_helpers/resolve/test_resolve.py | 4 +- .../_helpers/sql_test_assembly/helpers.py | 12 +- .../sql_test_assembly/test_execute_chain.py | 4 +- .../planner/_helpers/test_audit_entry.py | 8 +- .../_helpers/test_warehouse_cursor_reads.py | 2 +- .../_helpers/test_warehouse_snapshot.py | 6 +- .../sqlbuild/compiler/planner/main/helpers.py | 12 +- .../src/sqlbuild/compiler/scopes/helpers.py | 3 +- .../compiler/semantic_checks/helpers.py | 8 +- .../compiler/sql_test_glue/helpers.py | 4 +- .../executor/build/concurrent/helpers.py | 2 +- .../sqlbuild/executor/build/custom/helpers.py | 6 +- .../test_advanced_custom_materialization.py | 3 +- .../custom/test_custom_materialization.py | 2 +- .../diff/test_query_artifact_cleanup.py | 1 - .../executor/migrations/_helpers/helpers.py | 2 +- .../src/sqlbuild/executor/run/helpers.py | 6 +- .../executor/run/incremental/helpers.py | 6 +- .../executor/run/microbatch/helpers.py | 6 +- .../src/sqlbuild/executor/run/view/helpers.py | 6 +- .../src/sqlbuild/executor/scenario/helpers.py | 6 +- .../executor/scenario/test_capture_types.py | 3 +- .../scenario/test_fixture_lifecycle.py | 6 +- .../src/sqlbuild/executor/testing/helpers.py | 2 +- .../rule_engine/host_payload/helpers.py | 10 +- .../host_payload/test_host_payload_facts.py | 3 +- .../_helpers/coverage/helpers.py | 2 +- .../_helpers/build_planning/helpers.py | 3 +- .../commands/_helpers/check/test_planning.py | 3 +- .../sqlbuild/cli/commands/_helpers/helpers.py | 6 +- .../cli/commands/main/build/_test_types.py | 2 +- .../cli/commands/main/build/helpers.py | 2 +- .../commands/main/build/test_defer_clone.py | 6 +- .../cli/commands/main/compile/helpers.py | 10 +- .../main/compile/test_compile_output.py | 2 +- .../compile/test_compile_target_writer.py | 2 +- .../cli/commands/main/execution/helpers.py | 3 +- .../cli/commands/shared/_helpers/helpers.py | 3 +- .../main/test_attempt_import_footprint.py | 2 +- .../sqlbuild/cli/output/main/plan/helpers.py | 6 +- .../classes/test_build_sql_test_projection.py | 2 +- .../compiler/auditing/main/helpers.py | 10 +- .../compiler/auditing/main/test_rendering.py | 3 +- .../compiler/compile/_helpers/_test_types.py | 4 +- .../compiler/compile/_helpers/helpers.py | 20 +- .../_helpers/render/test_cursor_intrinsics.py | 2 +- .../compile/_helpers/test_assembly.py | 4 +- .../compile/_helpers/test_expansion_spans.py | 3 +- .../compiler/compile/_helpers/test_macros.py | 6 +- .../compile/_helpers/test_model_schemas.py | 4 +- .../_helpers/test_parameterized_sql_tests.py | 6 +- .../_helpers/test_rebind_declaration_scope.py | 2 +- .../compile/_helpers/test_required_columns.py | 8 +- .../_helpers/test_sql_analysis_columns.py | 8 +- .../compile/_helpers/test_sql_references.py | 2 +- .../_helpers/test_typed_macro_references.py | 3 +- .../sqlbuild/compiler/contracts/helpers.py | 12 +- .../contracts/test_promotion_conflicts.py | 3 +- .../src/sqlbuild/compiler/dag/main/helpers.py | 16 +- .../sqlbuild/compiler/dag/main/test_build.py | 4 +- .../src/sqlbuild/compiler/frontier/helpers.py | 3 +- .../src/sqlbuild/compiler/graph/helpers.py | 3 +- .../src/sqlbuild/compiler/lineage/helpers.py | 12 +- .../sqlbuild/compiler/lineage/test_columns.py | 2 +- .../macro_bridge/_helpers/_test_types.py | 2 +- .../compiler/macro_bridge/_helpers/helpers.py | 11 +- .../sqlbuild/compiler/manifest/_test_types.py | 5 +- .../src/sqlbuild/compiler/manifest/helpers.py | 12 +- .../sqlbuild/compiler/manifest/test_main.py | 6 +- .../compiler/pipeline/_helpers/helpers.py | 5 +- .../_helpers/target_validation/_test_types.py | 3 +- .../_helpers/target_validation/helpers.py | 6 +- .../_helpers/target_validation/test_main.py | 4 +- .../_helpers/test_deferred_locations.py | 5 +- .../compiler/pipeline/main/helpers.py | 8 +- .../main/test_compile_phase_timings.py | 2 +- .../compiler/planner/_helpers/_test_types.py | 8 +- .../planner/_helpers/changes/_test_helpers.py | 10 +- .../planner/_helpers/changes/_test_types.py | 2 +- .../planner/_helpers/changes/test_detect.py | 2 +- .../planner/_helpers/changes/test_schema.py | 2 +- .../compiler/planner/_helpers/conftest.py | 4 +- .../compiler/planner/_helpers/helpers.py | 18 +- .../planner/_helpers/migrations/helpers.py | 2 +- .../test_column_migration_candidates.py | 10 +- .../planner/_helpers/planning/helpers.py | 10 +- .../planner/_helpers/resolve/helpers.py | 8 +- .../planner/_helpers/resolve/test_refs.py | 6 +- .../_helpers/sql_test_assembly/helpers.py | 16 +- .../sql_test_assembly/test_plan_test.py | 10 +- .../planner/_helpers/test_audit_entry.py | 9 +- .../planner/_helpers/test_audit_scheduling.py | 6 +- .../_helpers/test_fixture_planning_work.py | 8 +- .../compiler/planner/_helpers/test_graph.py | 2 +- .../planner/_helpers/test_microbatch_limit.py | 6 +- .../planner/_helpers/test_model_closure.py | 3 +- .../test_plan_entry_cursor_overrides.py | 8 +- .../planner/_helpers/test_scenario_cli.py | 2 +- .../planner/_helpers/test_scenario_graph.py | 6 +- .../_helpers/test_scenario_relations.py | 2 +- .../_helpers/test_selection_staleness.py | 3 +- .../planner/_helpers/test_selectors.py | 4 +- .../planner/_helpers/test_source_columns.py | 6 +- .../planner/_helpers/test_source_freshness.py | 2 +- .../_helpers/test_source_load_nodes.py | 10 +- .../planner/_helpers/warehouse/helpers.py | 3 +- .../planner/classes/test_sql_test_planning.py | 2 +- .../main/execution/test_display_plan.py | 10 +- .../sqlbuild/compiler/planner/main/helpers.py | 10 +- .../compiler/planner/main/identity/helpers.py | 8 +- .../test_version_identity_function_hashes.py | 2 +- .../planner/main/pre_build/helpers.py | 2 +- .../compiler/planner/main/test_execution.py | 2 +- .../compiler/python_nodes/_helpers/helpers.py | 10 +- .../_helpers/test_run_lifecycle.py | 2 +- .../_helpers/test_unified_selectors.py | 2 +- .../scopes/test_builder_and_visibility.py | 2 +- .../compiler/source_freshness/main/helpers.py | 2 +- .../executor/auditing/main/helpers.py | 8 +- .../auditing/main/test_project_results.py | 3 +- .../executor/build/_helpers/helpers.py | 6 +- .../executor/build/_helpers/test_blocking.py | 2 +- .../_helpers/test_external_source_loads.py | 2 +- .../executor/build/_helpers/test_indexes.py | 3 +- .../executor/build/_helpers/test_retention.py | 2 +- .../executor/build/_helpers/test_scheduler.py | 2 +- .../build/_helpers/test_source_node.py | 3 +- .../test_build_scheduler_diagnostics.py | 2 +- .../classes/test_build_scheduler_identity.py | 2 +- .../test_build_scheduler_test_order.py | 3 +- .../clone/_helpers/fingerprinting/helpers.py | 3 +- .../executor/clone/_helpers/helpers.py | 10 +- .../sqlbuild/executor/janitor/main/helpers.py | 10 +- .../executor/migrations/_helpers/helpers.py | 2 +- .../executor/pipeline/_helpers/helpers.py | 10 +- .../pipeline/_helpers/test_auditing.py | 2 +- .../pipeline/_helpers/test_scenario.py | 8 +- .../pipeline/_helpers/test_testing.py | 8 +- .../executor/pipeline/main/helpers.py | 3 +- .../pipeline/main/test_schema_preflight.py | 11 +- .../sqlbuild/executor/run/_helpers/helpers.py | 6 +- .../run/_helpers/test_cursor_bounds.py | 3 +- .../executor/run/_helpers/test_hooks.py | 2 +- .../test_snapshot_adapter_boundary.py | 3 +- .../executor/scenario/_helpers/helpers.py | 6 +- .../executor/scenario/main/helpers.py | 6 +- .../sqlbuild/executor/seed/main/helpers.py | 8 +- .../sqlbuild/executor/testing/main/helpers.py | 3 +- .../src/sqlbuild/integrations/dbt/helpers.py | 8 +- .../sqlbuild/integrations/dbt/test_execute.py | 2 +- .../rule_engine/_helpers/engine/helpers.py | 3 +- .../_helpers/engine/test_custom_rules.py | 2 +- .../_helpers/engine/test_native_facts.py | 6 +- .../classes/audit_facts/helpers.py | 8 +- .../model_test_facts/test_model_test_facts.py | 2 +- .../classes/shared_fact_encoder/helpers.py | 2 +- .../rule_engine/main/evaluate/_test_types.py | 2 +- .../rule_engine/main/evaluate/helpers.py | 6 +- .../main/evaluate/test_evaluate.py | 3 +- 286 files changed, 2633 insertions(+), 1480 deletions(-) create mode 100644 crates/sqlbuild-python/src/bindings/_helpers/record_behaviours/compiled.rs create mode 100644 crates/sqlbuild-python/src/bindings/_helpers/record_behaviours/lineage_facts.rs create mode 100644 crates/sqlbuild-python/src/bindings/_helpers/record_behaviours/macros.rs delete mode 100644 src/sqlbuild/compiler/compile/models.py diff --git a/crates/sqlbuild-cache/src/hosted_shapes/constants.rs b/crates/sqlbuild-cache/src/hosted_shapes/constants.rs index 40d6468874..e29f5f9244 100644 --- a/crates/sqlbuild-cache/src/hosted_shapes/constants.rs +++ b/crates/sqlbuild-cache/src/hosted_shapes/constants.rs @@ -3,7 +3,7 @@ /// The extension module hosting the shapes. pub const NATIVE_MODULE: &str = "sqlbuild._native"; /// The Python model type a natively hosted compiler shape is captured as. -pub const HOSTED_SHAPE_TYPES: [(&str, &str); 450] = [ +pub const HOSTED_SHAPE_TYPES: [(&str, &str); 470] = [ ( "sqlbuild._native:AlignedInterval", "sqlbuild.cursor_algebra.models:AlignedInterval", @@ -104,6 +104,10 @@ pub const HOSTED_SHAPE_TYPES: [(&str, &str); 450] = [ "sqlbuild._native:ColumnTransformKind", "sqlbuild.compiler.lineage.types:ColumnTransformKind", ), + ( + "sqlbuild._native:CompactLineageFacts", + "sqlbuild.compiler.compile.models:CompactLineageFacts", + ), ( "sqlbuild._native:CompileAdapterContext", "sqlbuild.compiler.compile.models:CompileAdapterContext", @@ -128,6 +132,10 @@ pub const HOSTED_SHAPE_TYPES: [(&str, &str); 450] = [ "sqlbuild._native:CompileModelConfig", "sqlbuild.compiler.compile.models:CompileModelConfig", ), + ( + "sqlbuild._native:CompileModelInput", + "sqlbuild.compiler.compile.models:CompileModelInput", + ), ( "sqlbuild._native:CompileModelSqlTestInputPayload", "sqlbuild.compiler.compile.models:CompileModelSqlTestInputPayload", @@ -144,6 +152,10 @@ pub const HOSTED_SHAPE_TYPES: [(&str, &str); 450] = [ "sqlbuild._native:CompileProjectInputs", "sqlbuild.compiler.compile.models:CompileProjectInputs", ), + ( + "sqlbuild._native:CompileSqlReference", + "sqlbuild.compiler.compile.models:CompileSqlReference", + ), ( "sqlbuild._native:CompileSqlScenarioCte", "sqlbuild.compiler.compile.models:CompileSqlScenarioCte", @@ -160,6 +172,10 @@ pub const HOSTED_SHAPE_TYPES: [(&str, &str); 450] = [ "sqlbuild._native:CompileSqlTestInput", "sqlbuild.compiler.compile.models:CompileSqlTestInput", ), + ( + "sqlbuild._native:CompiledAudit", + "sqlbuild.compiler.compile.models:CompiledAudit", + ), ( "sqlbuild._native:CompiledDirectLogicSqlTestPayload", "sqlbuild.compiler.compile.models:CompiledDirectLogicSqlTestPayload", @@ -172,14 +188,34 @@ pub const HOSTED_SHAPE_TYPES: [(&str, &str); 450] = [ "sqlbuild._native:CompiledLineageColumnFact", "sqlbuild.compiler.compile.models:CompiledLineageColumnFact", ), + ( + "sqlbuild._native:CompiledLineageSourceFact", + "sqlbuild.compiler.compile.models:CompiledLineageSourceFact", + ), + ( + "sqlbuild._native:CompiledModel", + "sqlbuild.compiler.compile.models:CompiledModel", + ), ( "sqlbuild._native:CompiledModelSqlTestPayload", "sqlbuild.compiler.compile.models:CompiledModelSqlTestPayload", ), + ( + "sqlbuild._native:CompiledObjectKey", + "sqlbuild.compiler.compile.models:CompiledObjectKey", + ), + ( + "sqlbuild._native:CompiledProject", + "sqlbuild.compiler.compile.models:CompiledProject", + ), ( "sqlbuild._native:CompiledProjectPhaseResult", "sqlbuild.compiler.pipeline.models:CompiledProjectPhaseResult", ), + ( + "sqlbuild._native:CompiledRelationLocation", + "sqlbuild.compiler.compile.models:CompiledRelationLocation", + ), ( "sqlbuild._native:CompiledSeed", "sqlbuild.compiler.compile.models:CompiledSeed", @@ -192,10 +228,22 @@ pub const HOSTED_SHAPE_TYPES: [(&str, &str); 450] = [ "sqlbuild._native:CompiledSqlExpansion", "sqlbuild.compiler.compile.models:CompiledSqlExpansion", ), + ( + "sqlbuild._native:CompiledSqlScenario", + "sqlbuild.compiler.compile.models:CompiledSqlScenario", + ), + ( + "sqlbuild._native:CompiledSqlTest", + "sqlbuild.compiler.compile.models:CompiledSqlTest", + ), ( "sqlbuild._native:CompiledSqlTestResource", "sqlbuild.compiler.compile.models:CompiledSqlTestResource", ), + ( + "sqlbuild._native:CompilerDiagnostic", + "sqlbuild.compiler.compile.models:CompilerDiagnostic", + ), ( "sqlbuild._native:CompilerStage", "sqlbuild.compiler.frontier.types:CompilerStage", @@ -364,6 +412,10 @@ pub const HOSTED_SHAPE_TYPES: [(&str, &str); 450] = [ "sqlbuild._native:DeclarationReport", "sqlbuild.compiler.scopes.models:DeclarationReport", ), + ( + "sqlbuild._native:DeclarationResolutionContext", + "sqlbuild.compiler.compile.models:DeclarationResolutionContext", + ), ( "sqlbuild._native:DeclarationRuntimeProjection", "sqlbuild.compiler.compile.models:DeclarationRuntimeProjection", @@ -740,6 +792,10 @@ pub const HOSTED_SHAPE_TYPES: [(&str, &str); 450] = [ "sqlbuild._native:IncrementalStrategy", "sqlbuild.compiler.planner.types:IncrementalStrategy", ), + ( + "sqlbuild._native:InferredColumn", + "sqlbuild.compiler.compile.models:InferredColumn", + ), ( "sqlbuild._native:InitialValidFrom", "sqlbuild.compiler.planner.types:InitialValidFrom", @@ -818,6 +874,10 @@ pub const HOSTED_SHAPE_TYPES: [(&str, &str); 450] = [ "sqlbuild._native:LiteralSqlRelation", "sqlbuild.compiler.references.models:LiteralSqlRelation", ), + ( + "sqlbuild._native:LoadedMacro", + "sqlbuild.compiler.compile.models:LoadedMacro", + ), ( "sqlbuild._native:LoaderConnectionMode", "sqlbuild.compiler.discovery.types:LoaderConnectionMode", @@ -846,6 +906,10 @@ pub const HOSTED_SHAPE_TYPES: [(&str, &str); 450] = [ "sqlbuild._native:MacroArgumentError", "sqlbuild.compiler.compile.exceptions:MacroArgumentError", ), + ( + "sqlbuild._native:MacroContext", + "sqlbuild.compiler.compile.models:MacroContext", + ), ( "sqlbuild._native:MacroExpansionResult", "sqlbuild.compiler.compile.models:MacroExpansionResult", @@ -958,6 +1022,10 @@ pub const HOSTED_SHAPE_TYPES: [(&str, &str); 450] = [ "sqlbuild._native:ModelCursorSnapshot", "sqlbuild.compiler.planner.models:ModelCursorSnapshot", ), + ( + "sqlbuild._native:ModelInputBuildContext", + "sqlbuild.compiler.compile.models:ModelInputBuildContext", + ), ( "sqlbuild._native:ModelMigrationPlanEntry", "sqlbuild.compiler.planner.models:ModelMigrationPlanEntry", @@ -1710,6 +1778,18 @@ pub const HOSTED_SHAPE_TYPES: [(&str, &str); 450] = [ "sqlbuild._native:StaticCommandContext", "sqlbuild.compiler.pipeline.models:StaticCommandContext", ), + ( + "sqlbuild._native:StaticMacroExport", + "sqlbuild.compiler.compile.models:StaticMacroExport", + ), + ( + "sqlbuild._native:StaticMacroFault", + "sqlbuild.compiler.compile.models:StaticMacroFault", + ), + ( + "sqlbuild._native:StaticMacroInventory", + "sqlbuild.compiler.compile.models:StaticMacroInventory", + ), ( "sqlbuild._native:TableType", "sqlbuild.spec.contracts.types:TableType", diff --git a/crates/sqlbuild-python/src/bindings/_helpers/analysis/explicit_references.rs b/crates/sqlbuild-python/src/bindings/_helpers/analysis/explicit_references.rs index a45a5041db..c9b585108b 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/analysis/explicit_references.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/analysis/explicit_references.rs @@ -32,7 +32,7 @@ use crate::bindings::types::CompilerDetach; const EXPLICIT_REFERENCES_CONTEXT: &str = "native explicit-reference checks"; const COMPILE_ERRORS_MODULE: &str = "sqlbuild._native"; const COMPILE_INPUT_ERROR: &str = "CompileInputError"; -const COMPILE_MODELS_MODULE: &str = "sqlbuild.compiler.compile.models"; +const COMPILE_MODELS_MODULE: &str = "sqlbuild._native"; const REFERENCE_MODELS_MODULE: &str = "sqlbuild.compiler.references.models"; const CONTRACT_MODELS_MODULE: &str = "sqlbuild._native"; const REFERENCE_TYPES_MODULE: &str = "sqlbuild._native"; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/analysis/required_columns.rs b/crates/sqlbuild-python/src/bindings/_helpers/analysis/required_columns.rs index c7620efbe8..99bd354073 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/analysis/required_columns.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/analysis/required_columns.rs @@ -12,7 +12,7 @@ use crate::bindings::_helpers::boundary::panics::value_error; use crate::bindings::_helpers::compile_driver::python_calls::{call_with, imported}; use crate::bindings::types::CompilerDetach; -const COMPILE_MODELS_MODULE: &str = "sqlbuild.compiler.compile.models"; +const COMPILE_MODELS_MODULE: &str = "sqlbuild._native"; /// CTE-aware column reads of `query_sql`, resolved to the external resources it references. #[pyfunction] diff --git a/crates/sqlbuild-python/src/bindings/_helpers/analysis_session/fixture_columns.rs b/crates/sqlbuild-python/src/bindings/_helpers/analysis_session/fixture_columns.rs index 2ff4d0fb6d..1c6dfec894 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/analysis_session/fixture_columns.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/analysis_session/fixture_columns.rs @@ -19,7 +19,7 @@ use crate::bindings::types::CompilerDetach; const NATIVE_RECORDS_MODULE: &str = "sqlbuild._native"; const ANALYSIS_MODELS_MODULE: &str = "sqlbuild._native"; -const COMPILE_MODELS_MODULE: &str = "sqlbuild.compiler.compile.models"; +const COMPILE_MODELS_MODULE: &str = "sqlbuild._native"; const LINEAGE_TYPES_MODULE: &str = "sqlbuild.compiler.lineage.types"; const PROJECTION_FACTS_CONTEXT: &str = "native fixture projection facts"; const LEGACY_MODE: &str = "legacy"; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/analysis_session/outcome_objects.rs b/crates/sqlbuild-python/src/bindings/_helpers/analysis_session/outcome_objects.rs index 28a3b907ff..9769df9baa 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/analysis_session/outcome_objects.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/analysis_session/outcome_objects.rs @@ -7,10 +7,13 @@ use pyo3::types::{PyDict, PyDictMethods, PyTuple}; use pyo3::{pyfunction, wrap_pyfunction}; use crate::bindings::_helpers::boundary::keyword_request::Request; +use crate::bindings::_helpers::record_behaviours::lineage_facts::{ + confidence_member, transform_kind_member, +}; const NATIVE_RECORDS_MODULE: &str = "sqlbuild._native"; -const COMPILE_MODELS_MODULE: &str = "sqlbuild.compiler.compile.models"; +const COMPILE_MODELS_MODULE: &str = "sqlbuild._native"; const LINEAGE_TYPES_MODULE: &str = "sqlbuild.compiler.lineage.types"; const LINEAGE_NATIVE: &str = "native"; @@ -233,7 +236,6 @@ fn lineage_facts<'py>(rows: &Bound<'py, PyAny>) -> PyResult> { let column_class: Bound<'py, PyAny> = shape_class(models.as_any(), "CompiledLineageColumnFact")?; let source_class: Bound<'py, PyAny> = models.getattr("CompiledLineageSourceFact")?; - let compact: Bound<'py, PyAny> = models.getattr("CompactLineageFacts")?; let mut facts: Vec> = Vec::new(); for row in rows.try_iter()? { let (output_column, transform_code, confidence_code, sources): ( @@ -265,11 +267,11 @@ fn lineage_facts<'py>(rows: &Bound<'py, PyAny>) -> PyResult> { ("upstream_columns", PyTuple::new(py, upstream)?.into_any()), ( "transform_kind", - compact.call_method1("transform_kind", (transform_code,))?, + transform_kind_member(py, transform_code.extract()?)?.into_bound(py), ), ( "confidence", - compact.call_method1("confidence", (confidence_code,))?, + confidence_member(py, confidence_code.extract()?)?.into_bound(py), ), ], )?); diff --git a/crates/sqlbuild-python/src/bindings/_helpers/analysis_session/stage_functions.rs b/crates/sqlbuild-python/src/bindings/_helpers/analysis_session/stage_functions.rs index ef50a1c090..3635f6b608 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/analysis_session/stage_functions.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/analysis_session/stage_functions.rs @@ -33,7 +33,7 @@ const LINEAGE_RESOURCE_TYPES: [(&str, &str); 4] = [ ("table_fn", "table_fn"), ]; const ANALYSIS_MODELS_MODULE: &str = "sqlbuild._native"; -const COMPILE_MODELS_MODULE: &str = "sqlbuild.compiler.compile.models"; +const COMPILE_MODELS_MODULE: &str = "sqlbuild._native"; const LINEAGE_TYPES_MODULE: &str = "sqlbuild.compiler.lineage.types"; /// Each query's `(succeeded, column rows, has star)`; a native internal failure raises. diff --git a/crates/sqlbuild-python/src/bindings/_helpers/build_command/lifecycle.rs b/crates/sqlbuild-python/src/bindings/_helpers/build_command/lifecycle.rs index f1e02d7543..968c5fc8c1 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/build_command/lifecycle.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/build_command/lifecycle.rs @@ -21,7 +21,7 @@ const NATIVE_RECORDS_MODULE: &str = "sqlbuild._native"; const NATIVE_MODULE: &str = "sqlbuild._native"; const LOADER_DAG_MODULE: &str = "sqlbuild._native"; -const COMPILE_MODELS: &str = "sqlbuild.compiler.compile.models"; +const COMPILE_MODELS: &str = "sqlbuild._native"; const COMPILE_TYPES: &str = "sqlbuild.compiler.compile.types"; const INGRESS_MODULE: &str = "sqlbuild.executor.python_nodes.main._ingress"; const READ_SIDE_MODULE: &str = "sqlbuild.executor.python_nodes.main.read_side"; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/check/check_command.rs b/crates/sqlbuild-python/src/bindings/_helpers/check/check_command.rs index 3a9c5024c0..7e3ff2fac1 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/check/check_command.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/check/check_command.rs @@ -34,7 +34,7 @@ const REJECT_UNIT_TESTS_MODULE: &str = "sqlbuild._native"; const STATIC_COMMAND_MODULE: &str = "sqlbuild._native"; const RELATION_PLAN_MODULE: &str = "sqlbuild._native"; const SELECTOR_PARSE_MODULE: &str = "sqlbuild._native"; -const COMPILE_MODELS: &str = "sqlbuild.compiler.compile.models"; +const COMPILE_MODELS: &str = "sqlbuild._native"; const COMPILE_TYPES: &str = "sqlbuild.compiler.compile.types"; const PYTHON_NODE_TYPES: &str = "sqlbuild.compiler.python_nodes.types"; const EXECUTOR_MODELS: &str = "sqlbuild.executor.python_nodes.models"; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/clone/clone_execution.rs b/crates/sqlbuild-python/src/bindings/_helpers/clone/clone_execution.rs index 1fedbc14e2..a0f6fd6b81 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/clone/clone_execution.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/clone/clone_execution.rs @@ -15,7 +15,7 @@ use crate::bindings::_helpers::clone::clone_progress::{ use crate::bindings::_helpers::compile_driver::python_calls::{call_with, imported, within}; const NATIVE_MODULE: &str = "sqlbuild._native"; -const COMPILE_MODELS: &str = "sqlbuild.compiler.compile.models"; +const COMPILE_MODELS: &str = "sqlbuild._native"; const COMPILE_TYPES: &str = "sqlbuild.compiler.compile.types"; const CLONE_MODELS: &str = "sqlbuild.executor.clone.models"; const RETENTION_REQUESTS_MODULE: &str = "sqlbuild.executor.clone.main._build_retention_requests"; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compile_attachment/model_inputs.rs b/crates/sqlbuild-python/src/bindings/_helpers/compile_attachment/model_inputs.rs index bda1aed684..0bf6e22f21 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compile_attachment/model_inputs.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compile_attachment/model_inputs.rs @@ -19,7 +19,7 @@ use crate::bindings::_helpers::render::macro_expansion_result::macro_expansion_c const NATIVE_RECORDS_MODULE: &str = "sqlbuild._native"; const NATIVE_MODULE: &str = "sqlbuild._native"; -const COMPILE_MODELS_MODULE: &str = "sqlbuild.compiler.compile.models"; +const COMPILE_MODELS_MODULE: &str = "sqlbuild._native"; const COMPILE_ERRORS_MODULE: &str = "sqlbuild._native"; const DISCOVERY_MODELS_MODULE: &str = "sqlbuild._native"; const PROJECT_CONFIG_FILENAME: &str = "sqlbuild_project.toml"; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compile_attachment/sql_expansion_context.rs b/crates/sqlbuild-python/src/bindings/_helpers/compile_attachment/sql_expansion_context.rs index 7f8b50d169..bf28cee411 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compile_attachment/sql_expansion_context.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compile_attachment/sql_expansion_context.rs @@ -12,7 +12,7 @@ const NATIVE_RECORDS_MODULE: &str = "sqlbuild._native"; const NATIVE_MODULE: &str = "sqlbuild._native"; const USER_MACROS_MODULE: &str = "sqlbuild.compiler.compile._helpers.render.user_macros"; -const COMPILE_MODELS_MODULE: &str = "sqlbuild.compiler.compile.models"; +const COMPILE_MODELS_MODULE: &str = "sqlbuild._native"; /// Build the declaration scope of `discovered`; a static scope lends only its index to macros. fn declaration_scope<'py>( diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compile_command/report.rs b/crates/sqlbuild-python/src/bindings/_helpers/compile_command/report.rs index fcbb4a85bf..eeb8de479d 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compile_command/report.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compile_command/report.rs @@ -25,7 +25,7 @@ use crate::bindings::_helpers::semantic_shapes::coverage::semantic_coverage; const NATIVE_RECORDS_MODULE: &str = "sqlbuild._native"; const HOOK_IDENTITIES_MODULE: &str = "sqlbuild.compiler.python_nodes.main._hook_identities"; -const COMPILE_MODELS_MODULE: &str = "sqlbuild.compiler.compile.models"; +const COMPILE_MODELS_MODULE: &str = "sqlbuild._native"; const METADATA_MODULE: &str = "importlib.metadata"; const TARGET_DIRECTORY: &str = "target"; const COMPILED_DIRECTORY: &str = "compiled"; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compile_command/target_writer.rs b/crates/sqlbuild-python/src/bindings/_helpers/compile_command/target_writer.rs index 532e6d8a22..c0ede29fd4 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compile_command/target_writer.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compile_command/target_writer.rs @@ -48,7 +48,7 @@ const PUBLISHED_STAGING_CHANGED: &str = "changed"; const PUBLISHED_TREE: &str = "tree"; const OUTPUT_MODELS_MODULE: &str = "sqlbuild._native"; const CLI_EXCEPTIONS_MODULE: &str = "sqlbuild._native"; -const COMPILE_MODELS_MODULE: &str = "sqlbuild.compiler.compile.models"; +const COMPILE_MODELS_MODULE: &str = "sqlbuild._native"; const PLANNER_EXCEPTIONS_MODULE: &str = "sqlbuild._native"; const MODEL_CHAIN_MODULE: &str = "sqlbuild._native"; const PLAN_ARTIFACTS_MODULE: &str = "sqlbuild._native"; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/analysis.rs b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/analysis.rs index 2997c586de..1700417c93 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/analysis.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/analysis.rs @@ -31,7 +31,7 @@ const CONTRACTS_MODULE: &str = "sqlbuild._native"; const CONTRACT_MODELS_MODULE: &str = "sqlbuild._native"; const SELECTION_MODULE: &str = "sqlbuild._native"; const RULE_THREADS_MODULE: &str = "sqlbuild._native"; -const COMPILE_MODELS_MODULE: &str = "sqlbuild.compiler.compile.models"; +const COMPILE_MODELS_MODULE: &str = "sqlbuild._native"; const CLI_COMPILE_MODELS_MODULE: &str = "sqlbuild._native"; const GENERIC_DIALECT: &str = "generic"; const TARGET_DIRECTORY: &str = "target"; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/assembly_checks.rs b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/assembly_checks.rs index fb915fda00..7f9878328f 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/assembly_checks.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/assembly_checks.rs @@ -10,7 +10,7 @@ use crate::bindings::_helpers::render::declaration_values::declaration_scope_res const NATIVE_RECORDS_MODULE: &str = "sqlbuild._native"; -const COMPILE_MODELS_MODULE: &str = "sqlbuild.compiler.compile.models"; +const COMPILE_MODELS_MODULE: &str = "sqlbuild._native"; const COMPILE_TYPES_MODULE: &str = "sqlbuild.compiler.compile.types"; const NATIVE_MODULE: &str = "sqlbuild._native"; /// The `CompiledSqlScenario` fields copied from its input, in Python's keyword order. diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/assembly_project.rs b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/assembly_project.rs index 28e30a54b9..e848b2a2cf 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/assembly_project.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/assembly_project.rs @@ -24,7 +24,7 @@ const NATIVE_MODULE: &str = "sqlbuild._native"; pub(crate) const ADAPTER_NAME_MODULE: &str = "sqlbuild._native"; const ADAPTER_TYPES_MODULE: &str = "sqlbuild.adapter.contract.types"; const SCENARIO_CONFIG_MODULE: &str = "sqlbuild._native"; -const COMPILE_MODELS_MODULE: &str = "sqlbuild.compiler.compile.models"; +const COMPILE_MODELS_MODULE: &str = "sqlbuild._native"; const SOURCE_SLOT: &str = "source"; const SEED_SLOT: &str = "seed"; const AUDIT_SLOT: &str = "audit"; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/deferral.rs b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/deferral.rs index 6385633870..a07e375fd3 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/deferral.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/deferral.rs @@ -16,7 +16,7 @@ use crate::bindings::_helpers::compile_driver::python_calls::{ const NATIVE_RECORDS_MODULE: &str = "sqlbuild._native"; const PLANNER_EXCEPTIONS_MODULE: &str = "sqlbuild._native"; -const COMPILE_MODELS_MODULE: &str = "sqlbuild.compiler.compile.models"; +const COMPILE_MODELS_MODULE: &str = "sqlbuild._native"; const PROJECT_VARS_MODULE: &str = "sqlbuild._native"; const PRESERVE_TARGET_VALUE: &str = "preserve"; /// `sorted(DEFERRED_TARGET_CONTEXT_KEYS)`. diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/inputs.rs b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/inputs.rs index 944e325e08..95088a29f3 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/inputs.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/inputs.rs @@ -14,7 +14,7 @@ const ATTACHMENT_MODULE: &str = "sqlbuild._native"; const DECLARATION_SCOPE_MODULE: &str = "sqlbuild._native"; const COLLECTOR_MODULE: &str = "sqlbuild._native"; const MACROS_MODULE: &str = "sqlbuild.compiler.compile._helpers.render.user_macros"; -const COMPILE_MODELS_MODULE: &str = "sqlbuild.compiler.compile.models"; +const COMPILE_MODELS_MODULE: &str = "sqlbuild._native"; const ATTACH_CHECKS_MODULE: &str = "sqlbuild._native"; const ATTACH_RESOURCES_MODULE: &str = "sqlbuild._native"; const INPUTS_VIEW_MODULE: &str = "sqlbuild._native"; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/phases.rs b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/phases.rs index 043957d495..3f400630ca 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/phases.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/phases.rs @@ -20,7 +20,7 @@ const ADAPTER_NAME_MODULE: &str = "sqlbuild._native"; const OUTPUT_MODELS_MODULE: &str = "sqlbuild._native"; const COMPILE_EXCEPTIONS_MODULE: &str = "sqlbuild._native"; const PLANNER_EXCEPTIONS_MODULE: &str = "sqlbuild._native"; -const COMPILE_MODELS_MODULE: &str = "sqlbuild.compiler.compile.models"; +const COMPILE_MODELS_MODULE: &str = "sqlbuild._native"; const EMPTY_DAG_PATH: &str = ""; const TARGET_DIRECTORY: &str = "target"; const DEFAULT_DAG_FILE: &str = "sqlbuild_dag.json"; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/target_defaults.rs b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/target_defaults.rs index c0cbc5c347..18b224e71d 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/target_defaults.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/target_defaults.rs @@ -6,7 +6,7 @@ use pyo3::types::PyTuple; use crate::bindings::_helpers::compile_driver::python_calls::{call_with, imported, keywords}; const NATIVE_MODULE: &str = "sqlbuild._native"; -const COMPILE_MODELS_MODULE: &str = "sqlbuild.compiler.compile.models"; +const COMPILE_MODELS_MODULE: &str = "sqlbuild._native"; /// The adapter defaults every compiled location falls back to. pub(crate) struct TargetDefaults<'py> { diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compile_records/constants.rs b/crates/sqlbuild-python/src/bindings/_helpers/compile_records/constants.rs index 35b0657118..b868c74650 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compile_records/constants.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compile_records/constants.rs @@ -1167,6 +1167,430 @@ const R_DISCOVERED_RUNTIME_EXTENSIONS: RecordSpec = RecordSpec { FieldSpec::new("lifecycle_shutdown_timeout_seconds", FieldDefault::None), ], }; +const R_COMPILER_DIAGNOSTIC: RecordSpec = RecordSpec { + name: "CompilerDiagnostic", + doc: "One project diagnostic produced by compile-time checks.", + fields: &[ + FieldSpec::required("phase"), + FieldSpec::required("severity"), + FieldSpec::required("code"), + FieldSpec::required("message"), + FieldSpec::new("resource_type", FieldDefault::None), + FieldSpec::new("resource_name", FieldDefault::None), + FieldSpec::new("column_name", FieldDefault::None), + FieldSpec::new("path", FieldDefault::None), + FieldSpec::new("line", FieldDefault::None), + FieldSpec::new("column", FieldDefault::None), + FieldSpec::new("location", FieldDefault::None), + FieldSpec::new("related_locations", FieldDefault::EmptyTuple), + FieldSpec::new("help", FieldDefault::None), + FieldSpec::new("notes", FieldDefault::EmptyTuple), + FieldSpec::new("affected_rules", FieldDefault::EmptyTuple), + ], +}; +const R_INFERRED_COLUMN: RecordSpec = RecordSpec { + name: "InferredColumn", + doc: "One output column inferred from query SQL via sql_analysis parsing.", + fields: &[ + FieldSpec::required("name"), + FieldSpec::new("type", FieldDefault::None), + FieldSpec::new( + "nullability", + FieldDefault::Member( + "sqlbuild.compiler.lineage.types", + "InferredNullability", + "UNKNOWN", + ), + ), + ], +}; +const R_LOADED_MACRO: RecordSpec = RecordSpec { + name: "LoadedMacro", + doc: "One loaded project macro available for compile-time SQL expansion.", + fields: &[ + FieldSpec::required("name"), + FieldSpec::required("file_path"), + FieldSpec::required("relative_path"), + FieldSpec::required("raw_source"), + FieldSpec::required("function"), + FieldSpec::new("dependencies", FieldDefault::EmptyTuple), + ], +}; +const R_STATIC_MACRO_EXPORT: RecordSpec = RecordSpec { + name: "StaticMacroExport", + doc: "Value-free AST inventory for one project-owned macro export.", + fields: &[ + FieldSpec::required("name"), + FieldSpec::required("relative_path"), + FieldSpec::required("parameters"), + FieldSpec::required("line"), + FieldSpec::required("source_digest"), + FieldSpec::new("dependencies", FieldDefault::EmptyTuple), + ], +}; +const R_STATIC_MACRO_FAULT: RecordSpec = RecordSpec { + name: "StaticMacroFault", + doc: "One value-free static macro inventory fault.", + fields: &[ + FieldSpec::required("relative_path"), + FieldSpec::required("message"), + ], +}; +const R_STATIC_MACRO_INVENTORY: RecordSpec = RecordSpec { + name: "StaticMacroInventory", + doc: "AST macro exports and faults collected without module execution.", + fields: &[ + FieldSpec::new("exports", FieldDefault::EmptyTuple), + FieldSpec::new("faults", FieldDefault::EmptyTuple), + ], +}; +const R_MACRO_CONTEXT: RecordSpec = RecordSpec { + name: "MacroContext", + doc: "Expose target settings and caller-visible declarations to a Python SQL macro.", + fields: &[ + FieldSpec::required("adapter_name"), + FieldSpec::required("sql_analysis_enabled"), + FieldSpec::required("target_name"), + FieldSpec::new("vars", FieldDefault::EmptyDict), + FieldSpec::new("constants", FieldDefault::EmptyDict), + FieldSpec::new("enums", FieldDefault::EmptyDict), + FieldSpec { + name: "_value_renderer", + default: FieldDefault::None, + compared: false, + }, + FieldSpec { + name: "_collection_rendering", + default: FieldDefault::Member("sqlbuild._native", "CollectionRendering", "VALUE_LIST"), + compared: false, + }, + FieldSpec { + name: "_constant_declarations", + default: FieldDefault::EmptyDict, + compared: false, + }, + FieldSpec { + name: "_enforce_explicit_references", + default: FieldDefault::True, + compared: false, + }, + ], +}; +const R_DECLARATION_RESOLUTION_CONTEXT: RecordSpec = RecordSpec { + name: "DeclarationResolutionContext", + doc: "Declarations visible during one authored SQL expansion.", + fields: &[ + FieldSpec::new("enums", FieldDefault::EmptyDict), + FieldSpec::new("constants", FieldDefault::EmptyDict), + FieldSpec::new("inaccessible_enums", FieldDefault::EmptyDict), + FieldSpec::new("inaccessible_constants", FieldDefault::EmptyDict), + FieldSpec::new("enum_visibility", FieldDefault::EmptyDict), + FieldSpec::new("constant_visibility", FieldDefault::EmptyDict), + FieldSpec::new("macros", FieldDefault::EmptyDict), + FieldSpec::new("macro_records", FieldDefault::EmptyDict), + FieldSpec::new("macro_visibility", FieldDefault::EmptyDict), + FieldSpec::new("inaccessible_macros", FieldDefault::EmptyDict), + FieldSpec::new("consumer", FieldDefault::None), + ], +}; +const R_MODEL_INPUT_BUILD_CONTEXT: RecordSpec = RecordSpec { + name: "ModelInputBuildContext", + doc: "Run-constant config and macros for building model compile inputs.", + fields: &[ + FieldSpec::required("effective_vars"), + FieldSpec::required("effective_settings"), + FieldSpec::required("target_config"), + FieldSpec::required("effective_target_name"), + FieldSpec::required("run_id"), + FieldSpec::required("macro_context"), + FieldSpec::required("loaded_macros"), + FieldSpec::required("value_renderer"), + FieldSpec::required("collection_rendering"), + FieldSpec::required("sql_lexical_syntax"), + FieldSpec::new("public_enums", FieldDefault::EmptyDict), + FieldSpec::new("public_constants", FieldDefault::EmptyDict), + FieldSpec::new("public_model_schemas", FieldDefault::EmptyDict), + FieldSpec::new("declaration_resolver", FieldDefault::None), + ], +}; +const R_COMPILE_SQL_REFERENCE: RecordSpec = RecordSpec { + name: "CompileSqlReference", + doc: "One logical SQL reference discovered from compiled SQL text.", + fields: &[ + FieldSpec::required("ref_kind"), + FieldSpec::required("ref_name"), + FieldSpec::new("ref_package", FieldDefault::None), + FieldSpec::new("call_argument_count", FieldDefault::None), + ], +}; +const R_COMPILED_LINEAGE_SOURCE_FACT: RecordSpec = RecordSpec { + name: "CompiledLineageSourceFact", + doc: "Compact upstream column fact extracted during SQL analysis.", + fields: &[ + FieldSpec::required("resource_type"), + FieldSpec::required("resource_name"), + FieldSpec::required("column_name"), + ], +}; +const R_COMPACT_LINEAGE_FACTS: RecordSpec = RecordSpec { + name: "CompactLineageFacts", + doc: "Indexed native lineage rows with lazy object projection.", + fields: &[ + FieldSpec::required("string_pool"), + FieldSpec::required("rows"), + FieldSpec::new("resource_name_indexes", FieldDefault::EmptyDict), + FieldSpec::new("resource_names", FieldDefault::EmptyDict), + ], +}; +const R_COMPILED_OBJECT_KEY: RecordSpec = RecordSpec { + name: "CompiledObjectKey", + doc: "Stable logical identity for one compiled resource or external dependency.", + fields: &[ + FieldSpec::required("resource_type"), + FieldSpec::required("name"), + ], +}; +const R_COMPILE_MODEL_INPUT: RecordSpec = RecordSpec { + name: "CompileModelInput", + doc: "One discovered model file with its attached schema metadata, if any.", + fields: &[ + FieldSpec::required("model_file"), + FieldSpec::new( + "config", + FieldDefault::Call("sqlbuild._native", "CompileModelConfig"), + ), + FieldSpec::new("query_sql", FieldDefault::Text("")), + FieldSpec::new("macro_source_sql", FieldDefault::Text("")), + FieldSpec::new("references", FieldDefault::EmptyTuple), + FieldSpec::new("schema_entry", FieldDefault::None), + FieldSpec::new("schema_file", FieldDefault::None), + FieldSpec::new("sql_validation_enabled", FieldDefault::False), + FieldSpec::new("rejected_sql_analysis_opt_out", FieldDefault::None), + FieldSpec::new("enum_declarations", FieldDefault::EmptyTuple), + FieldSpec::new("constant_declarations", FieldDefault::EmptyTuple), + FieldSpec::new("enum_columns", FieldDefault::EmptyDict), + FieldSpec::new("macro_deps", FieldDefault::EmptyTuple), + FieldSpec::new("macro_usages", FieldDefault::EmptyTuple), + FieldSpec::new("declaration_usages", FieldDefault::EmptyTuple), + FieldSpec { + name: "sql_expansion", + default: FieldDefault::None, + compared: false, + }, + FieldSpec { + name: "lint_expansion", + default: FieldDefault::None, + compared: false, + }, + ], +}; +const R_COMPILED_RELATION_LOCATION: RecordSpec = RecordSpec { + name: "CompiledRelationLocation", + doc: "Logical and physical relation location resolved during compile.", + fields: &[ + FieldSpec::required("database"), + FieldSpec::required("schema"), + FieldSpec::required("name"), + FieldSpec::required("qualified_name"), + FieldSpec::new("logical_schema", FieldDefault::None), + FieldSpec::new("logical_database", FieldDefault::None), + ], +}; +const R_COMPILED_MODEL: RecordSpec = RecordSpec { + name: "CompiledModel", + doc: "Planner-ready compiled model metadata.", + fields: &[ + FieldSpec::required("key"), + FieldSpec::required("deps"), + FieldSpec::required("name"), + FieldSpec::required("relative_path"), + FieldSpec::required("query_sql"), + FieldSpec::required("config"), + FieldSpec::required("destination"), + FieldSpec::new("references", FieldDefault::EmptyTuple), + FieldSpec::new("schema_entry", FieldDefault::None), + FieldSpec::new("inferred_columns", FieldDefault::None), + FieldSpec::new("fast_lineage_columns", FieldDefault::None), + FieldSpec::new("fast_lineage_has_star", FieldDefault::False), + FieldSpec::new("fast_lineage_star_resolved", FieldDefault::False), + FieldSpec::new("authored_sql", FieldDefault::Text("")), + FieldSpec::new("authored_query_sql", FieldDefault::Text("")), + FieldSpec::new("output_column_locations", FieldDefault::EmptyDict), + FieldSpec::new("extract_implicit_alias_columns", FieldDefault::True), + FieldSpec::new("macro_deps", FieldDefault::EmptyTuple), + FieldSpec::new("enum_declarations", FieldDefault::EmptyTuple), + FieldSpec::new("constant_declarations", FieldDefault::EmptyTuple), + FieldSpec::new("enum_columns", FieldDefault::EmptyDict), + FieldSpec::new("binding_diagnostics", FieldDefault::EmptyTuple), + FieldSpec::new("binding_validated", FieldDefault::False), + FieldSpec::new("dynamic_column_contract", FieldDefault::None), + FieldSpec::new("unchecked_output_columns", FieldDefault::EmptyFrozenset), + FieldSpec::new("rejected_sql_analysis_opt_out", FieldDefault::None), + ], +}; +const R_COMPILED_AUDIT: RecordSpec = RecordSpec { + name: "CompiledAudit", + doc: "Compiled audit metadata selected by scope dependencies.", + fields: &[ + FieldSpec::required("key"), + FieldSpec::required("scope_deps"), + FieldSpec::required("name"), + FieldSpec::required("definition_name"), + FieldSpec::required("audit_file"), + FieldSpec::required("audit_block"), + FieldSpec::required("sql_body"), + FieldSpec::new( + "evaluation_mode", + FieldDefault::Member( + "sqlbuild.compiler.auditing.types", + "AuditEvaluationMode", + "VIOLATIONS", + ), + ), + FieldSpec::new("measurement_contract", FieldDefault::None), + FieldSpec::new("thresholds", FieldDefault::None), + FieldSpec::new("minimum_samples", FieldDefault::None), + FieldSpec::new("measure_sql", FieldDefault::None), + FieldSpec::new("evidence_sql", FieldDefault::None), + FieldSpec::new("evidence_limit", FieldDefault::None), + FieldSpec::new("references", FieldDefault::EmptyTuple), + FieldSpec::new("attached_target_kind", FieldDefault::None), + FieldSpec::new("attached_target_name", FieldDefault::None), + FieldSpec::new("attached_column_name", FieldDefault::None), + FieldSpec::new("severity", FieldDefault::None), + FieldSpec::new("run_scope", FieldDefault::None), + FieldSpec::new("always_run", FieldDefault::False), + FieldSpec::new("description", FieldDefault::None), + ], +}; +const R_COMPILED_SQL_SCENARIO: RecordSpec = RecordSpec { + name: "CompiledSqlScenario", + doc: "Compiled SQL-native scenario metadata selected by inferred graph targets.", + fields: &[ + FieldSpec::required("key"), + FieldSpec::required("name"), + FieldSpec::required("scenario_file"), + FieldSpec::required("sql_body"), + FieldSpec::new("authored_ctes", FieldDefault::EmptyTuple), + FieldSpec::new("expected_ctes", FieldDefault::EmptyTuple), + FieldSpec::new("assertion_ctes", FieldDefault::EmptyTuple), + FieldSpec::new("source_fixture_names", FieldDefault::EmptyTuple), + FieldSpec::new("ref_fixture_names", FieldDefault::EmptyTuple), + FieldSpec::new("seed_fixture_names", FieldDefault::EmptyTuple), + FieldSpec::new("dbt_ref_fixture_names", FieldDefault::EmptyTuple), + FieldSpec::new("expected_model_names", FieldDefault::EmptyTuple), + FieldSpec::new("assertion_names", FieldDefault::EmptyTuple), + FieldSpec::new("assertion_target_model_names", FieldDefault::EmptyTuple), + FieldSpec::new("target_model_names", FieldDefault::EmptyTuple), + FieldSpec::new("source_path", FieldDefault::None), + FieldSpec::new("ownership_root", FieldDefault::None), + ], +}; +const R_COMPILED_PROJECT: RecordSpec = RecordSpec { + name: "CompiledProject", + doc: "Planner-ready whole-project compile output.", + fields: &[ + FieldSpec::required("run_id"), + FieldSpec::required("effective_target_name"), + FieldSpec::required("effective_connection"), + FieldSpec::required("effective_vars"), + FieldSpec { + name: "binding_catalog", + default: FieldDefault::None, + compared: false, + }, + FieldSpec::new("effective_target_database", FieldDefault::None), + FieldSpec::new("effective_target_schema", FieldDefault::None), + FieldSpec::new("sql_analysis_dialect", FieldDefault::None), + FieldSpec::new( + "sql_lexical_syntax", + FieldDefault::Call("sqlbuild._native", "SqlLexicalSyntax"), + ), + FieldSpec::new("compile_cache_dir", FieldDefault::None), + FieldSpec::new( + "settings", + FieldDefault::Call("sqlbuild._native", "SettingsConfig"), + ), + FieldSpec::new( + "scenario", + FieldDefault::Call("sqlbuild._native", "ScenarioConfig"), + ), + FieldSpec::new("models", FieldDefault::EmptyTuple), + FieldSpec::new("sources", FieldDefault::EmptyTuple), + FieldSpec::new("seeds", FieldDefault::EmptyTuple), + FieldSpec::new("functions", FieldDefault::EmptyTuple), + FieldSpec::new("audits", FieldDefault::EmptyTuple), + FieldSpec::new("sql_tests", FieldDefault::EmptyTuple), + FieldSpec::new("sql_scenarios", FieldDefault::EmptyTuple), + FieldSpec::new("loader_functions", FieldDefault::EmptyTuple), + FieldSpec::new("hook_functions", FieldDefault::EmptyTuple), + FieldSpec::new("enforce_explicit_references", FieldDefault::True), + FieldSpec::new("sql_hook_files", FieldDefault::EmptyTuple), + FieldSpec::new("materialization_files", FieldDefault::EmptyTuple), + FieldSpec::new("public_enums", FieldDefault::EmptyDict), + FieldSpec::new("public_constants", FieldDefault::EmptyDict), + FieldSpec::new("loaded_macros", FieldDefault::EmptyDict), + FieldSpec::new("diagnostics", FieldDefault::EmptyTuple), + FieldSpec::new("unmatched_literal_sql_relations", FieldDefault::EmptyTuple), + FieldSpec::new("external_sql_reference_resolver", FieldDefault::None), + FieldSpec::new( + "scope_index", + FieldDefault::Call("sqlbuild._native", "ScopeIndex"), + ), + FieldSpec { + name: "sql_expansions", + default: FieldDefault::EmptyDict, + compared: false, + }, + FieldSpec { + name: "native_session", + default: FieldDefault::None, + compared: false, + }, + FieldSpec { + name: "native_project", + default: FieldDefault::None, + compared: false, + }, + ], +}; +const R_COMPILED_SQL_TEST: RecordSpec = RecordSpec { + name: "CompiledSqlTest", + doc: "Compiled SQL-native unit test metadata selected by expected targets.", + fields: &[ + FieldSpec::required("key"), + FieldSpec::required("scope_deps"), + FieldSpec::required("name"), + FieldSpec::required("test_file"), + FieldSpec::required("test_block"), + FieldSpec::required("sql_body"), + FieldSpec::new( + "mode", + FieldDefault::Member("sqlbuild.compiler.compile.types", "SqlTestMode", "MODEL"), + ), + FieldSpec::new( + "payload", + FieldDefault::Call("sqlbuild._native", "CompiledModelSqlTestPayload"), + ), + FieldSpec::new("source_path", FieldDefault::None), + FieldSpec::new("ownership_root", FieldDefault::None), + FieldSpec::new("block_index", FieldDefault::None), + FieldSpec::new("explicit_name", FieldDefault::None), + FieldSpec::new("parent_name", FieldDefault::None), + FieldSpec::new("case_name", FieldDefault::None), + FieldSpec::new("case_index", FieldDefault::None), + FieldSpec::new("case_fingerprint", FieldDefault::None), + FieldSpec::new("parameter_schema", FieldDefault::EmptyTuple), + FieldSpec::new("parameter_values", FieldDefault::EmptyTuple), + FieldSpec::new("expected_model_names", FieldDefault::EmptyTuple), + FieldSpec::new("assertion_names", FieldDefault::EmptyTuple), + FieldSpec::new("assertion_target_model_names", FieldDefault::EmptyTuple), + FieldSpec::new("read_helper_names", FieldDefault::EmptyTuple), + FieldSpec::new("reference_target_model_names", FieldDefault::EmptyTuple), + FieldSpec::new("target_model_names", FieldDefault::EmptyTuple), + FieldSpec::new("tested_resources", FieldDefault::EmptyTuple), + ], +}; /// The compile and discovery value records. pub(crate) const COMPILE_VALUE_RECORDS: &[RecordSpec] = &[ @@ -1257,4 +1681,24 @@ pub(crate) const COMPILE_VALUE_RECORDS: &[RecordSpec] = &[ R_DISCOVERED_PYTHON_NODE_FUNCTIONS, R_DISCOVERED_PROJECT_INPUTS, R_DISCOVERED_RUNTIME_EXTENSIONS, + R_COMPILER_DIAGNOSTIC, + R_INFERRED_COLUMN, + R_LOADED_MACRO, + R_STATIC_MACRO_EXPORT, + R_STATIC_MACRO_FAULT, + R_STATIC_MACRO_INVENTORY, + R_MACRO_CONTEXT, + R_DECLARATION_RESOLUTION_CONTEXT, + R_MODEL_INPUT_BUILD_CONTEXT, + R_COMPILE_SQL_REFERENCE, + R_COMPILED_LINEAGE_SOURCE_FACT, + R_COMPACT_LINEAGE_FACTS, + R_COMPILED_OBJECT_KEY, + R_COMPILE_MODEL_INPUT, + R_COMPILED_RELATION_LOCATION, + R_COMPILED_MODEL, + R_COMPILED_AUDIT, + R_COMPILED_SQL_SCENARIO, + R_COMPILED_PROJECT, + R_COMPILED_SQL_TEST, ]; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/attachment.rs b/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/attachment.rs index 39daf1c150..ec2abe5bcb 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/attachment.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/attachment.rs @@ -29,7 +29,7 @@ const NATIVE_RECORDS_MODULE: &str = "sqlbuild._native"; const NATIVE_RECORDS: &str = "sqlbuild._native"; const COMPILE_ERRORS_MODULE: &str = "sqlbuild._native"; -const COMPILE_MODELS_MODULE: &str = "sqlbuild.compiler.compile.models"; +const COMPILE_MODELS_MODULE: &str = "sqlbuild._native"; const COMPILE_TYPES_MODULE: &str = "sqlbuild.compiler.compile.types"; const SCOPE_TYPES_MODULE: &str = "sqlbuild._native"; const SCOPE_MODELS_MODULE: &str = "sqlbuild._native"; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/compiled_model_analysis.rs b/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/compiled_model_analysis.rs index dfd411e73c..fc7c9d8483 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/compiled_model_analysis.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/compiled_model_analysis.rs @@ -22,6 +22,9 @@ use crate::bindings::_helpers::compiled_project::compiled_model_view::{ CompiledViewClasses, InferredRow, }; use crate::bindings::_helpers::compiled_project::model_input_view::ModelInputRecord; +use crate::bindings::_helpers::record_behaviours::lineage_facts::{ + confidence_member, transform_kind_member, +}; const COMPILE_PHASE: &str = "compile"; const MODEL_RESOURCE: &str = "model"; @@ -296,15 +299,11 @@ fn lineage_facts(classes: &CompiledViewClasses<'_>, rows: &[LineageRow]) -> PyRe fields.set_item("upstream_columns", PyTuple::new(py, sources)?)?; fields.set_item( "transform_kind", - classes - .compact_lineage_facts - .call_method1("transform_kind", (row.transform_code,))?, + transform_kind_member(py, usize::from(row.transform_code))?, )?; fields.set_item( "confidence", - classes - .compact_lineage_facts - .call_method1("confidence", (row.confidence_code,))?, + confidence_member(py, usize::from(row.confidence_code))?, )?; facts.push( classes diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/loop_classes.rs b/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/loop_classes.rs index 4759a5be83..083f4d9d8d 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/loop_classes.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/loop_classes.rs @@ -8,7 +8,7 @@ use crate::bindings::_helpers::compile_driver::python_calls::imported; const NATIVE_RECORDS_MODULE: &str = "sqlbuild._native"; -const COMPILE_MODELS_MODULE: &str = "sqlbuild.compiler.compile.models"; +const COMPILE_MODELS_MODULE: &str = "sqlbuild._native"; const COMPILE_TYPES_MODULE: &str = "sqlbuild.compiler.compile.types"; const COMPILE_ERRORS_MODULE: &str = "sqlbuild._native"; const AUDITING_TYPES_MODULE: &str = "sqlbuild.compiler.auditing.types"; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/records.rs b/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/records.rs index e286a8f9a0..846deceaae 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/records.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/records.rs @@ -16,7 +16,7 @@ use crate::bindings::_helpers::compiled_project::project_view::NativeProjectView const NATIVE_RECORDS_MODULE: &str = "sqlbuild._native"; -const COMPILE_MODELS_MODULE: &str = "sqlbuild.compiler.compile.models"; +const COMPILE_MODELS_MODULE: &str = "sqlbuild._native"; const LINEAGE_TYPES_MODULE: &str = "sqlbuild.compiler.lineage.types"; const DATACLASSES_MODULE: &str = "dataclasses"; const DECIMAL_KIND: &str = "decimal"; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/contracts/contract_diagnostics.rs b/crates/sqlbuild-python/src/bindings/_helpers/contracts/contract_diagnostics.rs index ff294bfbdb..5de21e3b7c 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/contracts/contract_diagnostics.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/contracts/contract_diagnostics.rs @@ -16,7 +16,7 @@ use crate::bindings::_helpers::sqltext::model_headers::{ColumnLocation, location const NATIVE_RECORDS_MODULE: &str = "sqlbuild._native"; const NATIVE_MODULE: &str = "sqlbuild._native"; -const COMPILE_MODELS_MODULE: &str = "sqlbuild.compiler.compile.models"; +const COMPILE_MODELS_MODULE: &str = "sqlbuild._native"; const COMPILE_TYPES_MODULE: &str = "sqlbuild.compiler.compile.types"; const CONTRACT_MODELS_MODULE: &str = "sqlbuild._native"; const SPEC_MODELS_MODULE: &str = "sqlbuild._native"; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/dbt_interop/reference_resolver.rs b/crates/sqlbuild-python/src/bindings/_helpers/dbt_interop/reference_resolver.rs index 88847322fd..d7803540f0 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/dbt_interop/reference_resolver.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/dbt_interop/reference_resolver.rs @@ -13,7 +13,7 @@ use crate::bindings::_helpers::compile_driver::python_calls::{ const NATIVE_MODULE: &str = "sqlbuild._native"; const COMPILE_ERRORS_MODULE: &str = "sqlbuild._native"; -const COMPILE_MODELS_MODULE: &str = "sqlbuild.compiler.compile.models"; +const COMPILE_MODELS_MODULE: &str = "sqlbuild._native"; const LIFECYCLE_MODULE: &str = "sqlbuild.runtime.observability.classes.operation_lifecycle"; const VALIDATE_OPTIONS: (&[&str], &[&str]) = ( &[ diff --git a/crates/sqlbuild-python/src/bindings/_helpers/freshness/planning.rs b/crates/sqlbuild-python/src/bindings/_helpers/freshness/planning.rs index c2e5ca131a..637249be3c 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/freshness/planning.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/freshness/planning.rs @@ -25,7 +25,7 @@ const NATIVE_RECORDS_MODULE: &str = "sqlbuild._native"; const NATIVE_RECORDS: &str = "sqlbuild._native"; const ADAPTER_STRATEGY: &str = "adapter"; -const COMPILE_MODELS: &str = "sqlbuild.compiler.compile.models"; +const COMPILE_MODELS: &str = "sqlbuild._native"; const COMPILE_TYPES: &str = "sqlbuild.compiler.compile.types"; const NO_CONFIG_MESSAGE: &str = "no freshness config and adapter metadata unavailable"; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/graph/execution_gates.rs b/crates/sqlbuild-python/src/bindings/_helpers/graph/execution_gates.rs index 5352ea008f..b8bdd2f925 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/graph/execution_gates.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/graph/execution_gates.rs @@ -13,7 +13,7 @@ const GRAPH_MODELS_MODULE: &str = "sqlbuild._native"; const GRAPH_TYPES_MODULE: &str = "sqlbuild._native"; const DISCOVERY_MODELS_MODULE: &str = "sqlbuild._native"; const COMPILE_TYPES_MODULE: &str = "sqlbuild.compiler.compile.types"; -const COMPILE_MODELS_MODULE: &str = "sqlbuild.compiler.compile.models"; +const COMPILE_MODELS_MODULE: &str = "sqlbuild._native"; const PATHLIB_MODULE: &str = "pathlib"; /// The model config keys whose entries are hooks, in lifecycle order. const MODEL_HOOK_KEYS: [&str; 2] = ["pre_hooks", "post_hooks"]; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/graph/keys.rs b/crates/sqlbuild-python/src/bindings/_helpers/graph/keys.rs index e80d6eb2e2..87705c188c 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/graph/keys.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/graph/keys.rs @@ -3,7 +3,7 @@ use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult, Python}; use pyo3::types::{PyDict, PyDictMethods}; -const COMPILE_MODELS_MODULE: &str = "sqlbuild.compiler.compile.models"; +const COMPILE_MODELS_MODULE: &str = "sqlbuild._native"; const COMPILE_TYPES_MODULE: &str = "sqlbuild.compiler.compile.types"; /// One `CompiledObjectKey` per native pair, so shared keys convert once. diff --git a/crates/sqlbuild-python/src/bindings/_helpers/hand_built/compiled_model.rs b/crates/sqlbuild-python/src/bindings/_helpers/hand_built/compiled_model.rs index c20ac302ce..ab3123db9e 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/hand_built/compiled_model.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/hand_built/compiled_model.rs @@ -5,7 +5,7 @@ use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult, Python}; use pyo3::types::{PyDict, PyList, PyString, PyTuple}; const NATIVE_MODULE: &str = "sqlbuild._native"; -const COMPILE_MODELS_MODULE: &str = "sqlbuild.compiler.compile.models"; +const COMPILE_MODELS_MODULE: &str = "sqlbuild._native"; const COMPILE_TYPES_MODULE: &str = "sqlbuild.compiler.compile.types"; const LINEAGE_TYPES_MODULE: &str = "sqlbuild.compiler.lineage.types"; const CONTRACT_MODELS_MODULE: &str = "sqlbuild._native"; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/lineage/column_lineage_graph.rs b/crates/sqlbuild-python/src/bindings/_helpers/lineage/column_lineage_graph.rs index 53c7ff78cf..f7ec90bca9 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/lineage/column_lineage_graph.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/lineage/column_lineage_graph.rs @@ -11,6 +11,9 @@ use sqlbuild_analysis::lineage_graph::main::edge_index::edge_index; use sqlbuild_analysis::lineage_graph::models::{EdgeIndex, EdgeKey}; use crate::bindings::_helpers::lineage::lineage_objects::LineageClasses; +use crate::bindings::_helpers::record_behaviours::lineage_facts::{ + confidence_member, transform_kind_member, +}; /// One compact model's facts and root-star flag. struct CompactModel { @@ -470,7 +473,10 @@ impl RecordBuilder { } } -/// `facts.transform_kind(code)` and `facts.confidence(code)`, looked up once per code. +/// Looks up the enum member of one native code. +type CodeLookup = fn(Python<'_>, usize) -> PyResult>; + +/// The transform kind and confidence members of native codes, looked up once per code. #[derive(Default)] struct CodeMembers { members: HashMap<(&'static str, i64), Py>, @@ -478,26 +484,24 @@ struct CodeMembers { impl CodeMembers { fn transform(&mut self, facts: &Bound<'_, PyAny>, code: i64) -> PyResult> { - self.member(facts, "transform_kind", code) + self.member(facts.py(), ("transform_kind", transform_kind_member), code) } fn confidence(&mut self, facts: &Bound<'_, PyAny>, code: i64) -> PyResult> { - self.member(facts, "confidence", code) + self.member(facts.py(), ("confidence", confidence_member), code) } fn member( &mut self, - facts: &Bound<'_, PyAny>, - method: &'static str, + py: Python<'_>, + (kind, lookup): (&'static str, CodeLookup), code: i64, ) -> PyResult> { - if let Some(member) = self.members.get(&(method, code)) { - return Ok(member.clone_ref(facts.py())); + if let Some(member) = self.members.get(&(kind, code)) { + return Ok(member.clone_ref(py)); } - let member: Py = facts.call_method1(method, (code,))?.unbind(); - let _ = self - .members - .insert((method, code), member.clone_ref(facts.py())); + let member: Py = lookup(py, usize::try_from(code)?)?; + let _ = self.members.insert((kind, code), member.clone_ref(py)); Ok(member) } } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/lineage/lineage_index.rs b/crates/sqlbuild-python/src/bindings/_helpers/lineage/lineage_index.rs index 5b18e3cbef..5edcd89fd8 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/lineage/lineage_index.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/lineage/lineage_index.rs @@ -6,7 +6,7 @@ use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult, Python}; use pyo3::types::{PyDict, PyDictMethods, PyFrozenSet, PyTuple}; use sqlbuild_analysis::lineage_command::models::{LineageIndex, LineageKey, LineageNode}; -const COMPILE_MODELS: &str = "sqlbuild.compiler.compile.models"; +const COMPILE_MODELS: &str = "sqlbuild._native"; const GRAPH_MODELS: &str = "sqlbuild._native"; /// A Python `CompiledObjectKey` as a lineage key. diff --git a/crates/sqlbuild-python/src/bindings/_helpers/lineage/lineage_objects.rs b/crates/sqlbuild-python/src/bindings/_helpers/lineage/lineage_objects.rs index 31b7d98744..a55df86374 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/lineage/lineage_objects.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/lineage/lineage_objects.rs @@ -7,7 +7,7 @@ use sqlbuild_analysis::lineage::models::{LineageColumn, LineageSource}; const LINEAGE_MODELS_MODULE: &str = "sqlbuild._native"; const LINEAGE_TYPES_MODULE: &str = "sqlbuild.compiler.lineage.types"; -const COMPILE_MODELS_MODULE: &str = "sqlbuild.compiler.compile.models"; +const COMPILE_MODELS_MODULE: &str = "sqlbuild._native"; const COMPILE_TYPES_MODULE: &str = "sqlbuild.compiler.compile.types"; const LOGGING_MODULE: &str = "logging"; const DEBUG_EVENT_MODULE: &str = "sqlbuild.diagnostics.main.log_debug_event"; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_entries/values.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_entries/values.rs index c2ee234c33..f35a07d9ae 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_entries/values.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_entries/values.rs @@ -14,7 +14,7 @@ pub(super) const MICROBATCH_CONCURRENCY_MESSAGE: &str = "batch_concurrency > 1 requires concurrent microbatches; selected model(s): "; pub(super) const COMPILE_EXCEPTIONS: &str = "sqlbuild._native"; pub(super) const COMPILE_TYPES: &str = "sqlbuild.compiler.compile.types"; -pub(super) const COMPILE_MODELS: &str = "sqlbuild.compiler.compile.models"; +pub(super) const COMPILE_MODELS: &str = "sqlbuild._native"; pub(super) const DISCOVERY_MODELS: &str = "sqlbuild._native"; /// `sqlbuild._native.name(**pairs)`. diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/old_name_objects.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/old_name_objects.rs index 2ac012fb39..140befe160 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/old_name_objects.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/old_name_objects.rs @@ -31,11 +31,7 @@ pub(super) fn location<'py>( )?; fields.push(("qualified_name", qualified)); call_with( - &imported( - runtime.py(), - "sqlbuild.compiler.compile.models", - "CompiledRelationLocation", - )?, + &imported(runtime.py(), "sqlbuild._native", "CompiledRelationLocation")?, &fields, ) } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/planning_origin.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/planning_origin.rs index 279a1daf0d..975eaeb167 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/planning_origin.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/planning_origin.rs @@ -19,11 +19,7 @@ pub(super) fn location<'py>( let mut fields = fields.to_vec(); fields.push(("qualified_name", qualified)); call_with( - &imported( - runtime.py(), - "sqlbuild.compiler.compile.models", - "CompiledRelationLocation", - )?, + &imported(runtime.py(), "sqlbuild._native", "CompiledRelationLocation")?, &fields, ) } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_phases/objects.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_phases/objects.rs index cc5e3514be..cce89afb2c 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_phases/objects.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_phases/objects.rs @@ -161,7 +161,7 @@ pub(crate) fn model_key<'py>( name: &Bound<'py, PyAny>, ) -> PyResult> { call_with( - &imported(py, "sqlbuild.compiler.compile.models", "CompiledObjectKey")?, + &imported(py, "sqlbuild._native", "CompiledObjectKey")?, &[ ( "resource_type", diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_scenario/relations.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_scenario/relations.rs index d5aca78b04..cb8e14f3d1 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_scenario/relations.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_scenario/relations.rs @@ -142,11 +142,7 @@ fn target_for_artifact<'py>( ], )?; call_with( - &imported( - py, - "sqlbuild.compiler.compile.models", - "CompiledRelationLocation", - )?, + &imported(py, "sqlbuild._native", "CompiledRelationLocation")?, &[ ("database", database), ("schema", schema), diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_strategy/audits.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_strategy/audits.rs index fbf692a524..b0e46157c8 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_strategy/audits.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_strategy/audits.rs @@ -16,7 +16,7 @@ use crate::bindings::_helpers::planner_strategy::values::{ }; const AUDITING_TYPES_MODULE: &str = "sqlbuild.compiler.auditing.types"; -const COMPILE_MODELS_MODULE: &str = "sqlbuild.compiler.compile.models"; +const COMPILE_MODELS_MODULE: &str = "sqlbuild._native"; const COMPILE_TYPES_MODULE: &str = "sqlbuild.compiler.compile.types"; const MODEL_RESOURCE_TYPE: &str = "model"; const FINAL_RUN_SCOPE: &str = "final"; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_strategy/constants.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_strategy/constants.rs index 8c7249e73c..b306b03409 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_strategy/constants.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_strategy/constants.rs @@ -3,7 +3,7 @@ pub(crate) const NATIVE_MODULE: &str = "sqlbuild._native"; pub(crate) const AUDIT_TYPES_MODULE: &str = "sqlbuild.compiler.auditing.types"; pub(crate) const COMPILE_TYPES_MODULE: &str = "sqlbuild.compiler.compile.types"; -pub(crate) const COMPILE_MODELS_MODULE: &str = "sqlbuild.compiler.compile.models"; +pub(crate) const COMPILE_MODELS_MODULE: &str = "sqlbuild._native"; pub(crate) const PLAN_MODELS_MODULE: &str = "sqlbuild._native"; pub(crate) const MODEL_ATTACHMENT: &str = "model"; pub(crate) const SOURCE_ATTACHMENT: &str = "source"; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/record_behaviours/compiled.rs b/crates/sqlbuild-python/src/bindings/_helpers/record_behaviours/compiled.rs new file mode 100644 index 0000000000..d04e1d558e --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/record_behaviours/compiled.rs @@ -0,0 +1,384 @@ +//! Normalization and derived views of the natively hosted compile output records. + +use pyo3::IntoPyObjectExt; +use pyo3::prelude::{Bound, Py, PyAny, PyAnyMethods, PyErr, PyResult, Python}; +use pyo3::types::{PyDict, PyDictMethods, PyList, PyListMethods, PyTuple}; + +use crate::bindings::_helpers::compile_driver::python_calls::imported; +use crate::bindings::_helpers::records::constants::NATIVE_MODULE; +use crate::bindings::_helpers::rules::relation_keys::entry_relation_key_row; + +/// Module of the compiled resource type enum. +const COMPILE_TYPES_MODULE: &str = "sqlbuild.compiler.compile.types"; +/// `CompilerDiagnostic` positions. +const DIAGNOSTIC_PHASE_FIELD: usize = 0; +const DIAGNOSTIC_SEVERITY_FIELD: usize = 1; +const DIAGNOSTIC_RESOURCE_TYPE_FIELD: usize = 4; +const DIAGNOSTIC_PATH_FIELD: usize = 7; +const DIAGNOSTIC_LINE_FIELD: usize = 8; +const DIAGNOSTIC_COLUMN_FIELD: usize = 9; +const DIAGNOSTIC_LOCATION_FIELD: usize = 10; +/// `CompiledSqlScenario` positions. +const SCENARIO_FILE_FIELD: usize = 2; +const SCENARIO_EXPECTED_MODELS_FIELD: usize = 11; +const SCENARIO_ASSERTION_TARGETS_FIELD: usize = 13; +const SCENARIO_TARGET_MODELS_FIELD: usize = 14; +const SCENARIO_SOURCE_PATH_FIELD: usize = 15; +const SCENARIO_OWNERSHIP_ROOT_FIELD: usize = 16; +/// `CompiledSqlTest` positions. +const TEST_FILE_FIELD: usize = 3; +const TEST_BLOCK_FIELD: usize = 4; +const TEST_SOURCE_PATH_FIELD: usize = 8; +const TEST_OWNERSHIP_ROOT_FIELD: usize = 9; +const TEST_BLOCK_INDEX_FIELD: usize = 10; +const TEST_EXPLICIT_NAME_FIELD: usize = 11; +const TEST_PARENT_NAME_FIELD: usize = 12; + +/// `class(value)`. +fn coerce(py: Python<'_>, value: &Py, (module, class): (&str, &str)) -> PyResult> { + Ok(imported(py, module, class)? + .call1((value.bind(py),))? + .unbind()) +} + +/// `value` coerced to `CompiledResourceType` unless it is `None`. +fn coerce_resource_type(py: Python<'_>, value: &Py) -> PyResult> { + if value.is_none(py) { + return Ok(value.clone_ref(py)); + } + coerce(py, value, (COMPILE_TYPES_MODULE, "CompiledResourceType")) +} + +/// `CompilerDiagnostic`: enum fields coerced, and its location and path, line and column aligned. +pub(crate) fn finish_compiler_diagnostic( + py: Python<'_>, + mut values: Vec>, +) -> PyResult>> { + values[DIAGNOSTIC_PHASE_FIELD] = coerce( + py, + &values[DIAGNOSTIC_PHASE_FIELD], + (NATIVE_MODULE, "DiagnosticPhase"), + )?; + values[DIAGNOSTIC_SEVERITY_FIELD] = coerce( + py, + &values[DIAGNOSTIC_SEVERITY_FIELD], + (NATIVE_MODULE, "DiagnosticSeverity"), + )?; + values[DIAGNOSTIC_RESOURCE_TYPE_FIELD] = + coerce_resource_type(py, &values[DIAGNOSTIC_RESOURCE_TYPE_FIELD])?; + let positioned: bool = [ + DIAGNOSTIC_PATH_FIELD, + DIAGNOSTIC_LINE_FIELD, + DIAGNOSTIC_COLUMN_FIELD, + ] + .iter() + .all(|position| !values[*position].is_none(py)); + if values[DIAGNOSTIC_LOCATION_FIELD].is_none(py) && positioned { + let keywords: Bound<'_, PyDict> = PyDict::new(py); + keywords.set_item("path", values[DIAGNOSTIC_PATH_FIELD].bind(py))?; + keywords.set_item("line", values[DIAGNOSTIC_LINE_FIELD].bind(py))?; + keywords.set_item("column", values[DIAGNOSTIC_COLUMN_FIELD].bind(py))?; + let location: Bound<'_, PyAny> = + imported(py, NATIVE_MODULE, "SourceLocation")?.call((), Some(&keywords))?; + values[DIAGNOSTIC_LOCATION_FIELD] = location.unbind(); + } + let location: Bound<'_, PyAny> = values[DIAGNOSTIC_LOCATION_FIELD].bind(py).clone(); + if !location.is_none() { + values[DIAGNOSTIC_PATH_FIELD] = location.getattr("path")?.unbind(); + values[DIAGNOSTIC_LINE_FIELD] = location.getattr("line")?.unbind(); + values[DIAGNOSTIC_COLUMN_FIELD] = location.getattr("column")?.unbind(); + } + Ok(values) +} + +/// `CompilerDiagnostic.is_error`: whether the diagnostic fails the command. +pub(crate) fn diagnostic_is_error(instance: &Bound<'_, PyAny>) -> PyResult> { + let py: Python<'_> = instance.py(); + let error: Bound<'_, PyAny> = + imported(py, NATIVE_MODULE, "DiagnosticSeverity")?.getattr("ERROR")?; + instance.getattr("severity")?.eq(error)?.into_py_any(py) +} + +/// `CompileSqlReference`: its kind coerced to `SqlReferenceKind`. +pub(crate) fn finish_sql_reference( + py: Python<'_>, + mut values: Vec>, +) -> PyResult>> { + values[0] = coerce(py, &values[0], (NATIVE_MODULE, "SqlReferenceKind"))?; + Ok(values) +} + +/// `CompiledLineageSourceFact` and `CompiledObjectKey`: the resource type coerced. +pub(crate) fn finish_resource_typed( + py: Python<'_>, + mut values: Vec>, +) -> PyResult>> { + values[0] = coerce( + py, + &values[0], + (COMPILE_TYPES_MODULE, "CompiledResourceType"), + )?; + Ok(values) +} + +/// `value`, or `fallback` when `value` is `None`. +fn or_fallback(py: Python<'_>, value: &Py, fallback: Bound<'_, PyAny>) -> Py { + if value.is_none(py) { + return fallback.unbind(); + } + value.clone_ref(py) +} + +/// The distinct names of `first` then `second`, in first-seen order. +fn merged_names<'py>( + first: &Bound<'py, PyAny>, + second: &Bound<'py, PyAny>, +) -> PyResult> { + let py: Python<'py> = first.py(); + let seen: Bound<'py, PyDict> = PyDict::new(py); + for name in first.try_iter()?.chain(second.try_iter()?) { + seen.set_item(name?, py.None())?; + } + Ok(PyTuple::new(py, seen.keys())?.into_any()) +} + +/// `CompiledSqlScenario`: source path, ownership and target models taken from its file. +pub(crate) fn finish_sql_scenario( + py: Python<'_>, + mut values: Vec>, +) -> PyResult>> { + let scenario_file: Bound<'_, PyAny> = values[SCENARIO_FILE_FIELD].bind(py).clone(); + values[SCENARIO_SOURCE_PATH_FIELD] = or_fallback( + py, + &values[SCENARIO_SOURCE_PATH_FIELD], + scenario_file.getattr("relative_path")?, + ); + values[SCENARIO_OWNERSHIP_ROOT_FIELD] = or_fallback( + py, + &values[SCENARIO_OWNERSHIP_ROOT_FIELD], + scenario_file.getattr("ownership_root")?, + ); + if !values[SCENARIO_TARGET_MODELS_FIELD].bind(py).is_truthy()? { + let targets: Bound<'_, PyAny> = merged_names( + values[SCENARIO_EXPECTED_MODELS_FIELD].bind(py), + values[SCENARIO_ASSERTION_TARGETS_FIELD].bind(py), + )?; + values[SCENARIO_TARGET_MODELS_FIELD] = targets.unbind(); + } + Ok(values) +} + +/// `CompiledSqlTest`: source path, ownership, block index and names taken from its file and block. +pub(crate) fn finish_sql_test( + py: Python<'_>, + mut values: Vec>, +) -> PyResult>> { + let test_file: Bound<'_, PyAny> = values[TEST_FILE_FIELD].bind(py).clone(); + let test_block: Bound<'_, PyAny> = values[TEST_BLOCK_FIELD].bind(py).clone(); + values[TEST_SOURCE_PATH_FIELD] = or_fallback( + py, + &values[TEST_SOURCE_PATH_FIELD], + test_file.getattr("relative_path")?, + ); + values[TEST_OWNERSHIP_ROOT_FIELD] = or_fallback( + py, + &values[TEST_OWNERSHIP_ROOT_FIELD], + test_file.getattr("ownership_root")?, + ); + values[TEST_BLOCK_INDEX_FIELD] = or_fallback( + py, + &values[TEST_BLOCK_INDEX_FIELD], + test_block.getattr("test_index")?, + ); + let block_name: Bound<'_, PyAny> = test_block.getattr("name")?; + if !block_name.is_none() { + values[TEST_EXPLICIT_NAME_FIELD] = + or_fallback(py, &values[TEST_EXPLICIT_NAME_FIELD], block_name.clone()); + } + if values[TEST_PARENT_NAME_FIELD].is_none(py) { + let parent: Bound<'_, PyAny> = if block_name.is_truthy()? { + block_name + } else { + test_file.getattr("relative_path")?.getattr("stem")? + }; + values[TEST_PARENT_NAME_FIELD] = parent.unbind(); + } + Ok(values) +} + +/// `(kind, name, package, call argument count)` of each reference in `references`. +fn reference_rows(references: &Bound<'_, PyAny>) -> PyResult> { + let py: Python<'_> = references.py(); + let rows: Bound<'_, PyList> = PyList::empty(py); + for reference in references.try_iter()? { + let reference: Bound<'_, PyAny> = reference?; + rows.append(( + reference.getattr("ref_kind")?.str()?, + reference.getattr("ref_name")?, + reference.getattr("ref_package")?, + reference.getattr("call_argument_count")?, + ))?; + } + Ok(rows.into_any().unbind()) +} + +/// `CompileModelInput.reference_rows` and `CompiledModel.reference_rows`. +pub(crate) fn record_reference_rows(instance: &Bound<'_, PyAny>) -> PyResult> { + reference_rows(&instance.getattr("references")?) +} + +/// The declared key facts of model `name` with `instance`'s config and schema entry. +fn model_relation_key_row(instance: &Bound<'_, PyAny>, name: String) -> PyResult> { + let values: Bound<'_, PyAny> = instance.getattr("config")?.getattr("values")?; + entry_relation_key_row( + "ref", + name, + values.cast::()?, + &instance.getattr("schema_entry")?, + )? + .into_py_any(instance.py()) +} + +/// `CompileModelInput.relation_key_row`: keyed by the model file's stem. +pub(crate) fn input_relation_key_row(instance: &Bound<'_, PyAny>) -> PyResult> { + let name: String = instance + .getattr("model_file")? + .getattr("file_path")? + .getattr("stem")? + .extract()?; + model_relation_key_row(instance, name) +} + +/// `CompiledModel.relation_key_row`: keyed by the model name. +pub(crate) fn model_relation_key_row_property(instance: &Bound<'_, PyAny>) -> PyResult> { + let name: String = instance.getattr("name")?.extract()?; + model_relation_key_row(instance, name) +} + +/// `CompileModelInput.attached_schema_name`: the attached schema entry's name, or `None`. +pub(crate) fn input_attached_schema_name(instance: &Bound<'_, PyAny>) -> PyResult> { + let entry: Bound<'_, PyAny> = instance.getattr("schema_entry")?; + if entry.is_none() { + return Ok(entry.unbind()); + } + Ok(entry.getattr("name")?.unbind()) +} + +/// `(resource type, name)` of one compiled object key. +fn key_row<'py>(key: &Bound<'py, PyAny>) -> PyResult<(String, String)> { + Ok(( + key.getattr("resource_type")?.str()?.extract()?, + key.getattr("name")?.extract()?, + )) +} + +/// `(resource type, name)` of each compiled object key. +fn key_rows(keys: &Bound<'_, PyAny>) -> PyResult> { + keys.try_iter()?.map(|key| key_row(&key?)).collect() +} + +/// `CompiledModel.key_row`. +pub(crate) fn model_key_row(instance: &Bound<'_, PyAny>) -> PyResult> { + key_row(&instance.getattr("key")?)?.into_py_any(instance.py()) +} + +/// `CompiledModel.dep_rows`. +pub(crate) fn model_dep_rows(instance: &Bound<'_, PyAny>) -> PyResult> { + key_rows(&instance.getattr("deps")?)?.into_py_any(instance.py()) +} + +/// `CompiledModel.destination_row`: `(database, schema, name)` of the destination. +pub(crate) fn model_destination_row(instance: &Bound<'_, PyAny>) -> PyResult> { + let destination: Bound<'_, PyAny> = instance.getattr("destination")?; + ( + destination.getattr("database")?, + destination.getattr("schema")?, + destination.getattr("name")?, + ) + .into_py_any(instance.py()) +} + +/// `CompiledModel.inferred_rows`: each inferred column as `(name, type, nullability)`. +pub(crate) fn model_inferred_rows(instance: &Bound<'_, PyAny>) -> PyResult> { + let columns: Bound<'_, PyAny> = instance.getattr("inferred_columns")?; + if columns.is_none() { + return Ok(columns.unbind()); + } + let rows: Bound<'_, PyList> = PyList::empty(instance.py()); + for column in columns.try_iter()? { + let column: Bound<'_, PyAny> = column?; + rows.append(( + column.getattr("name")?, + column.getattr("type")?, + column.getattr("nullability")?.str()?, + ))?; + } + Ok(rows.into_any().unbind()) +} + +/// `CompiledModel.schema_column_rows`: each declared schema column as `(name, type)`. +pub(crate) fn model_schema_column_rows(instance: &Bound<'_, PyAny>) -> PyResult> { + let entry: Bound<'_, PyAny> = instance.getattr("schema_entry")?; + if entry.is_none() { + return Ok(entry.unbind()); + } + let rows: Bound<'_, PyList> = PyList::empty(instance.py()); + for column in entry.getattr("columns")?.try_iter()? { + let column: Bound<'_, PyAny> = column?; + rows.append((column.getattr("name")?, column.getattr("type")?))?; + } + Ok(rows.into_any().unbind()) +} + +/// `CompiledModel.has_dynamic_columns`. +pub(crate) fn model_has_dynamic_columns(instance: &Bound<'_, PyAny>) -> PyResult> { + let entry: Bound<'_, PyAny> = instance.getattr("schema_entry")?; + let dynamic: bool = !entry.is_none() && entry.getattr("dynamic_columns")?.is_truthy()?; + dynamic.into_py_any(instance.py()) +} + +/// `CompiledModel.output_column_count`: inferred, else declared, columns, else zero. +pub(crate) fn model_output_column_count(instance: &Bound<'_, PyAny>) -> PyResult> { + let inferred: Bound<'_, PyAny> = instance.getattr("inferred_columns")?; + let entry: Bound<'_, PyAny> = instance.getattr("schema_entry")?; + let count: usize = if !inferred.is_none() { + inferred.len()? + } else if !entry.is_none() { + entry.getattr("columns")?.len()? + } else { + 0 + }; + count.into_py_any(instance.py()) +} + +/// `ModelInputBuildContext.declaration_expansion`: the public declarations for non-model SQL. +pub(crate) fn build_context_declaration_expansion( + instance: &Bound<'_, PyAny>, +) -> PyResult> { + let py: Python<'_> = instance.py(); + let declarations: Bound<'_, PyDict> = PyDict::new(py); + declarations.set_item("enums", instance.getattr("public_enums")?)?; + declarations.set_item("constants", instance.getattr("public_constants")?)?; + let keywords: Bound<'_, PyDict> = PyDict::new(py); + keywords.set_item( + "declarations", + imported(py, NATIVE_MODULE, "DeclarationResolutionContext")? + .call((), Some(&declarations))?, + )?; + keywords.set_item("value_renderer", instance.getattr("value_renderer")?)?; + keywords.set_item( + "collection_rendering", + instance.getattr("collection_rendering")?, + )?; + keywords.set_item("resolver", instance.getattr("declaration_resolver")?)?; + Ok(imported(py, NATIVE_MODULE, "DeclarationExpansionContext")? + .call((), Some(&keywords))? + .unbind()) +} + +/// A `CompileInputError` carrying `message`. +pub(crate) fn compile_input_error(py: Python<'_>, message: &str) -> PyResult { + Ok(PyErr::from_value( + imported(py, NATIVE_MODULE, "CompileInputError")?.call1((message,))?, + )) +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/record_behaviours/constants.rs b/crates/sqlbuild-python/src/bindings/_helpers/record_behaviours/constants.rs index c3b8a68354..eb6fa71ad2 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/record_behaviours/constants.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/record_behaviours/constants.rs @@ -3,6 +3,13 @@ use crate::bindings::_helpers::record_behaviours::auditing::{ finish_measurement_contract, finish_threshold_bound, finish_thresholds, }; +use crate::bindings::_helpers::record_behaviours::compiled::{ + build_context_declaration_expansion, diagnostic_is_error, finish_compiler_diagnostic, + finish_resource_typed, finish_sql_reference, finish_sql_scenario, finish_sql_test, + input_attached_schema_name, input_relation_key_row, model_dep_rows, model_destination_row, + model_has_dynamic_columns, model_inferred_rows, model_key_row, model_output_column_count, + model_relation_key_row_property, model_schema_column_rows, record_reference_rows, +}; use crate::bindings::_helpers::record_behaviours::config::settings_sql_validation; use crate::bindings::_helpers::record_behaviours::cursors::{ finish_cursor_bounds, finish_cursor_snapshot, finish_future_evidence, finish_input_evidence, @@ -28,6 +35,13 @@ use crate::bindings::_helpers::record_behaviours::freshness::{ use crate::bindings::_helpers::record_behaviours::lexical::{ finish_lexical_syntax, lexical_cache_key, lexical_native_mapping, lexical_reads_differently, }; +use crate::bindings::_helpers::record_behaviours::lineage_facts::{ + facts_eq, facts_getitem, facts_iter, facts_len, facts_projected, facts_resource_name, +}; +use crate::bindings::_helpers::record_behaviours::macros::{ + context_render_constant, context_render_enum_member, macro_injects_context, + project_lineage_graph, +}; use crate::bindings::_helpers::record_behaviours::migrations::{ column_decision_blocks_build, column_decision_label, column_decision_records_event, column_decision_renames, column_event_mentions, column_event_renamed, decision_blocks_build, @@ -70,6 +84,12 @@ pub(crate) const BEHAVIOUR_FINISHERS: &[(&str, RecordFinish)] = &[ ("FutureCursorSafetyEvidence", finish_future_evidence), ("MaximumStartSafetyEvidence", finish_maximum_evidence), ("CursorInputRelation", finish_input_relation), + ("CompilerDiagnostic", finish_compiler_diagnostic), + ("CompileSqlReference", finish_sql_reference), + ("CompiledLineageSourceFact", finish_resource_typed), + ("CompiledObjectKey", finish_resource_typed), + ("CompiledSqlScenario", finish_sql_scenario), + ("CompiledSqlTest", finish_sql_test), ]; /// Computed attributes of compiler shapes. pub(crate) const BEHAVIOUR_PROPERTIES: &[(&str, &str, ClassProperty)] = &[ @@ -188,10 +208,56 @@ pub(crate) const BEHAVIOUR_PROPERTIES: &[(&str, &str, ClassProperty)] = &[ ("ProjectGraph", "tag_index", graph_tag_index), ("ProjectGraph", "path_index", graph_path_index), ("ProjectGraph", "all_keys", graph_all_keys), + ("CompilerDiagnostic", "is_error", diagnostic_is_error), + ( + "ModelInputBuildContext", + "declaration_expansion", + build_context_declaration_expansion, + ), + ("CompileModelInput", "reference_rows", record_reference_rows), + ( + "CompileModelInput", + "relation_key_row", + input_relation_key_row, + ), + ( + "CompileModelInput", + "attached_schema_name", + input_attached_schema_name, + ), + ("CompiledModel", "reference_rows", record_reference_rows), + ( + "CompiledModel", + "relation_key_row", + model_relation_key_row_property, + ), + ("CompiledModel", "key_row", model_key_row), + ("CompiledModel", "dep_rows", model_dep_rows), + ("CompiledModel", "destination_row", model_destination_row), + ("CompiledModel", "inferred_rows", model_inferred_rows), + ( + "CompiledModel", + "schema_column_rows", + model_schema_column_rows, + ), + ( + "CompiledModel", + "has_dynamic_columns", + model_has_dynamic_columns, + ), + ( + "CompiledModel", + "output_column_count", + model_output_column_count, + ), ]; /// Computed attributes of compiler shapes that a record computes once and keeps. -pub(crate) const BEHAVIOUR_CACHED_PROPERTIES: &[(&str, &str, ClassProperty)] = - &[("ProjectGraph", "lineage_views", graph_lineage_views)]; +pub(crate) const BEHAVIOUR_CACHED_PROPERTIES: &[(&str, &str, ClassProperty)] = &[ + ("ProjectGraph", "lineage_views", graph_lineage_views), + ("LoadedMacro", "injects_context", macro_injects_context), + ("CompactLineageFacts", "facts", facts_projected), + ("CompiledProject", "lineage_graph", project_lineage_graph), +]; /// Methods of compiler shapes. pub(crate) const BEHAVIOUR_METHODS: &[(&str, &str, ClassMethod)] = &[ ( @@ -242,7 +308,20 @@ pub(crate) const BEHAVIOUR_METHODS: &[(&str, &str, ClassMethod)] = &[ ), ("RulesConfigError", "__init__", rules_config_error_init), ("Duration", "subtract_from", duration_subtract_from), + ("MacroContext", "render_constant", context_render_constant), + ( + "MacroContext", + "render_enum_member", + context_render_enum_member, + ), + ("CompactLineageFacts", "__len__", facts_len), + ("CompactLineageFacts", "__getitem__", facts_getitem), + ("CompactLineageFacts", "__iter__", facts_iter), + ("CompactLineageFacts", "__eq__", facts_eq), + ("CompactLineageFacts", "resource_name", facts_resource_name), ]; +/// Compiler records registered as `collections.abc.Sequence` virtual subclasses. +pub(crate) const BEHAVIOUR_SEQUENCES: &[&str] = &["CompactLineageFacts"]; /// Compiler records ordered by their compared fields. pub(crate) const BEHAVIOUR_ORDERED: &[&str] = &["ResourceIdentity", "DeclarationIdentity", "OwnershipRoot"]; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/record_behaviours/lineage_facts.rs b/crates/sqlbuild-python/src/bindings/_helpers/record_behaviours/lineage_facts.rs new file mode 100644 index 0000000000..21d5d48e4a --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/record_behaviours/lineage_facts.rs @@ -0,0 +1,190 @@ +//! Sequence behaviour of the natively hosted `CompactLineageFacts` record. + +use pyo3::IntoPyObjectExt; +use pyo3::exceptions::PyIndexError; +use pyo3::prelude::{Bound, Py, PyAny, PyAnyMethods, PyResult, Python}; +use pyo3::types::{PyDict, PyDictMethods, PySlice, PyTuple, PyTupleMethods}; + +use crate::bindings::_helpers::compile_driver::python_calls::imported; +use crate::bindings::_helpers::record_behaviours::call_arguments::call_argument; +use crate::bindings::_helpers::records::constants::NATIVE_MODULE; + +/// `ColumnTransformKind` members by native transform code. +const TRANSFORM_KINDS: [&str; 6] = [ + "DIRECT", + "CAST", + "EXPRESSION", + "AGGREGATION", + "STAR", + "CONSTANT", +]; +/// `ColumnLineageConfidence` members by native confidence code. +const CONFIDENCES: [&str; 3] = ["UNKNOWN", "HIGH", "MEDIUM"]; + +/// The `ColumnTransformKind` member of a native transform code. +pub(crate) fn transform_kind_member(py: Python<'_>, code: usize) -> PyResult> { + coded_member(py, ("ColumnTransformKind", &TRANSFORM_KINDS), code) +} + +/// The `ColumnLineageConfidence` member of a native confidence code. +pub(crate) fn confidence_member(py: Python<'_>, code: usize) -> PyResult> { + coded_member(py, ("ColumnLineageConfidence", &CONFIDENCES), code) +} + +/// The enum member at `code` of `members`. +fn coded_member( + py: Python<'_>, + (class, members): (&str, &[&str]), + code: usize, +) -> PyResult> { + let Some(member) = members.get(code) else { + return Err(PyIndexError::new_err("tuple index out of range")); + }; + Ok(imported(py, NATIVE_MODULE, class)? + .getattr(*member)? + .unbind()) +} + +/// `CompactLineageFacts.resource_name(index)`: the model's name for a native relation index. +fn resource_name<'py>( + facts: &Bound<'py, PyAny>, + index: &Bound<'py, PyAny>, +) -> PyResult> { + let names: Bound<'py, PyAny> = facts.getattr("resource_names")?; + let name: Bound<'py, PyAny> = names.call_method1("get", (index,))?; + if !name.is_none() { + return Ok(name); + } + let pooled: Bound<'py, PyAny> = facts + .getattr("resource_name_indexes")? + .call_method1("get", (index, index))?; + facts.getattr("string_pool")?.get_item(pooled) +} + +/// `CompactLineageFacts.resource_name(index)`. +pub(crate) fn facts_resource_name( + instance: &Bound<'_, PyAny>, + args: &Bound<'_, PyTuple>, + kwargs: Option<&Bound<'_, PyDict>>, +) -> PyResult> { + let index: Bound<'_, PyAny> = call_argument((args, kwargs), "index", 0)?; + Ok(resource_name(instance, &index)?.unbind()) +} + +/// One compact row projected to a `CompiledLineageColumnFact`. +fn project_row(facts: &Bound<'_, PyAny>, row: &Bound<'_, PyAny>) -> PyResult> { + let py: Python<'_> = facts.py(); + let pool: Bound<'_, PyAny> = facts.getattr("string_pool")?; + let (name_index, transform_code, confidence_code, sources): ( + Bound<'_, PyAny>, + usize, + usize, + Bound<'_, PyAny>, + ) = row.extract()?; + let source_class: Bound<'_, PyAny> = imported(py, NATIVE_MODULE, "CompiledLineageSourceFact")?; + let mut upstream: Vec> = Vec::new(); + for source in sources.try_iter()? { + let source: Bound<'_, PyTuple> = source?.cast_into::()?; + let keywords: Bound<'_, PyDict> = PyDict::new(py); + keywords.set_item("resource_type", pool.get_item(source.get_item(0)?)?)?; + keywords.set_item("resource_name", resource_name(facts, &source.get_item(1)?)?)?; + keywords.set_item("column_name", pool.get_item(source.get_item(2)?)?)?; + upstream.push(source_class.call((), Some(&keywords))?); + } + let keywords: Bound<'_, PyDict> = PyDict::new(py); + keywords.set_item("output_column", pool.get_item(name_index)?)?; + keywords.set_item("upstream_columns", PyTuple::new(py, upstream)?)?; + keywords.set_item( + "transform_kind", + coded_member( + py, + ("ColumnTransformKind", &TRANSFORM_KINDS), + transform_code, + )?, + )?; + keywords.set_item( + "confidence", + coded_member( + py, + ("ColumnLineageConfidence", &CONFIDENCES), + confidence_code, + )?, + )?; + Ok(imported(py, NATIVE_MODULE, "CompiledLineageColumnFact")? + .call((), Some(&keywords))? + .unbind()) +} + +/// `CompactLineageFacts.facts`: every row projected once, in order. +pub(crate) fn facts_projected(instance: &Bound<'_, PyAny>) -> PyResult> { + let projected: Vec> = instance + .getattr("rows")? + .try_iter()? + .map(|row| project_row(instance, &row?)) + .collect::>()?; + Ok(PyTuple::new(instance.py(), projected)?.into_any().unbind()) +} + +/// `len(facts)`. +pub(crate) fn facts_len( + instance: &Bound<'_, PyAny>, + _args: &Bound<'_, PyTuple>, + _kwargs: Option<&Bound<'_, PyDict>>, +) -> PyResult> { + instance.getattr("rows")?.len()?.into_py_any(instance.py()) +} + +/// `facts[index]`: one fact, or a tuple of facts for a slice. +pub(crate) fn facts_getitem( + instance: &Bound<'_, PyAny>, + args: &Bound<'_, PyTuple>, + _kwargs: Option<&Bound<'_, PyDict>>, +) -> PyResult> { + let index: Bound<'_, PyAny> = args.get_item(0)?; + let projected: Bound<'_, PyAny> = instance.getattr("facts")?; + if index.is_instance_of::() { + return Ok(projected.get_item(index)?.unbind()); + } + let length: isize = isize::try_from(projected.len()?)?; + let requested: isize = index.extract()?; + let normalized: isize = if requested >= 0 { + requested + } else { + length + requested + }; + if normalized < 0 || normalized >= length { + return Err(PyIndexError::new_err(index.unbind())); + } + Ok(projected.get_item(normalized)?.unbind()) +} + +/// `iter(facts)`. +pub(crate) fn facts_iter( + instance: &Bound<'_, PyAny>, + _args: &Bound<'_, PyTuple>, + _kwargs: Option<&Bound<'_, PyDict>>, +) -> PyResult> { + Ok(instance.getattr("facts")?.try_iter()?.into_any().unbind()) +} + +/// `facts == other`: item-wise equality with any sequence. +pub(crate) fn facts_eq( + instance: &Bound<'_, PyAny>, + args: &Bound<'_, PyTuple>, + _kwargs: Option<&Bound<'_, PyDict>>, +) -> PyResult> { + let py: Python<'_> = instance.py(); + let other: Bound<'_, PyAny> = args.get_item(0)?; + let sequence: Bound<'_, PyAny> = py.import("collections.abc")?.getattr("Sequence")?; + if !other.is_instance(&sequence)? { + return Ok(py.NotImplemented()); + } + let mine: Bound<'_, PyAny> = instance.getattr("facts")?; + let theirs: Bound<'_, PyTuple> = PyTuple::new( + py, + other + .try_iter()? + .collect::>>>()?, + )?; + mine.eq(theirs)?.into_py_any(py) +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/record_behaviours/macros.rs b/crates/sqlbuild-python/src/bindings/_helpers/record_behaviours/macros.rs new file mode 100644 index 0000000000..2ba6bba172 --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/record_behaviours/macros.rs @@ -0,0 +1,197 @@ +//! Behaviour of the natively hosted macro records and the compiled project's lineage graph. + +use pyo3::IntoPyObjectExt; +use pyo3::prelude::{Bound, Py, PyAny, PyAnyMethods, PyResult, Python}; +use pyo3::types::{PyDict, PyList, PyTuple, PyTupleMethods}; + +use crate::bindings::_helpers::graph::project_graph::{ResourceRow, project_graph}; +use crate::bindings::_helpers::record_behaviours::call_arguments::call_argument; +use crate::bindings::_helpers::record_behaviours::compiled::compile_input_error; +use crate::bindings::_helpers::render::declaration_values::{ + constant_declaration_sql, render_enum_member_value, +}; + +/// The parameter name through which a macro receives its context. +const MACRO_CONTEXT_PARAMETER_NAME: &str = "ctx"; +/// Parameter kinds that can receive the context positionally. +const POSITIONAL_KINDS: [&str; 2] = ["POSITIONAL_ONLY", "POSITIONAL_OR_KEYWORD"]; + +/// `LoadedMacro.injects_context`: whether the first positional parameter is the macro context. +pub(crate) fn macro_injects_context(instance: &Bound<'_, PyAny>) -> PyResult> { + let py: Python<'_> = instance.py(); + let inspect: Bound<'_, PyAny> = py.import("inspect")?.into_any(); + let parameters: Bound<'_, PyTuple> = PyTuple::new( + py, + inspect + .getattr("signature")? + .call1((instance.getattr("function")?,))? + .getattr("parameters")? + .call_method0("values")? + .try_iter()? + .collect::>>>()?, + )?; + let Ok(first) = parameters.get_item(0) else { + return false.into_py_any(py); + }; + let kind: Bound<'_, PyAny> = first.getattr("kind")?; + let mut positional: bool = false; + for name in POSITIONAL_KINDS { + positional |= kind.eq(inspect.getattr("Parameter")?.getattr(name)?)?; + } + let named: bool = first.getattr("name")?.eq(MACRO_CONTEXT_PARAMETER_NAME)?; + (positional && named).into_py_any(py) +} + +/// `MacroContext.render_constant(name)`: one visible constant as an adapter-safe SQL value. +pub(crate) fn context_render_constant( + instance: &Bound<'_, PyAny>, + args: &Bound<'_, PyTuple>, + kwargs: Option<&Bound<'_, PyDict>>, +) -> PyResult> { + let py: Python<'_> = instance.py(); + let name: Bound<'_, PyAny> = call_argument((args, kwargs), "name", 0)?; + let value_renderer: Bound<'_, PyAny> = instance.getattr("_value_renderer")?; + if value_renderer.is_none() { + return Err(compile_input_error( + py, + "Constant SQL rendering is available only during a project macro invocation", + )?); + } + let _ = instance.getattr("constants")?.get_item(&name)?; + let declaration: Bound<'_, PyAny> = instance + .getattr("_constant_declarations")? + .call_method1("get", (&name,))?; + if declaration.is_none() { + return Err(compile_input_error( + py, + "Constant SQL rendering requires a declaration resolved for this macro invocation", + )?); + } + constant_declaration_sql( + &declaration, + &value_renderer, + (&instance.getattr("_collection_rendering")?, None), + )? + .into_py_any(py) +} + +/// `MacroContext.render_enum_member(*, enum_name, member_name)`: one member as a SQL literal. +pub(crate) fn context_render_enum_member( + instance: &Bound<'_, PyAny>, + args: &Bound<'_, PyTuple>, + kwargs: Option<&Bound<'_, PyDict>>, +) -> PyResult> { + if !args.is_empty() { + return Err(pyo3::exceptions::PyTypeError::new_err( + "render_enum_member() takes keyword arguments enum_name and member_name only", + )); + } + let enum_name: Bound<'_, PyAny> = call_argument((args, kwargs), "enum_name", 0)?; + let member_name: Bound<'_, PyAny> = call_argument((args, kwargs), "member_name", 1)?; + let value: Bound<'_, PyAny> = instance + .getattr("enums")? + .get_item(enum_name)? + .get_item(member_name)?; + render_enum_member_value(&value)?.into_py_any(instance.py()) +} + +/// `(resource type, name)` of one compiled object key. +fn graph_key(key: &Bound<'_, PyAny>) -> PyResult<(String, String)> { + Ok(( + key.getattr("resource_type")?.str()?.extract()?, + key.getattr("name")?.extract()?, + )) +} + +/// `(resource type, name)` of each compiled object key. +fn graph_keys(keys: &Bound<'_, PyAny>) -> PyResult> { + keys.try_iter()?.map(|key| graph_key(&key?)).collect() +} + +/// The texts of a list or tuple of tags; anything else has none. +fn graph_tags(value: &Bound<'_, PyAny>) -> PyResult> { + if !(value.is_instance_of::() || value.is_instance_of::()) { + return Ok(Vec::new()); + } + value + .try_iter()? + .map(|item| item?.str()?.extract()) + .collect() +} + +/// A resource's tags and model folder. +type ResourceTags = (Vec, Option); +/// Reads one resource's tags and model folder. +type DescribeResource = fn(&Bound<'_, PyAny>) -> PyResult; + +/// The graph rows of each resource in `resources`, with tags and folder from `describe`. +fn resource_rows( + resources: &Bound<'_, PyAny>, + describe: DescribeResource, +) -> PyResult> { + let mut rows: Vec = Vec::new(); + for resource in resources.try_iter()? { + let resource: Bound<'_, PyAny> = resource?; + let (tags, folder) = describe(&resource)?; + rows.push(( + graph_key(&resource.getattr("key")?)?, + graph_keys(&resource.getattr("deps")?)?, + tags, + folder, + )); + } + Ok(rows) +} + +/// A model's configured tags and folder. +fn model_tags(model: &Bound<'_, PyAny>) -> PyResult { + let tags: Bound<'_, PyAny> = model + .getattr("config")? + .getattr("values")? + .call_method1("get", ("tags",))?; + let folder: String = model + .getattr("relative_path")? + .getattr("parent")? + .str()? + .extract()?; + Ok((graph_tags(&tags)?, Some(folder))) +} + +/// A source has neither tags nor folder. +fn source_tags(_source: &Bound<'_, PyAny>) -> PyResult { + Ok((Vec::new(), None)) +} + +/// A seed's schema entry tags. +fn seed_tags(seed: &Bound<'_, PyAny>) -> PyResult { + let tags: Vec = seed + .getattr("schema_entry")? + .getattr("tags")? + .try_iter()? + .map(|tag| tag?.extract()) + .collect::>()?; + Ok((tags, None)) +} + +/// A function's tags. +fn function_tags(function: &Bound<'_, PyAny>) -> PyResult { + let tags: Vec = function + .getattr("tags")? + .try_iter()? + .map(|tag| tag?.extract()) + .collect::>()?; + Ok((tags, None)) +} + +/// `CompiledProject.lineage_graph`: lineage edges, tags, folders and names indexed natively. +pub(crate) fn project_lineage_graph(instance: &Bound<'_, PyAny>) -> PyResult> { + let mut rows: Vec = Vec::new(); + rows.extend(resource_rows(&instance.getattr("models")?, model_tags)?); + rows.extend(resource_rows(&instance.getattr("sources")?, source_tags)?); + rows.extend(resource_rows(&instance.getattr("seeds")?, seed_tags)?); + rows.extend(resource_rows( + &instance.getattr("functions")?, + function_tags, + )?); + project_graph(rows)?.into_py_any(instance.py()) +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/record_behaviours/mod.rs b/crates/sqlbuild-python/src/bindings/_helpers/record_behaviours/mod.rs index 482410a68c..b8853c66d4 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/record_behaviours/mod.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/record_behaviours/mod.rs @@ -1,5 +1,6 @@ pub(crate) mod auditing; pub(crate) mod call_arguments; +pub(crate) mod compiled; pub(crate) mod config; pub(crate) mod constants; pub(crate) mod cursors; @@ -8,6 +9,8 @@ pub(crate) mod durations; pub(crate) mod error_inits; pub(crate) mod freshness; pub(crate) mod lexical; +pub(crate) mod lineage_facts; +pub(crate) mod macros; pub(crate) mod migrations; pub(crate) mod planner; pub(crate) mod python_nodes; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/records/constants.rs b/crates/sqlbuild-python/src/bindings/_helpers/records/constants.rs index 02e4ffb586..b36842fba5 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/records/constants.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/records/constants.rs @@ -9,7 +9,7 @@ use crate::bindings::_helpers::freshness::observation::normalize_observation; use crate::bindings::_helpers::planner_records::constants::PLANNER_RECORDS; use crate::bindings::_helpers::record_behaviours::constants::{ BEHAVIOUR_CACHED_PROPERTIES, BEHAVIOUR_FINISHERS, BEHAVIOUR_METHODS, BEHAVIOUR_ORDERED, - BEHAVIOUR_PROPERTIES, + BEHAVIOUR_PROPERTIES, BEHAVIOUR_SEQUENCES, }; use crate::bindings::_helpers::records::models::{ ClassValue, EnumSpec, ErrorSpec, FieldDefault, FieldSpec, RecordSpec, @@ -173,6 +173,8 @@ pub(crate) const CACHED_PROPERTIES: &[&[(&str, &str, ClassProperty)]] = pub(crate) const CLASS_METHODS: &[&[(&str, &str, ClassMethod)]] = &[BEHAVIOUR_METHODS]; /// Records ordered by their compared fields, as `order=True` dataclasses are. pub(crate) const ORDERED_RECORDS: &[&[&str]] = &[BEHAVIOUR_ORDERED]; +/// Native records registered as `collections.abc.Sequence` virtual subclasses. +pub(crate) const SEQUENCE_RECORDS: &[&[&str]] = &[BEHAVIOUR_SEQUENCES]; /// `SourceFreshnessObservation` field positions. pub(crate) const DATA_VERSION_FIELD: usize = 2; pub(crate) const OBSERVED_AT_FIELD: usize = 4; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/records/registration.rs b/crates/sqlbuild-python/src/bindings/_helpers/records/registration.rs index c550070001..09501ed91e 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/records/registration.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/records/registration.rs @@ -7,8 +7,8 @@ use pyo3::types::{PyDict, PyDictMethods, PyString, PyTuple}; use crate::bindings::_helpers::records::constants::{ BUILTIN_PARENT_PREFIX, BUILTINS_MODULE, CACHED_PROPERTIES, CLASS_METHODS, CLASS_PROPERTIES, DATACLASS_FIELD_KIND, DATACLASS_FIELDS_ATTRIBUTE, DATACLASSES_MODULE, ENUM_MODULE, ENUMS, - ERROR_SPEC_ATTRIBUTE, ERRORS, EXCEPTION_PARENT, NATIVE_MODULE, RECORD_SPEC_ATTRIBUTE, STR_ENUM, - VALUE_ERROR_PARENT, + ERROR_SPEC_ATTRIBUTE, ERRORS, EXCEPTION_PARENT, NATIVE_MODULE, RECORD_SPEC_ATTRIBUTE, + SEQUENCE_RECORDS, STR_ENUM, VALUE_ERROR_PARENT, }; use crate::bindings::_helpers::records::error_classes::{ BoundErrorInitializer, ErrorInitializer, class_value, @@ -80,6 +80,10 @@ pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { let descriptor: Py = Py::new(py, NativeMethod { method: *method })?; module.getattr(*class)?.setattr(*name, descriptor)?; } + let sequence: Bound<'_, PyAny> = py.import("collections.abc")?.getattr("Sequence")?; + for class in SEQUENCE_RECORDS.iter().flat_map(|group| group.iter()) { + sequence.call_method1("register", (module.getattr(*class)?,))?; + } Ok(()) } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/refactor/offline_compile.rs b/crates/sqlbuild-python/src/bindings/_helpers/refactor/offline_compile.rs index 7d8128bb56..24d25ddda6 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/refactor/offline_compile.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/refactor/offline_compile.rs @@ -12,7 +12,7 @@ use crate::bindings::_helpers::compile_driver::python_calls::{call_with, importe const NATIVE_RECORDS_MODULE: &str = "sqlbuild._native"; const NATIVE_MODULE: &str = "sqlbuild._native"; -const COMPILE_MODELS: &str = "sqlbuild.compiler.compile.models"; +const COMPILE_MODELS: &str = "sqlbuild._native"; const COMPILE_TYPES: &str = "sqlbuild.compiler.compile.types"; const COMPILE_ERRORS: &str = "sqlbuild._native"; const PLANNER_ERRORS: &str = "sqlbuild._native"; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/render/authored_sql.rs b/crates/sqlbuild-python/src/bindings/_helpers/render/authored_sql.rs index 12d7560771..137c2a04a2 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/render/authored_sql.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/render/authored_sql.rs @@ -22,7 +22,7 @@ use crate::bindings::_helpers::sqltext::static_sql::{InterpolationSources, inter const NATIVE_RECORDS_MODULE: &str = "sqlbuild._native"; const NATIVE_MODULE: &str = "sqlbuild._native"; -const COMPILE_MODELS_MODULE: &str = "sqlbuild.compiler.compile.models"; +const COMPILE_MODELS_MODULE: &str = "sqlbuild._native"; const COMPILE_ERRORS_MODULE: &str = "sqlbuild._native"; const SCOPE_MODELS_MODULE: &str = "sqlbuild._native"; const FUNCTOOLS_MODULE: &str = "functools"; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/render/declaration_values.rs b/crates/sqlbuild-python/src/bindings/_helpers/render/declaration_values.rs index 696d223fa0..b2aca182b7 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/render/declaration_values.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/render/declaration_values.rs @@ -11,7 +11,7 @@ use crate::bindings::_helpers::scopes::scope_lookup::build_scope_lookup; const NATIVE_RECORDS_MODULE: &str = "sqlbuild._native"; -const COMPILE_MODELS_MODULE: &str = "sqlbuild.compiler.compile.models"; +const COMPILE_MODELS_MODULE: &str = "sqlbuild._native"; const COMPILE_ERRORS_MODULE: &str = "sqlbuild._native"; const SCOPE_MODELS_MODULE: &str = "sqlbuild._native"; const SCOPE_TYPES_MODULE: &str = "sqlbuild._native"; @@ -437,7 +437,7 @@ fn render_constant_declaration<'py>( /// One enum member value as a SQL scalar literal. #[pyfunction] #[pyo3(signature = (*, value))] -fn render_enum_member_value(value: &Bound<'_, PyAny>) -> PyResult { +pub(crate) fn render_enum_member_value(value: &Bound<'_, PyAny>) -> PyResult { if value.is_instance_of::() { return value.str()?.extract(); } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/render/macro_expansion_result.rs b/crates/sqlbuild-python/src/bindings/_helpers/render/macro_expansion_result.rs index 1bf37a92d5..230858dbbf 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/render/macro_expansion_result.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/render/macro_expansion_result.rs @@ -19,7 +19,7 @@ use crate::bindings::_helpers::sqltext::sql_references::extract_references; const NATIVE_RECORDS_MODULE: &str = "sqlbuild._native"; const NATIVE_MODULE: &str = "sqlbuild._native"; -const COMPILE_MODELS_MODULE: &str = "sqlbuild.compiler.compile.models"; +const COMPILE_MODELS_MODULE: &str = "sqlbuild._native"; const COMPILE_EXCEPTIONS_MODULE: &str = "sqlbuild._native"; const FRONTIER_EXCEPTIONS_MODULE: &str = "sqlbuild._native"; const SCOPES_MODELS_MODULE: &str = "sqlbuild._native"; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/rules/custom_hosts.rs b/crates/sqlbuild-python/src/bindings/_helpers/rules/custom_hosts.rs index 0ac110b053..fa99e84757 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/rules/custom_hosts.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/rules/custom_hosts.rs @@ -33,7 +33,7 @@ use crate::bindings::_helpers::rules::custom_host_observation::{ const NATIVE_MODULE: &str = "sqlbuild._native"; const RULES_CONSTANTS_MODULE: &str = "sqlbuild.rule_engine.constants"; const RULES_ERRORS_MODULE: &str = "sqlbuild._native"; -const COMPILE_MODELS_MODULE: &str = "sqlbuild.compiler.compile.models"; +const COMPILE_MODELS_MODULE: &str = "sqlbuild._native"; const HOST_INPUTS: [&str; 3] = ["target", "rules-cache", "host-inputs"]; /// The model fields hosts decode; every other model field is dropped from their payload. const PAYLOAD_MODEL_FIELDS: [&str; 18] = [ diff --git a/crates/sqlbuild-python/src/bindings/_helpers/rules/relation_keys.rs b/crates/sqlbuild-python/src/bindings/_helpers/rules/relation_keys.rs index ec95af0ea3..dce80ccc23 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/rules/relation_keys.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/rules/relation_keys.rs @@ -123,7 +123,7 @@ fn unique_key(values: &Bound<'_, PyDict>) -> PyResult>> { /// The key row of a seed, source or hand-built model from its config values and schema entry. #[pyfunction(name = "relation_key_row")] #[pyo3(signature = (kind, name, values, entry))] -fn entry_relation_key_row( +pub(crate) fn entry_relation_key_row( kind: &str, name: String, values: &Bound<'_, PyDict>, diff --git a/crates/sqlbuild-python/src/bindings/_helpers/scopes/compile_usages.rs b/crates/sqlbuild-python/src/bindings/_helpers/scopes/compile_usages.rs index 1100ddc023..ec33f64641 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/scopes/compile_usages.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/scopes/compile_usages.rs @@ -23,7 +23,7 @@ use crate::bindings::_helpers::compiled_project::model_loop_named::scope_path_me const SCOPE_ERRORS_MODULE: &str = "sqlbuild._native"; const SCOPE_MODELS_MODULE: &str = "sqlbuild._native"; const COLLECTOR_MODULE: &str = "sqlbuild._native"; -const COMPILE_MODELS_MODULE: &str = "sqlbuild.compiler.compile.models"; +const COMPILE_MODELS_MODULE: &str = "sqlbuild._native"; const INACCESSIBLE_DECLARATION_CODE: &str = "S006"; const SCHEMA_USAGE_KINDS: [&str; 2] = ["schema", "enum"]; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/scopes/scope_errors.rs b/crates/sqlbuild-python/src/bindings/_helpers/scopes/scope_errors.rs index 76dcf00193..56c42b1674 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/scopes/scope_errors.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/scopes/scope_errors.rs @@ -4,7 +4,7 @@ use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult, Python}; use pyo3::types::{PyDict, PyDictMethods, PyString, PyTuple}; const COLLECTOR_MODULE: &str = "sqlbuild._native"; -const COMPILE_MODELS_MODULE: &str = "sqlbuild.compiler.compile.models"; +const COMPILE_MODELS_MODULE: &str = "sqlbuild._native"; const ERROR_SEVERITY: &str = "error"; /// `report_scope_index_errors`: each error-severity finding of `index`, keyed for the collector. diff --git a/crates/sqlbuild-python/src/bindings/_helpers/semantic_checks/resource_sql.rs b/crates/sqlbuild-python/src/bindings/_helpers/semantic_checks/resource_sql.rs index a58cd6d948..974a648c5e 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/semantic_checks/resource_sql.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/semantic_checks/resource_sql.rs @@ -18,7 +18,7 @@ use crate::bindings::models::ProjectCatalog; use crate::bindings::types::CompilerDetach; const COMPILE_ERRORS_MODULE: &str = "sqlbuild._native"; -const COMPILE_MODELS_MODULE: &str = "sqlbuild.compiler.compile.models"; +const COMPILE_MODELS_MODULE: &str = "sqlbuild._native"; const CONTRACT_MODELS_MODULE: &str = "sqlbuild._native"; const SQL_ANALYSIS_CONFIG_KEY: &str = "sql_analysis"; const MODEL_HOOK_KEYS: [&str; 2] = ["pre_hooks", "post_hooks"]; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/semantic_checks/stage_objects.rs b/crates/sqlbuild-python/src/bindings/_helpers/semantic_checks/stage_objects.rs index 1f9aca6bf6..7c2a2137b4 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/semantic_checks/stage_objects.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/semantic_checks/stage_objects.rs @@ -6,7 +6,7 @@ use pyo3::types::{PyDict, PyDictMethods, PyString}; const COMPILE_TYPES_MODULE: &str = "sqlbuild.compiler.compile.types"; -const COMPILE_MODELS_MODULE: &str = "sqlbuild.compiler.compile.models"; +const COMPILE_MODELS_MODULE: &str = "sqlbuild._native"; const UNNEEDED_SQL_ANALYSIS_OPT_OUT_CODE: &str = "P009"; const SQL_ANALYSIS_CONFIG_KEY: &str = "sql_analysis"; const CONTRACT_MODELS_MODULE: &str = "sqlbuild._native"; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/semantic_shapes/source_bindings.rs b/crates/sqlbuild-python/src/bindings/_helpers/semantic_shapes/source_bindings.rs index db1e80a2bd..f05b78d6f4 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/semantic_shapes/source_bindings.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/semantic_shapes/source_bindings.rs @@ -24,7 +24,7 @@ const COMPILE_TYPES_MODULE: &str = "sqlbuild.compiler.compile.types"; const NATIVE_MODULE: &str = "sqlbuild._native"; const ANALYSIS_MODELS_MODULE: &str = "sqlbuild._native"; -const COMPILE_MODELS_MODULE: &str = "sqlbuild.compiler.compile.models"; +const COMPILE_MODELS_MODULE: &str = "sqlbuild._native"; const LINEAGE_TYPES_MODULE: &str = "sqlbuild.compiler.lineage.types"; const BINDING_UNKNOWN_TABLE_INTERNAL_CODE: &str = "B000"; const DATACLASSES_MODULE: &str = "dataclasses"; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/sql_test_glue/compiled_tests.rs b/crates/sqlbuild-python/src/bindings/_helpers/sql_test_glue/compiled_tests.rs index 56616a76da..fd84264b42 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/sql_test_glue/compiled_tests.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/sql_test_glue/compiled_tests.rs @@ -10,7 +10,7 @@ use crate::bindings::_helpers::sql_test_glue::assembly::{ AssemblyRequest, AssemblyRow, DiagnosticRow, FactsRow, assemble_compiled_sql_tests, }; -const COMPILE_MODELS_MODULE: &str = "sqlbuild.compiler.compile.models"; +const COMPILE_MODELS_MODULE: &str = "sqlbuild._native"; const COMPILE_TYPES_MODULE: &str = "sqlbuild.compiler.compile.types"; const GLUE_MODELS_MODULE: &str = "sqlbuild._native"; const SPEC_MODELS_MODULE: &str = "sqlbuild._native"; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/sqltext/sql_references.rs b/crates/sqlbuild-python/src/bindings/_helpers/sqltext/sql_references.rs index 70520985c6..c74782f322 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/sqltext/sql_references.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/sqltext/sql_references.rs @@ -18,7 +18,7 @@ use crate::bindings::_helpers::sqltext::lexical_syntax::LexicalSyntaxInput; const NATIVE_RECORDS_MODULE: &str = "sqlbuild._native"; -const COMPILE_MODELS_MODULE: &str = "sqlbuild.compiler.compile.models"; +const COMPILE_MODELS_MODULE: &str = "sqlbuild._native"; const COMPILE_TYPES_MODULE: &str = "sqlbuild.compiler.compile.types"; const COMPILE_ERRORS_MODULE: &str = "sqlbuild._native"; const REFERENCE_TYPES_MODULE: &str = "sqlbuild._native"; diff --git a/python-boundary.toml b/python-boundary.toml index c6de58e414..80fd15ec88 100644 --- a/python-boundary.toml +++ b/python-boundary.toml @@ -2,7 +2,7 @@ # migration work that moves to Rust. `remaining_lines` must equal the current count of that work: # a PR that moves code lowers it, and adding movable Python fails the check. # Native failures must surface as errors; `handled_native_errors` lists the reviewed exceptions. -remaining_lines = 1586 +remaining_lines = 791 [[stays]] path = "executor" diff --git a/scripts/compiler_differential/_helpers/coverage/analysis.py b/scripts/compiler_differential/_helpers/coverage/analysis.py index 0730f7136e..5116c308c5 100644 --- a/scripts/compiler_differential/_helpers/coverage/analysis.py +++ b/scripts/compiler_differential/_helpers/coverage/analysis.py @@ -26,8 +26,7 @@ ANALYSIS_UPSTREAM_CHAIN_HOPS, COMPILED_PROJECT_CAPTURE_TYPE, ) -from sqlbuild._native import STAGE_CAPTURE_OMITTED_ATTRIBUTES -from sqlbuild.compiler.compile.models import CompiledProject +from sqlbuild._native import STAGE_CAPTURE_OMITTED_ATTRIBUTES, CompiledProject _SET_OPERATION: re.Pattern[str] = re.compile(r"\b(?:UNION|INTERSECT|EXCEPT)\b", re.IGNORECASE) _LEADING_CTE: re.Pattern[str] = re.compile(r"^\s*WITH\s+\w+\s+AS\s*\(", re.IGNORECASE) diff --git a/src/sqlbuild/_native.pyi b/src/sqlbuild/_native.pyi index d4affd3247..f6a757f77a 100644 --- a/src/sqlbuild/_native.pyi +++ b/src/sqlbuild/_native.pyi @@ -47,25 +47,6 @@ from sqlbuild.compiler.auditing.types import ( AuditSeverity, ThresholdOperator, ) -from sqlbuild.compiler.compile.models import ( - CompiledAudit, - CompiledLineageSourceFact, - CompiledModel, - CompiledObjectKey, - CompiledProject, - CompiledRelationLocation, - CompiledSqlScenario, - CompiledSqlTest, - CompilerDiagnostic, - CompileSqlReference, - DeclarationResolutionContext, - InferredColumn, - LoadedMacro, - MacroContext, - ModelInputBuildContext, - StaticMacroExport, - _native, -) from sqlbuild.compiler.compile.types import ( AttachedAuditTargetKind, AuditInput, @@ -654,7 +635,7 @@ class CompileInputsView(Protocol): @property def declaration_scope(self) -> DeclarationScopeBuild | None: ... @property - def native_project(self) -> _native.NativeCompiledProject | None: ... + def native_project(self) -> NativeCompiledProject | None: ... class ModelInput(Protocol): """One model's compile input: a native owner view, or a hand-built `CompileModelInput`.""" @@ -11323,7 +11304,7 @@ class DeclarationScopeResolver(NativeRecord): project_dir: Path | None lookup: ScopeLookup projection: DeclarationRuntimeProjection - native_contexts: _native.NativeDeclarationContexts + native_contexts: NativeDeclarationContexts resource_specific: frozenset[ResourceIdentity] | None contexts_by_directory: dict[tuple[str, str], DeclarationResolutionContext] def __init__( @@ -11331,7 +11312,7 @@ class DeclarationScopeResolver(NativeRecord): project_dir: Path | None, lookup: ScopeLookup, projection: DeclarationRuntimeProjection, - native_contexts: _native.NativeDeclarationContexts, + native_contexts: NativeDeclarationContexts, resource_specific: frozenset[ResourceIdentity] | None = ..., contexts_by_directory: dict[tuple[str, str], DeclarationResolutionContext] = ..., ) -> None: ... @@ -11673,7 +11654,7 @@ class CompileProjectInputs(NativeRecord): external_sql_reference_resolver: ExternalSqlReferenceResolver | None scope_index: ScopeIndex declaration_scope: DeclarationScopeBuild | None - native_project: _native.NativeCompiledProject | None + native_project: NativeCompiledProject | None def __init__( self, project_config: ProjectConfig, @@ -11703,7 +11684,7 @@ class CompileProjectInputs(NativeRecord): external_sql_reference_resolver: ExternalSqlReferenceResolver | None = ..., scope_index: ScopeIndex = ..., declaration_scope: DeclarationScopeBuild | None = ..., - native_project: _native.NativeCompiledProject | None = ..., + native_project: NativeCompiledProject | None = ..., ) -> None: ... class CompiledSource(NativeRecord): @@ -12019,14 +12000,14 @@ class NativeModelConfigSession(NativeRecord): """The native config builder, validator, and inherited storage policies for one compile.""" path_defaults: dict[str, dict[str, object]] - builder: _native.NativeModelConfigBuilder - validator: _native.NativeModelValidator + builder: NativeModelConfigBuilder + validator: NativeModelValidator inherited_storage: dict[str | None, tuple[ResolvedTimeTravelRetention, ResolvedTableType]] def __init__( self, path_defaults: dict[str, dict[str, object]], - builder: _native.NativeModelConfigBuilder, - validator: _native.NativeModelValidator, + builder: NativeModelConfigBuilder, + validator: NativeModelValidator, inherited_storage: dict[str | None, tuple[ResolvedTimeTravelRetention, ResolvedTableType]], ) -> None: ... @@ -17439,11 +17420,11 @@ class ProjectGraph(NativeRecord): """Static compiled project graph held natively; dict views are built only when read.""" project: CompiledProjectView - native: _native.NativeProjectGraph + native: NativeProjectGraph def __init__( self, project: CompiledProjectView, - native: _native.NativeProjectGraph, + native: NativeProjectGraph, ) -> None: ... @property def lineage_views(self) -> LineageGraphViews: ... @@ -17478,3 +17459,645 @@ class CompilePipelineResult(NativeRecord): compile_seconds: float | None = ..., planning_seconds: float | None = ..., ) -> None: ... + +class CompilerDiagnostic(NativeRecord): + """One project diagnostic produced by compile-time checks.""" + + phase: DiagnosticPhase | str + severity: DiagnosticSeverity | str + code: str + message: str + resource_type: CompiledResourceType | str | None + resource_name: str | None + column_name: str | None + path: Path | None + line: int | None + column: int | None + location: SourceLocation | None + related_locations: tuple[RelatedLocation, ...] + help: str | None + notes: tuple[str, ...] + affected_rules: tuple[str, ...] + def __init__( + self, + phase: DiagnosticPhase | str, + severity: DiagnosticSeverity | str, + code: str, + message: str, + resource_type: CompiledResourceType | str | None = ..., + resource_name: str | None = ..., + column_name: str | None = ..., + path: Path | None = ..., + line: int | None = ..., + column: int | None = ..., + location: SourceLocation | None = ..., + related_locations: tuple[RelatedLocation, ...] = ..., + help: str | None = ..., + notes: tuple[str, ...] = ..., + affected_rules: tuple[str, ...] = ..., + ) -> None: ... + @property + def is_error(self) -> bool: ... + +class InferredColumn(NativeRecord): + """One output column inferred from query SQL via sql_analysis parsing.""" + + name: str + type: str | None + nullability: InferredNullability + def __init__( + self, + name: str, + type: str | None = ..., + nullability: InferredNullability = ..., + ) -> None: ... + +class LoadedMacro(NativeRecord): + """One loaded project macro available for compile-time SQL expansion.""" + + name: str + file_path: Path + relative_path: Path + raw_source: str + function: Callable[..., object] + dependencies: tuple[DeclarationIdentity, ...] + def __init__( + self, + name: str, + file_path: Path, + relative_path: Path, + raw_source: str, + function: Callable[..., object], + dependencies: tuple[DeclarationIdentity, ...] = ..., + ) -> None: ... + @property + def injects_context(self) -> bool: ... + +class StaticMacroExport(NativeRecord): + """Value-free AST inventory for one project-owned macro export.""" + + name: str + relative_path: Path + parameters: tuple[str, ...] + line: int + source_digest: str + dependencies: tuple[DeclarationIdentity, ...] + def __init__( + self, + name: str, + relative_path: Path, + parameters: tuple[str, ...], + line: int, + source_digest: str, + dependencies: tuple[DeclarationIdentity, ...] = ..., + ) -> None: ... + +class StaticMacroFault(NativeRecord): + """One value-free static macro inventory fault.""" + + relative_path: Path + message: str + def __init__( + self, + relative_path: Path, + message: str, + ) -> None: ... + +class StaticMacroInventory(NativeRecord): + """AST macro exports and faults collected without module execution.""" + + exports: tuple[StaticMacroExport, ...] + faults: tuple[StaticMacroFault, ...] + def __init__( + self, + exports: tuple[StaticMacroExport, ...] = ..., + faults: tuple[StaticMacroFault, ...] = ..., + ) -> None: ... + +class MacroContext(NativeRecord): + """Expose target settings and caller-visible declarations to a Python SQL macro.""" + + adapter_name: str + sql_analysis_enabled: bool + target_name: str | None + vars: dict[str, object] + constants: Mapping[str, object] + enums: Mapping[str, Mapping[str, str | int]] + _value_renderer: TypedSqlValueRenderer | None + _collection_rendering: CollectionRendering + _constant_declarations: Mapping[str, ConstantDeclaration] + _enforce_explicit_references: bool + def __init__( + self, + adapter_name: str, + sql_analysis_enabled: bool, + target_name: str | None, + vars: dict[str, object] = ..., + constants: Mapping[str, object] = ..., + enums: Mapping[str, Mapping[str, str | int]] = ..., + _value_renderer: TypedSqlValueRenderer | None = ..., + _collection_rendering: CollectionRendering = ..., + _constant_declarations: Mapping[str, ConstantDeclaration] = ..., + _enforce_explicit_references: bool = ..., + ) -> None: ... + def render_constant(self, name: str) -> str: ... + def render_enum_member(self, *, enum_name: str, member_name: str) -> str: ... + +class DeclarationResolutionContext(NativeRecord): + """Declarations visible during one authored SQL expansion.""" + + enums: dict[str, EnumDeclaration] + constants: dict[str, ConstantDeclaration] + inaccessible_enums: dict[str, DeclarationRecord] + inaccessible_constants: dict[str, DeclarationRecord] + enum_visibility: dict[str, tuple[VisibilityRecord, ...]] + constant_visibility: dict[str, tuple[VisibilityRecord, ...]] + macros: dict[str, LoadedMacro] + macro_records: dict[str, DeclarationRecord] + macro_visibility: dict[str, tuple[VisibilityRecord, ...]] + inaccessible_macros: dict[str, DeclarationRecord] + consumer: ResourceIdentity | DeclarationIdentity | None + def __init__( + self, + enums: dict[str, EnumDeclaration] = ..., + constants: dict[str, ConstantDeclaration] = ..., + inaccessible_enums: dict[str, DeclarationRecord] = ..., + inaccessible_constants: dict[str, DeclarationRecord] = ..., + enum_visibility: dict[str, tuple[VisibilityRecord, ...]] = ..., + constant_visibility: dict[str, tuple[VisibilityRecord, ...]] = ..., + macros: dict[str, LoadedMacro] = ..., + macro_records: dict[str, DeclarationRecord] = ..., + macro_visibility: dict[str, tuple[VisibilityRecord, ...]] = ..., + inaccessible_macros: dict[str, DeclarationRecord] = ..., + consumer: ResourceIdentity | DeclarationIdentity | None = ..., + ) -> None: ... + +class ModelInputBuildContext(NativeRecord): + """Run-constant config and macros for building model compile inputs.""" + + effective_vars: dict[str, object] + effective_settings: SettingsConfig + target_config: TargetConfig | None + effective_target_name: str | None + run_id: str + macro_context: MacroContext + loaded_macros: dict[str, LoadedMacro] + value_renderer: TypedSqlValueRenderer + collection_rendering: CollectionRendering + sql_lexical_syntax: SqlLexicalSyntax + public_enums: dict[str, EnumDeclaration] + public_constants: dict[str, ConstantDeclaration] + public_model_schemas: dict[str, ModelSchemaDeclaration] + declaration_resolver: DeclarationScopeResolver | None + def __init__( + self, + effective_vars: dict[str, object], + effective_settings: SettingsConfig, + target_config: TargetConfig | None, + effective_target_name: str | None, + run_id: str, + macro_context: MacroContext, + loaded_macros: dict[str, LoadedMacro], + value_renderer: TypedSqlValueRenderer, + collection_rendering: CollectionRendering, + sql_lexical_syntax: SqlLexicalSyntax, + public_enums: dict[str, EnumDeclaration] = ..., + public_constants: dict[str, ConstantDeclaration] = ..., + public_model_schemas: dict[str, ModelSchemaDeclaration] = ..., + declaration_resolver: DeclarationScopeResolver | None = ..., + ) -> None: ... + @property + def declaration_expansion(self) -> DeclarationExpansionContext: ... + +class CompileSqlReference(NativeRecord): + """One logical SQL reference discovered from compiled SQL text.""" + + ref_kind: SqlReferenceKind | str + ref_name: str + ref_package: str | None + call_argument_count: int | None + def __init__( + self, + ref_kind: SqlReferenceKind | str, + ref_name: str, + ref_package: str | None = ..., + call_argument_count: int | None = ..., + ) -> None: ... + +class CompiledLineageSourceFact(NativeRecord): + """Compact upstream column fact extracted during SQL analysis.""" + + resource_type: CompiledResourceType | str + resource_name: str + column_name: str + def __init__( + self, + resource_type: CompiledResourceType | str, + resource_name: str, + column_name: str, + ) -> None: ... + +class CompactLineageFacts(NativeRecord, Sequence[CompiledLineageColumnFact]): + """Indexed native lineage rows with lazy object projection.""" + + string_pool: tuple[str, ...] + rows: tuple[tuple[int, int, int, tuple[tuple[int, int, int], ...]], ...] + resource_name_indexes: dict[int, int] + resource_names: dict[int, str] + def __init__( + self, + string_pool: tuple[str, ...], + rows: tuple[tuple[int, int, int, tuple[tuple[int, int, int], ...]], ...], + resource_name_indexes: dict[int, int] = ..., + resource_names: dict[int, str] = ..., + ) -> None: ... + @property + def facts(self) -> tuple[CompiledLineageColumnFact, ...]: ... + def __len__(self) -> int: ... + @overload + def __getitem__(self, index: int) -> CompiledLineageColumnFact: ... + @overload + def __getitem__(self, index: slice) -> Sequence[CompiledLineageColumnFact]: ... + def __iter__(self) -> Iterator[CompiledLineageColumnFact]: ... + def __eq__(self, other: object) -> bool: ... + def resource_name(self, index: int) -> str: ... + +class CompiledObjectKey(NativeRecord): + """Stable logical identity for one compiled resource or external dependency.""" + + resource_type: CompiledResourceType | str + name: str + def __init__( + self, + resource_type: CompiledResourceType | str, + name: str, + ) -> None: ... + +class CompileModelInput(NativeRecord): + """One discovered model file with its attached schema metadata, if any.""" + + model_file: DiscoveredSqlModelFile + config: CompileModelConfig + query_sql: str + macro_source_sql: str + references: tuple[CompileSqlReference, ...] + schema_entry: SchemaModelEntry | None + schema_file: DiscoveredSchemaFile | None + sql_validation_enabled: bool + rejected_sql_analysis_opt_out: SourceLocation | None + enum_declarations: tuple[EnumDeclaration, ...] + constant_declarations: tuple[ConstantDeclaration, ...] + enum_columns: dict[str, EnumDeclaration] + macro_deps: tuple[str, ...] + macro_usages: tuple[UsageRecord, ...] + declaration_usages: tuple[UsageRecord, ...] + sql_expansion: CompiledSqlExpansion | None + lint_expansion: CompiledSqlExpansion | None + def __init__( + self, + model_file: DiscoveredSqlModelFile, + config: CompileModelConfig = ..., + query_sql: str = ..., + macro_source_sql: str = ..., + references: tuple[CompileSqlReference, ...] = ..., + schema_entry: SchemaModelEntry | None = ..., + schema_file: DiscoveredSchemaFile | None = ..., + sql_validation_enabled: bool = ..., + rejected_sql_analysis_opt_out: SourceLocation | None = ..., + enum_declarations: tuple[EnumDeclaration, ...] = ..., + constant_declarations: tuple[ConstantDeclaration, ...] = ..., + enum_columns: dict[str, EnumDeclaration] = ..., + macro_deps: tuple[str, ...] = ..., + macro_usages: tuple[UsageRecord, ...] = ..., + declaration_usages: tuple[UsageRecord, ...] = ..., + sql_expansion: CompiledSqlExpansion | None = ..., + lint_expansion: CompiledSqlExpansion | None = ..., + ) -> None: ... + @property + def reference_rows(self) -> list[tuple[str, str, str | None, int | None]]: ... + @property + def relation_key_row(self) -> RelationKeyRow: ... + @property + def attached_schema_name(self) -> str | None: ... + +class CompiledRelationLocation(NativeRecord): + """Logical and physical relation location resolved during compile.""" + + database: str | None + schema: str | None + name: str + qualified_name: str | None + logical_schema: str | None + logical_database: str | None + def __init__( + self, + database: str | None, + schema: str | None, + name: str, + qualified_name: str | None, + logical_schema: str | None = ..., + logical_database: str | None = ..., + ) -> None: ... + +class CompiledModel(NativeRecord): + """Planner-ready compiled model metadata.""" + + key: CompiledObjectKey + deps: tuple[CompiledObjectKey, ...] + name: str + relative_path: Path + query_sql: str + config: CompileModelConfig + destination: CompiledRelationLocation + references: tuple[CompileSqlReference, ...] + schema_entry: SchemaModelEntry | None + inferred_columns: tuple[InferredColumn, ...] | None + fast_lineage_columns: Sequence[CompiledLineageColumnFact] | None + fast_lineage_has_star: bool + fast_lineage_star_resolved: bool + authored_sql: str + authored_query_sql: str + output_column_locations: dict[str, SourceLocation] + extract_implicit_alias_columns: bool + macro_deps: tuple[str, ...] + enum_declarations: tuple[EnumDeclaration, ...] + constant_declarations: tuple[ConstantDeclaration, ...] + enum_columns: dict[str, EnumDeclaration] + binding_diagnostics: tuple[CompilerDiagnostic, ...] + binding_validated: bool + dynamic_column_contract: DynamicColumnContractProof | None + unchecked_output_columns: frozenset[str] + rejected_sql_analysis_opt_out: SourceLocation | None + def __init__( + self, + key: CompiledObjectKey, + deps: tuple[CompiledObjectKey, ...], + name: str, + relative_path: Path, + query_sql: str, + config: CompileModelConfig, + destination: CompiledRelationLocation, + references: tuple[CompileSqlReference, ...] = ..., + schema_entry: SchemaModelEntry | None = ..., + inferred_columns: tuple[InferredColumn, ...] | None = ..., + fast_lineage_columns: Sequence[CompiledLineageColumnFact] | None = ..., + fast_lineage_has_star: bool = ..., + fast_lineage_star_resolved: bool = ..., + authored_sql: str = ..., + authored_query_sql: str = ..., + output_column_locations: dict[str, SourceLocation] = ..., + extract_implicit_alias_columns: bool = ..., + macro_deps: tuple[str, ...] = ..., + enum_declarations: tuple[EnumDeclaration, ...] = ..., + constant_declarations: tuple[ConstantDeclaration, ...] = ..., + enum_columns: dict[str, EnumDeclaration] = ..., + binding_diagnostics: tuple[CompilerDiagnostic, ...] = ..., + binding_validated: bool = ..., + dynamic_column_contract: DynamicColumnContractProof | None = ..., + unchecked_output_columns: frozenset[str] = ..., + rejected_sql_analysis_opt_out: SourceLocation | None = ..., + ) -> None: ... + @property + def reference_rows(self) -> list[tuple[str, str, str | None, int | None]]: ... + @property + def relation_key_row(self) -> RelationKeyRow: ... + @property + def key_row(self) -> tuple[str, str]: ... + @property + def dep_rows(self) -> list[tuple[str, str]]: ... + @property + def destination_row(self) -> tuple[str | None, str | None, str]: ... + @property + def inferred_rows(self) -> list[tuple[str, str | None, str]] | None: ... + @property + def schema_column_rows(self) -> list[tuple[str, str | None]] | None: ... + @property + def has_dynamic_columns(self) -> bool: ... + @property + def output_column_count(self) -> int: ... + +class CompiledAudit(NativeRecord): + """Compiled audit metadata selected by scope dependencies.""" + + key: CompiledObjectKey + scope_deps: tuple[CompiledObjectKey, ...] + name: str + definition_name: str + audit_file: DiscoveredAuditFile + audit_block: DiscoveredAuditBlock + sql_body: str + evaluation_mode: AuditEvaluationMode + measurement_contract: MeasurementContract | None + thresholds: MeasurementThresholds | None + minimum_samples: int | None + measure_sql: str | None + evidence_sql: str | None + evidence_limit: int | None + references: tuple[CompileSqlReference, ...] + attached_target_kind: AttachedAuditTargetKind | None + attached_target_name: str | None + attached_column_name: str | None + severity: AuditSeverity | None + run_scope: str | None + always_run: bool + description: str | None + def __init__( + self, + key: CompiledObjectKey, + scope_deps: tuple[CompiledObjectKey, ...], + name: str, + definition_name: str, + audit_file: DiscoveredAuditFile, + audit_block: DiscoveredAuditBlock, + sql_body: str, + evaluation_mode: AuditEvaluationMode = ..., + measurement_contract: MeasurementContract | None = ..., + thresholds: MeasurementThresholds | None = ..., + minimum_samples: int | None = ..., + measure_sql: str | None = ..., + evidence_sql: str | None = ..., + evidence_limit: int | None = ..., + references: tuple[CompileSqlReference, ...] = ..., + attached_target_kind: AttachedAuditTargetKind | None = ..., + attached_target_name: str | None = ..., + attached_column_name: str | None = ..., + severity: AuditSeverity | None = ..., + run_scope: str | None = ..., + always_run: bool = ..., + description: str | None = ..., + ) -> None: ... + +class CompiledSqlScenario(NativeRecord): + """Compiled SQL-native scenario metadata selected by inferred graph targets.""" + + key: CompiledObjectKey + name: str + scenario_file: DiscoveredSqlScenarioFile + sql_body: str + authored_ctes: tuple[CompileSqlScenarioCte, ...] + expected_ctes: tuple[CompileSqlScenarioCte, ...] + assertion_ctes: tuple[CompileSqlScenarioCte, ...] + source_fixture_names: tuple[str, ...] + ref_fixture_names: tuple[str, ...] + seed_fixture_names: tuple[str, ...] + dbt_ref_fixture_names: tuple[str, ...] + expected_model_names: tuple[str, ...] + assertion_names: tuple[str, ...] + assertion_target_model_names: tuple[str, ...] + target_model_names: tuple[str, ...] + source_path: Path | None + ownership_root: Path | None + def __init__( + self, + key: CompiledObjectKey, + name: str, + scenario_file: DiscoveredSqlScenarioFile, + sql_body: str, + authored_ctes: tuple[CompileSqlScenarioCte, ...] = ..., + expected_ctes: tuple[CompileSqlScenarioCte, ...] = ..., + assertion_ctes: tuple[CompileSqlScenarioCte, ...] = ..., + source_fixture_names: tuple[str, ...] = ..., + ref_fixture_names: tuple[str, ...] = ..., + seed_fixture_names: tuple[str, ...] = ..., + dbt_ref_fixture_names: tuple[str, ...] = ..., + expected_model_names: tuple[str, ...] = ..., + assertion_names: tuple[str, ...] = ..., + assertion_target_model_names: tuple[str, ...] = ..., + target_model_names: tuple[str, ...] = ..., + source_path: Path | None = ..., + ownership_root: Path | None = ..., + ) -> None: ... + +class CompiledProject(NativeRecord): + """Planner-ready whole-project compile output.""" + + run_id: str + effective_target_name: str | None + effective_connection: dict[str, object] + effective_vars: dict[str, object] + binding_catalog: Any | None + effective_target_database: str | None + effective_target_schema: str | None + sql_analysis_dialect: str | None + sql_lexical_syntax: SqlLexicalSyntax + compile_cache_dir: Path | None + settings: SettingsConfig + scenario: ScenarioConfig + models: tuple[CompiledModelView, ...] + sources: tuple[CompiledSourceView, ...] + seeds: tuple[CompiledSeedView, ...] + functions: tuple[CompiledFunctionView, ...] + audits: tuple[CompiledAuditView, ...] + sql_tests: tuple[CompiledSqlTest, ...] + sql_scenarios: tuple[CompiledSqlScenario, ...] + loader_functions: tuple[DiscoveredLoaderFunction, ...] + hook_functions: tuple[DiscoveredHookFunction, ...] + enforce_explicit_references: bool + sql_hook_files: tuple[DiscoveredSqlHookFile, ...] + materialization_files: tuple[DiscoveredMaterializationFile, ...] + public_enums: dict[str, EnumDeclaration] + public_constants: dict[str, ConstantDeclaration] + loaded_macros: dict[str, LoadedMacro] + diagnostics: tuple[CompilerDiagnostic, ...] + unmatched_literal_sql_relations: tuple[LiteralSqlRelation, ...] + external_sql_reference_resolver: ExternalSqlReferenceResolver | None + scope_index: ScopeIndex + sql_expansions: dict[Path, CompiledSqlExpansion] + native_session: Any | None + native_project: NativeCompiledProject | None + def __init__( + self, + run_id: str, + effective_target_name: str | None, + effective_connection: dict[str, object], + effective_vars: dict[str, object], + binding_catalog: Any | None = ..., + effective_target_database: str | None = ..., + effective_target_schema: str | None = ..., + sql_analysis_dialect: str | None = ..., + sql_lexical_syntax: SqlLexicalSyntax = ..., + compile_cache_dir: Path | None = ..., + settings: SettingsConfig = ..., + scenario: ScenarioConfig = ..., + models: tuple[CompiledModelView, ...] = ..., + sources: tuple[CompiledSourceView, ...] = ..., + seeds: tuple[CompiledSeedView, ...] = ..., + functions: tuple[CompiledFunctionView, ...] = ..., + audits: tuple[CompiledAuditView, ...] = ..., + sql_tests: tuple[CompiledSqlTest, ...] = ..., + sql_scenarios: tuple[CompiledSqlScenario, ...] = ..., + loader_functions: tuple[DiscoveredLoaderFunction, ...] = ..., + hook_functions: tuple[DiscoveredHookFunction, ...] = ..., + enforce_explicit_references: bool = ..., + sql_hook_files: tuple[DiscoveredSqlHookFile, ...] = ..., + materialization_files: tuple[DiscoveredMaterializationFile, ...] = ..., + public_enums: dict[str, EnumDeclaration] = ..., + public_constants: dict[str, ConstantDeclaration] = ..., + loaded_macros: dict[str, LoadedMacro] = ..., + diagnostics: tuple[CompilerDiagnostic, ...] = ..., + unmatched_literal_sql_relations: tuple[LiteralSqlRelation, ...] = ..., + external_sql_reference_resolver: ExternalSqlReferenceResolver | None = ..., + scope_index: ScopeIndex = ..., + sql_expansions: dict[Path, CompiledSqlExpansion] = ..., + native_session: Any | None = ..., + native_project: NativeCompiledProject | None = ..., + ) -> None: ... + @property + def lineage_graph(self) -> NativeProjectGraph: ... + +class CompiledSqlTest(NativeRecord): + """Compiled SQL-native unit test metadata selected by expected targets.""" + + key: CompiledObjectKey + scope_deps: tuple[CompiledObjectKey, ...] + name: str + test_file: DiscoveredSqlTestFile + test_block: DiscoveredSqlTestBlock + sql_body: str + mode: SqlTestMode + payload: CompiledModelSqlTestPayload | CompiledDirectLogicSqlTestPayload + source_path: Path | None + ownership_root: Path | None + block_index: int | None + explicit_name: str | None + parent_name: str | None + case_name: str | None + case_index: int | None + case_fingerprint: str | None + parameter_schema: tuple[SqlTestParameterDeclaration, ...] + parameter_values: tuple[tuple[str, SqlValue], ...] + expected_model_names: tuple[str, ...] + assertion_names: tuple[str, ...] + assertion_target_model_names: tuple[str, ...] + read_helper_names: tuple[str, ...] + reference_target_model_names: tuple[str, ...] + target_model_names: tuple[str, ...] + tested_resources: tuple[CompiledSqlTestResource, ...] + def __init__( + self, + key: CompiledObjectKey, + scope_deps: tuple[CompiledObjectKey, ...], + name: str, + test_file: DiscoveredSqlTestFile, + test_block: DiscoveredSqlTestBlock, + sql_body: str, + mode: SqlTestMode = ..., + payload: CompiledModelSqlTestPayload | CompiledDirectLogicSqlTestPayload = ..., + source_path: Path | None = ..., + ownership_root: Path | None = ..., + block_index: int | None = ..., + explicit_name: str | None = ..., + parent_name: str | None = ..., + case_name: str | None = ..., + case_index: int | None = ..., + case_fingerprint: str | None = ..., + parameter_schema: tuple[SqlTestParameterDeclaration, ...] = ..., + parameter_values: tuple[tuple[str, SqlValue], ...] = ..., + expected_model_names: tuple[str, ...] = ..., + assertion_names: tuple[str, ...] = ..., + assertion_target_model_names: tuple[str, ...] = ..., + read_helper_names: tuple[str, ...] = ..., + reference_target_model_names: tuple[str, ...] = ..., + target_model_names: tuple[str, ...] = ..., + tested_resources: tuple[CompiledSqlTestResource, ...] = ..., + ) -> None: ... diff --git a/src/sqlbuild/compiler/compile/_helpers/render/user_macros.py b/src/sqlbuild/compiler/compile/_helpers/render/user_macros.py index 29a17d9887..6c84590d94 100644 --- a/src/sqlbuild/compiler/compile/_helpers/render/user_macros.py +++ b/src/sqlbuild/compiler/compile/_helpers/render/user_macros.py @@ -20,21 +20,19 @@ DeclarationIdentity, DeclarationKind, DeclarationRecord, + DeclarationResolutionContext, DiscoveredMacroFile, EnumDeclaration, - SqlValue, - SqlValueKind, - validate_resource_identity, -) -from sqlbuild.compiler.compile.exceptions import MacroDeclarationLookupError -from sqlbuild.compiler.compile.models import ( - DeclarationResolutionContext, LoadedMacro, MacroContext, + SqlValue, + SqlValueKind, StaticMacroExport, StaticMacroFault, StaticMacroInventory, + validate_resource_identity, ) +from sqlbuild.compiler.compile.exceptions import MacroDeclarationLookupError _OBJECT_ENTRY_LENGTH: int = 2 _MACRO_MODULE_LOAD_LOCK: threading.RLock = threading.RLock() diff --git a/src/sqlbuild/compiler/compile/constants.py b/src/sqlbuild/compiler/compile/constants.py index 7ab946ed48..cd8a7ce9ad 100644 --- a/src/sqlbuild/compiler/compile/constants.py +++ b/src/sqlbuild/compiler/compile/constants.py @@ -2,12 +2,4 @@ from __future__ import annotations -from sqlbuild.compiler.compile.types import SqlTestMode - MIGRATE_FROM_CONFIG_KEY: str = "migrate_from" - - -MACRO_CONTEXT_PARAMETER_NAME: str = "ctx" - - -DEFAULT_SQL_TEST_MODE: SqlTestMode = SqlTestMode.MODEL diff --git a/src/sqlbuild/compiler/compile/models.py b/src/sqlbuild/compiler/compile/models.py deleted file mode 100644 index 0aef4bed50..0000000000 --- a/src/sqlbuild/compiler/compile/models.py +++ /dev/null @@ -1,787 +0,0 @@ -"""Structured runtime models for project compilation.""" - -from __future__ import annotations - -import inspect -from collections.abc import Callable, Iterator, Mapping, Sequence -from dataclasses import dataclass, field -from functools import cached_property -from pathlib import Path -from typing import Any, overload - -import sqlbuild._native as _native -from sqlbuild._native import ( - CollectionRendering, - ColumnLineageConfidence, - ColumnTransformKind, - CompiledDirectLogicSqlTestPayload, - CompiledLineageColumnFact, - CompiledModelSqlTestPayload, - CompiledSqlExpansion, - CompiledSqlTestResource, - CompileInputError, - CompileModelConfig, - CompileSqlScenarioCte, - ConstantDeclaration, - DeclarationExpansionContext, - DeclarationIdentity, - DeclarationRecord, - DeclarationScopeResolver, - DiagnosticPhase, - DiagnosticSeverity, - DiscoveredAuditBlock, - DiscoveredAuditFile, - DiscoveredHookFunction, - DiscoveredLoaderFunction, - DiscoveredMaterializationFile, - DiscoveredSchemaFile, - DiscoveredSqlHookFile, - DiscoveredSqlModelFile, - DiscoveredSqlScenarioFile, - DiscoveredSqlTestBlock, - DiscoveredSqlTestFile, - DynamicColumnContractProof, - EnumDeclaration, - LiteralSqlRelation, - MeasurementContract, - MeasurementThresholds, - ModelSchemaDeclaration, - RelatedLocation, - ResourceIdentity, - ScenarioConfig, - SchemaModelEntry, - ScopeIndex, - SettingsConfig, - SourceLocation, - SqlLexicalSyntax, - SqlReferenceKind, - SqlTestParameterDeclaration, - SqlValue, - TargetConfig, - UsageRecord, - VisibilityRecord, -) -from sqlbuild.compiler.auditing.types import AuditEvaluationMode, AuditSeverity -from sqlbuild.compiler.compile.constants import DEFAULT_SQL_TEST_MODE, MACRO_CONTEXT_PARAMETER_NAME -from sqlbuild.compiler.compile.types import ( - AttachedAuditTargetKind, - CompiledAuditView, - CompiledFunctionView, - CompiledModelView, - CompiledResourceType, - CompiledSeedView, - CompiledSourceView, - ExternalSqlReferenceResolver, - RelationKeyRow, - SqlTestMode, - TypedSqlValueRenderer, -) -from sqlbuild.compiler.lineage.types import InferredNullability - - -@dataclass(frozen=True) -class CompilerDiagnostic: - """One project diagnostic produced by compile-time checks.""" - - phase: DiagnosticPhase | str - severity: DiagnosticSeverity | str - code: str - message: str - resource_type: CompiledResourceType | str | None = None - resource_name: str | None = None - column_name: str | None = None - path: Path | None = None - line: int | None = None - column: int | None = None - location: SourceLocation | None = None - related_locations: tuple[RelatedLocation, ...] = field(default_factory=tuple) - help: str | None = None - notes: tuple[str, ...] = () - affected_rules: tuple[str, ...] = () - - def __post_init__(self) -> None: - object.__setattr__(self, "phase", DiagnosticPhase(self.phase)) - object.__setattr__(self, "severity", DiagnosticSeverity(self.severity)) - if self.resource_type is not None: - object.__setattr__( - self, - "resource_type", - CompiledResourceType(self.resource_type), - ) - if ( - self.location is None - and self.path is not None - and self.line is not None - and self.column is not None - ): - object.__setattr__( - self, - "location", - SourceLocation(path=self.path, line=self.line, column=self.column), - ) - if self.location is not None: - object.__setattr__(self, "path", self.location.path) - object.__setattr__(self, "line", self.location.line) - object.__setattr__(self, "column", self.location.column) - - @property - def is_error(self) -> bool: - """Return whether this diagnostic should fail the command.""" - - return self.severity == DiagnosticSeverity.ERROR - - -@dataclass(frozen=True) -class InferredColumn: - """One output column inferred from query SQL via sql_analysis parsing.""" - - name: str - type: str | None = None - nullability: InferredNullability = InferredNullability.UNKNOWN - - -@dataclass(frozen=True) -class LoadedMacro: - """One loaded project macro available for compile-time SQL expansion.""" - - name: str - file_path: Path - relative_path: Path - raw_source: str - function: Callable[..., object] - dependencies: tuple[DeclarationIdentity, ...] = field(default_factory=tuple) - - @cached_property - def injects_context(self) -> bool: - """Whether the first positional parameter receives the injected macro context.""" - - parameters: tuple[inspect.Parameter, ...] = tuple( - inspect.signature(self.function).parameters.values() - ) - return bool( - parameters - and parameters[0].kind - in {inspect.Parameter.POSITIONAL_ONLY, inspect.Parameter.POSITIONAL_OR_KEYWORD} - and parameters[0].name == MACRO_CONTEXT_PARAMETER_NAME - ) - - -@dataclass(frozen=True) -class StaticMacroExport: - """Value-free AST inventory for one project-owned macro export.""" - - name: str - relative_path: Path - parameters: tuple[str, ...] - line: int - source_digest: str - dependencies: tuple[DeclarationIdentity, ...] = field(default_factory=tuple) - - -@dataclass(frozen=True) -class StaticMacroFault: - """One value-free static macro inventory fault.""" - - relative_path: Path - message: str - - -@dataclass(frozen=True) -class StaticMacroInventory: - """AST macro exports and faults collected without module execution.""" - - exports: tuple[StaticMacroExport, ...] = field(default_factory=tuple) - faults: tuple[StaticMacroFault, ...] = field(default_factory=tuple) - - -@dataclass(frozen=True) -class MacroContext: - """Expose target settings and caller-visible declarations to a Python SQL macro.""" - - adapter_name: str - sql_analysis_enabled: bool - target_name: str | None - vars: dict[str, object] = field(default_factory=dict) - constants: Mapping[str, object] = field(default_factory=dict) - enums: Mapping[str, Mapping[str, str | int]] = field(default_factory=dict) - _value_renderer: TypedSqlValueRenderer | None = field(default=None, repr=False, compare=False) - _collection_rendering: CollectionRendering = field( - default=CollectionRendering.VALUE_LIST, repr=False, compare=False - ) - _constant_declarations: Mapping[str, ConstantDeclaration] = field( - default_factory=dict, repr=False, compare=False - ) - _enforce_explicit_references: bool = field(default=True, repr=False, compare=False) - - def render_constant(self, name: str) -> str: - """Render one visible constant as an adapter-safe SQL value.""" - - if self._value_renderer is None: - raise CompileInputError( - "Constant SQL rendering is available only during a project macro invocation" - ) - _ = self.constants[name] - declaration: ConstantDeclaration | None = self._constant_declarations.get(name) - if declaration is None: - raise CompileInputError( - "Constant SQL rendering requires a declaration resolved for this macro invocation" - ) - from sqlbuild._native import render_constant_declaration - - return render_constant_declaration( - declaration=declaration, - value_renderer=self._value_renderer, - collection_rendering=self._collection_rendering, - ) - - def render_enum_member(self, *, enum_name: str, member_name: str) -> str: - """Render one visible enum member as a SQL scalar literal.""" - - value: str | int = self.enums[enum_name][member_name] - from sqlbuild._native import render_enum_member_value - - return render_enum_member_value(value=value) - - -@dataclass(frozen=True) -class DeclarationResolutionContext: - """Declarations visible during one authored SQL expansion.""" - - enums: dict[str, EnumDeclaration] = field(default_factory=dict) - constants: dict[str, ConstantDeclaration] = field(default_factory=dict) - inaccessible_enums: dict[str, DeclarationRecord] = field(default_factory=dict) - inaccessible_constants: dict[str, DeclarationRecord] = field(default_factory=dict) - enum_visibility: dict[str, tuple[VisibilityRecord, ...]] = field(default_factory=dict) - constant_visibility: dict[str, tuple[VisibilityRecord, ...]] = field(default_factory=dict) - macros: dict[str, LoadedMacro] = field(default_factory=dict) - macro_records: dict[str, DeclarationRecord] = field(default_factory=dict) - macro_visibility: dict[str, tuple[VisibilityRecord, ...]] = field(default_factory=dict) - inaccessible_macros: dict[str, DeclarationRecord] = field(default_factory=dict) - consumer: ResourceIdentity | DeclarationIdentity | None = None - - -@dataclass(frozen=True) -class ModelInputBuildContext: - """Run-constant config and macros for building model compile inputs.""" - - effective_vars: dict[str, object] - effective_settings: SettingsConfig - target_config: TargetConfig | None - effective_target_name: str | None - run_id: str - macro_context: MacroContext - loaded_macros: dict[str, LoadedMacro] - value_renderer: TypedSqlValueRenderer - collection_rendering: CollectionRendering - sql_lexical_syntax: SqlLexicalSyntax - public_enums: dict[str, EnumDeclaration] = field(default_factory=dict) - public_constants: dict[str, ConstantDeclaration] = field(default_factory=dict) - public_model_schemas: dict[str, ModelSchemaDeclaration] = field(default_factory=dict) - declaration_resolver: DeclarationScopeResolver | None = None - - @property - def declaration_expansion(self) -> DeclarationExpansionContext: - """Build the public declaration expansion context for non-model resources.""" - - return DeclarationExpansionContext( - declarations=DeclarationResolutionContext( - enums=self.public_enums, - constants=self.public_constants, - ), - value_renderer=self.value_renderer, - collection_rendering=self.collection_rendering, - resolver=self.declaration_resolver, - ) - - -@dataclass(frozen=True) -class CompileSqlReference: - """One logical SQL reference discovered from compiled SQL text.""" - - ref_kind: SqlReferenceKind | str - ref_name: str - ref_package: str | None = None - call_argument_count: int | None = None - - def __post_init__(self) -> None: - object.__setattr__(self, "ref_kind", SqlReferenceKind(self.ref_kind)) - - -@dataclass(frozen=True) -class CompiledLineageSourceFact: - """Compact upstream column fact extracted during SQL analysis.""" - - resource_type: CompiledResourceType | str - resource_name: str - column_name: str - - def __post_init__(self) -> None: - object.__setattr__(self, "resource_type", CompiledResourceType(self.resource_type)) - - -@dataclass(frozen=True, eq=False) -class CompactLineageFacts(Sequence[CompiledLineageColumnFact]): - """Indexed native lineage rows with lazy object projection.""" - - string_pool: tuple[str, ...] - rows: tuple[ - tuple[int, int, int, tuple[tuple[int, int, int], ...]], - ..., - ] - resource_name_indexes: dict[int, int] = field(default_factory=dict) - resource_names: dict[int, str] = field(default_factory=dict) - _cache: dict[int, CompiledLineageColumnFact] = field( - default_factory=dict, - init=False, - repr=False, - compare=False, - ) - - def __len__(self) -> int: - return len(self.rows) - - @overload - def __getitem__(self, index: int) -> CompiledLineageColumnFact: ... - - @overload - def __getitem__(self, index: slice) -> Sequence[CompiledLineageColumnFact]: ... - - def __getitem__( - self, index: int | slice - ) -> CompiledLineageColumnFact | Sequence[CompiledLineageColumnFact]: - if isinstance(index, slice): - return tuple(self[item_index] for item_index in range(*index.indices(len(self)))) - normalized_index: int = index if index >= 0 else len(self) + index - if normalized_index < 0 or normalized_index >= len(self): - raise IndexError(index) - cached: CompiledLineageColumnFact | None = self._cache.get(normalized_index) - if cached is not None: - return cached - name_index, transform_code, confidence_code, sources = self.rows[normalized_index] - fact: CompiledLineageColumnFact = CompiledLineageColumnFact( - output_column=self.string_pool[name_index], - upstream_columns=tuple( - CompiledLineageSourceFact( - resource_type=self.string_pool[source[0]], - resource_name=self.resource_name(source[1]), - column_name=self.string_pool[source[2]], - ) - for source in sources - ), - transform_kind=self.transform_kind(transform_code), - confidence=self.confidence(confidence_code), - ) - self._cache[normalized_index] = fact - return fact - - def __iter__(self) -> Iterator[CompiledLineageColumnFact]: - return (self[index] for index in range(len(self))) - - def __eq__(self, other: object) -> bool: - if not isinstance(other, Sequence): - return NotImplemented - return tuple(self) == tuple(other) - - def resource_name(self, index: int) -> str: - """Resolve a canonical native relation index to this model's resource name.""" - - name: str | None = self.resource_names.get(index) - if name is not None: - return name - return self.string_pool[self.resource_name_indexes.get(index, index)] - - @staticmethod - def transform_kind(code: int) -> ColumnTransformKind: - return ( - ColumnTransformKind.DIRECT, - ColumnTransformKind.CAST, - ColumnTransformKind.EXPRESSION, - ColumnTransformKind.AGGREGATION, - ColumnTransformKind.STAR, - ColumnTransformKind.CONSTANT, - )[code] - - @staticmethod - def confidence(code: int) -> ColumnLineageConfidence: - return ( - ColumnLineageConfidence.UNKNOWN, - ColumnLineageConfidence.HIGH, - ColumnLineageConfidence.MEDIUM, - )[code] - - -@dataclass(frozen=True) -class CompiledObjectKey: - """Stable logical identity for one compiled resource or external dependency.""" - - resource_type: CompiledResourceType | str - name: str - - def __post_init__(self) -> None: - object.__setattr__(self, "resource_type", CompiledResourceType(self.resource_type)) - - -@dataclass(frozen=True) -class CompileModelInput: - """One discovered model file with its attached schema metadata, if any.""" - - model_file: DiscoveredSqlModelFile - config: CompileModelConfig = field(default_factory=CompileModelConfig) - query_sql: str = "" - macro_source_sql: str = "" - references: tuple[CompileSqlReference, ...] = field(default_factory=tuple) - schema_entry: SchemaModelEntry | None = None - schema_file: DiscoveredSchemaFile | None = None - sql_validation_enabled: bool = False - rejected_sql_analysis_opt_out: SourceLocation | None = None - enum_declarations: tuple[EnumDeclaration, ...] = field(default_factory=tuple) - constant_declarations: tuple[ConstantDeclaration, ...] = field(default_factory=tuple) - enum_columns: dict[str, EnumDeclaration] = field(default_factory=dict) - macro_deps: tuple[str, ...] = field(default_factory=tuple) - macro_usages: tuple[UsageRecord, ...] = field(default_factory=tuple) - declaration_usages: tuple[UsageRecord, ...] = field(default_factory=tuple) - - @property - def reference_rows(self) -> list[tuple[str, str, str | None, int | None]]: - """Each reference as `(kind, name, package, call argument count)`.""" - - return [ - (str(ref.ref_kind), ref.ref_name, ref.ref_package, ref.call_argument_count) - for ref in self.references - ] - - @property - def relation_key_row(self) -> RelationKeyRow: - """The model's declared key facts, for deriving its unique keys.""" - - return _native.relation_key_row( - "ref", self.model_file.file_path.stem, self.config.values, self.schema_entry - ) - - @property - def attached_schema_name(self) -> str | None: - """The name of the schema entry attached to the model, or `None`.""" - - return self.schema_entry.name if self.schema_entry is not None else None - - sql_expansion: CompiledSqlExpansion | None = field(default=None, compare=False, repr=False) - lint_expansion: CompiledSqlExpansion | None = field(default=None, compare=False, repr=False) - """The compile's whole expansion, variables included, which early SQL lint reuses.""" - - -@dataclass(frozen=True) -class CompiledRelationLocation: - """Logical and physical relation location resolved during compile.""" - - database: str | None - schema: str | None - name: str - qualified_name: str | None - logical_schema: str | None = None - logical_database: str | None = None - - -@dataclass(frozen=True) -class CompiledModel: - """Planner-ready compiled model metadata.""" - - key: CompiledObjectKey - deps: tuple[CompiledObjectKey, ...] - name: str - relative_path: Path - query_sql: str - config: CompileModelConfig - destination: CompiledRelationLocation - references: tuple[CompileSqlReference, ...] = field(default_factory=tuple) - schema_entry: SchemaModelEntry | None = None - inferred_columns: tuple[InferredColumn, ...] | None = None - fast_lineage_columns: Sequence[CompiledLineageColumnFact] | None = None - fast_lineage_has_star: bool = False - fast_lineage_star_resolved: bool = False - authored_sql: str = "" - authored_query_sql: str = "" - output_column_locations: dict[str, SourceLocation] = field(default_factory=dict) - extract_implicit_alias_columns: bool = True - macro_deps: tuple[str, ...] = field(default_factory=tuple) - enum_declarations: tuple[EnumDeclaration, ...] = field(default_factory=tuple) - constant_declarations: tuple[ConstantDeclaration, ...] = field(default_factory=tuple) - enum_columns: dict[str, EnumDeclaration] = field(default_factory=dict) - binding_diagnostics: tuple[CompilerDiagnostic, ...] = field(default_factory=tuple) - binding_validated: bool = False - dynamic_column_contract: DynamicColumnContractProof | None = None - unchecked_output_columns: frozenset[str] = frozenset() - rejected_sql_analysis_opt_out: SourceLocation | None = None - - @property - def reference_rows(self) -> list[tuple[str, str, str | None, int | None]]: - """Each reference as `(kind, name, package, call argument count)`.""" - - return [ - (str(ref.ref_kind), ref.ref_name, ref.ref_package, ref.call_argument_count) - for ref in self.references - ] - - @property - def relation_key_row(self) -> RelationKeyRow: - """The model's declared key facts, for deriving its unique keys.""" - - return _native.relation_key_row("ref", self.name, self.config.values, self.schema_entry) - - @property - def key_row(self) -> tuple[str, str]: - """`(resource type, name)` of the model's key.""" - - return (str(self.key.resource_type), self.key.name) - - @property - def dep_rows(self) -> list[tuple[str, str]]: - """`(resource type, name)` of each dependency.""" - - return [(str(dep.resource_type), dep.name) for dep in self.deps] - - @property - def destination_row(self) -> tuple[str | None, str | None, str]: - """`(database, schema, name)` of the model's destination.""" - - return (self.destination.database, self.destination.schema, self.destination.name) - - @property - def inferred_rows(self) -> list[tuple[str, str | None, str]] | None: - """Each inferred column as `(name, type, nullability)`, or `None`.""" - - if self.inferred_columns is None: - return None - return [ - (column.name, column.type, str(column.nullability)) for column in self.inferred_columns - ] - - @property - def schema_column_rows(self) -> list[tuple[str, str | None]] | None: - """Each declared column of the attached schema as `(name, type)`, or `None`.""" - - if self.schema_entry is None: - return None - return [(column.name, column.type) for column in self.schema_entry.columns] - - @property - def has_dynamic_columns(self) -> bool: - """Whether the attached schema declares dynamic columns.""" - - return self.schema_entry is not None and bool(self.schema_entry.dynamic_columns) - - @property - def output_column_count(self) -> int: - """Inferred columns, else the attached schema's declared columns, else zero.""" - - if self.inferred_columns is not None: - return len(self.inferred_columns) - return len(self.schema_entry.columns) if self.schema_entry is not None else 0 - - -@dataclass(frozen=True) -class CompiledAudit: - """Compiled audit metadata selected by scope dependencies.""" - - key: CompiledObjectKey - scope_deps: tuple[CompiledObjectKey, ...] - name: str - definition_name: str - audit_file: DiscoveredAuditFile - audit_block: DiscoveredAuditBlock - sql_body: str - evaluation_mode: AuditEvaluationMode = AuditEvaluationMode.VIOLATIONS - measurement_contract: MeasurementContract | None = None - thresholds: MeasurementThresholds | None = None - minimum_samples: int | None = None - measure_sql: str | None = None - evidence_sql: str | None = None - evidence_limit: int | None = None - references: tuple[CompileSqlReference, ...] = field(default_factory=tuple) - attached_target_kind: AttachedAuditTargetKind | None = None - attached_target_name: str | None = None - attached_column_name: str | None = None - severity: AuditSeverity | None = None - run_scope: str | None = None - always_run: bool = False - description: str | None = None - - -@dataclass(frozen=True) -class CompiledSqlScenario: - """Compiled SQL-native scenario metadata selected by inferred graph targets.""" - - key: CompiledObjectKey - name: str - scenario_file: DiscoveredSqlScenarioFile - sql_body: str - authored_ctes: tuple[CompileSqlScenarioCte, ...] = field(default_factory=tuple) - expected_ctes: tuple[CompileSqlScenarioCte, ...] = field(default_factory=tuple) - assertion_ctes: tuple[CompileSqlScenarioCte, ...] = field(default_factory=tuple) - source_fixture_names: tuple[str, ...] = field(default_factory=tuple) - ref_fixture_names: tuple[str, ...] = field(default_factory=tuple) - seed_fixture_names: tuple[str, ...] = field(default_factory=tuple) - dbt_ref_fixture_names: tuple[str, ...] = field(default_factory=tuple) - expected_model_names: tuple[str, ...] = field(default_factory=tuple) - assertion_names: tuple[str, ...] = field(default_factory=tuple) - assertion_target_model_names: tuple[str, ...] = field(default_factory=tuple) - target_model_names: tuple[str, ...] = field(default_factory=tuple) - source_path: Path | None = None - ownership_root: Path | None = None - - def __post_init__(self) -> None: - if self.source_path is None: - object.__setattr__(self, "source_path", self.scenario_file.relative_path) - if self.ownership_root is None: - object.__setattr__(self, "ownership_root", self.scenario_file.ownership_root) - if not self.target_model_names: - object.__setattr__( - self, - "target_model_names", - tuple( - dict.fromkeys((*self.expected_model_names, *self.assertion_target_model_names)) - ), - ) - - -@dataclass(frozen=True) -class CompiledProject: - """Planner-ready whole-project compile output.""" - - run_id: str - effective_target_name: str | None - effective_connection: dict[str, object] - effective_vars: dict[str, object] - binding_catalog: Any | None = field(default=None, repr=False, compare=False) - effective_target_database: str | None = None - effective_target_schema: str | None = None - sql_analysis_dialect: str | None = None - sql_lexical_syntax: SqlLexicalSyntax = field(default_factory=SqlLexicalSyntax) - compile_cache_dir: Path | None = None - settings: SettingsConfig = field(default_factory=SettingsConfig) - scenario: ScenarioConfig = field(default_factory=ScenarioConfig) - models: tuple[CompiledModelView, ...] = field(default_factory=tuple) - sources: tuple[CompiledSourceView, ...] = field(default_factory=tuple) - seeds: tuple[CompiledSeedView, ...] = field(default_factory=tuple) - functions: tuple[CompiledFunctionView, ...] = field(default_factory=tuple) - audits: tuple[CompiledAuditView, ...] = field(default_factory=tuple) - sql_tests: tuple[CompiledSqlTest, ...] = field(default_factory=tuple) - sql_scenarios: tuple[CompiledSqlScenario, ...] = field(default_factory=tuple) - loader_functions: tuple[DiscoveredLoaderFunction, ...] = field(default_factory=tuple) - hook_functions: tuple[DiscoveredHookFunction, ...] = field(default_factory=tuple) - enforce_explicit_references: bool = True - sql_hook_files: tuple[DiscoveredSqlHookFile, ...] = field(default_factory=tuple) - materialization_files: tuple[DiscoveredMaterializationFile, ...] = field(default_factory=tuple) - public_enums: dict[str, EnumDeclaration] = field(default_factory=dict) - public_constants: dict[str, ConstantDeclaration] = field(default_factory=dict) - loaded_macros: dict[str, LoadedMacro] = field(default_factory=dict) - diagnostics: tuple[CompilerDiagnostic, ...] = field(default_factory=tuple) - unmatched_literal_sql_relations: tuple[LiteralSqlRelation, ...] = field(default_factory=tuple) - external_sql_reference_resolver: ExternalSqlReferenceResolver | None = None - scope_index: ScopeIndex = field(default_factory=ScopeIndex) - sql_expansions: dict[Path, CompiledSqlExpansion] = field( - default_factory=dict, compare=False, repr=False - ) - native_session: Any | None = field(default=None, compare=False, repr=False) - native_project: _native.NativeCompiledProject | None = field( - default=None, compare=False, repr=False - ) - - @cached_property - def lineage_graph(self) -> _native.NativeProjectGraph: - """The lineage edges, tags, model folders and names, indexed natively once per project.""" - - rows: list[tuple[tuple[str, str], list[tuple[str, str]], list[str], str | None]] = [ - ( - self._graph_key(model.key), - self._graph_keys(model.deps), - self._graph_tags(model.config.values.get("tags")), - str(model.relative_path.parent), - ) - for model in self.models - ] - rows.extend( - (self._graph_key(source.key), self._graph_keys(source.deps), [], None) - for source in self.sources - ) - rows.extend( - ( - self._graph_key(seed.key), - self._graph_keys(seed.deps), - list(seed.schema_entry.tags), - None, - ) - for seed in self.seeds - ) - rows.extend( - ( - self._graph_key(function.key), - self._graph_keys(function.deps), - list(function.tags), - None, - ) - for function in self.functions - ) - return _native.NativeProjectGraph.from_resources(rows) - - @staticmethod - def _graph_key(key: CompiledObjectKey) -> tuple[str, str]: - return (str(key.resource_type), key.name) - - @classmethod - def _graph_keys(cls, keys: Sequence[CompiledObjectKey]) -> list[tuple[str, str]]: - return [cls._graph_key(key) for key in keys] - - @staticmethod - def _graph_tags(value: object) -> list[str]: - if isinstance(value, list | tuple): - return [str(item) for item in value] - return [] - - -@dataclass(frozen=True) -class CompiledSqlTest: - """Compiled SQL-native unit test metadata selected by expected targets.""" - - key: CompiledObjectKey - scope_deps: tuple[CompiledObjectKey, ...] - name: str - test_file: DiscoveredSqlTestFile - test_block: DiscoveredSqlTestBlock - sql_body: str - mode: SqlTestMode = DEFAULT_SQL_TEST_MODE - payload: CompiledModelSqlTestPayload | CompiledDirectLogicSqlTestPayload = field( - default_factory=CompiledModelSqlTestPayload - ) - source_path: Path | None = None - ownership_root: Path | None = None - block_index: int | None = None - explicit_name: str | None = None - parent_name: str | None = None - case_name: str | None = None - case_index: int | None = None - case_fingerprint: str | None = None - parameter_schema: tuple[SqlTestParameterDeclaration, ...] = field(default_factory=tuple) - parameter_values: tuple[tuple[str, SqlValue], ...] = field(default_factory=tuple) - expected_model_names: tuple[str, ...] = field(default_factory=tuple) - assertion_names: tuple[str, ...] = field(default_factory=tuple) - assertion_target_model_names: tuple[str, ...] = field(default_factory=tuple) - read_helper_names: tuple[str, ...] = field(default_factory=tuple) - reference_target_model_names: tuple[str, ...] = field(default_factory=tuple) - target_model_names: tuple[str, ...] = field(default_factory=tuple) - tested_resources: tuple[CompiledSqlTestResource, ...] = field(default_factory=tuple) - - def __post_init__(self) -> None: - if self.source_path is None: - object.__setattr__(self, "source_path", self.test_file.relative_path) - if self.ownership_root is None: - object.__setattr__(self, "ownership_root", self.test_file.ownership_root) - if self.block_index is None: - object.__setattr__(self, "block_index", self.test_block.test_index) - if self.explicit_name is None and self.test_block.name is not None: - object.__setattr__(self, "explicit_name", self.test_block.name) - if self.parent_name is None: - object.__setattr__( - self, - "parent_name", - self.test_block.name or self.test_file.relative_path.stem, - ) diff --git a/src/sqlbuild/compiler/compile/types.py b/src/sqlbuild/compiler/compile/types.py index 711ccd3c20..f1eac540c4 100644 --- a/src/sqlbuild/compiler/compile/types.py +++ b/src/sqlbuild/compiler/compile/types.py @@ -14,10 +14,16 @@ import sqlbuild._native as _native from sqlbuild._native import ( CompiledLineageColumnFact, + CompiledObjectKey, + CompiledRelationLocation, CompiledSeed, CompiledSource, CompiledSqlExpansion, + CompiledSqlScenario, + CompiledSqlTest, CompileModelConfig, + CompilerDiagnostic, + CompileSqlReference, ConstantDeclaration, DiscoveredAuditBlock, DiscoveredAuditFile, @@ -32,7 +38,9 @@ EnumDeclaration, FunctionArgument, FunctionReturnColumn, + InferredColumn, LiteralSqlRelation, + LoadedMacro, MeasurementContract, MeasurementThresholds, ScenarioConfig, @@ -46,16 +54,6 @@ UsageRecord, ) from sqlbuild.compiler.auditing.types import AuditEvaluationMode, AuditSeverity - from sqlbuild.compiler.compile.models import ( - CompiledObjectKey, - CompiledRelationLocation, - CompiledSqlScenario, - CompiledSqlTest, - CompilerDiagnostic, - CompileSqlReference, - InferredColumn, - LoadedMacro, - ) type RelationKeyRow = tuple[ diff --git a/src/sqlbuild/executor/auditing/_helpers/execution.py b/src/sqlbuild/executor/auditing/_helpers/execution.py index 4a0d5d7310..edfe1f4fd4 100644 --- a/src/sqlbuild/executor/auditing/_helpers/execution.py +++ b/src/sqlbuild/executor/auditing/_helpers/execution.py @@ -11,6 +11,7 @@ from sqlbuild._native import ( AuditPlanEntry, + CompiledRelationLocation, CostContext, SourceEntry, assert_no_unresolved_sql_markers, @@ -23,7 +24,6 @@ AuditOutcome, AuditRunScope, ) -from sqlbuild.compiler.compile.models import CompiledRelationLocation from sqlbuild.diagnostics.main.diagnostics_context import diagnostics_context from sqlbuild.executor.auditing.constants import ( DEFAULT_EVIDENCE_ROW_LIMIT, diff --git a/src/sqlbuild/executor/auditing/_helpers/result_projection.py b/src/sqlbuild/executor/auditing/_helpers/result_projection.py index 80d7baec19..5e1ca99d0f 100644 --- a/src/sqlbuild/executor/auditing/_helpers/result_projection.py +++ b/src/sqlbuild/executor/auditing/_helpers/result_projection.py @@ -16,13 +16,13 @@ from sqlbuild._native import ( AuditIdentity, AuditPlanEntry, + CompiledRelationLocation, MeasurementThresholdBound, PlanOutput, build_audit_gate_identity, ) from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter from sqlbuild.compiler.auditing.types import AuditEvaluationMode, ThresholdOperator -from sqlbuild.compiler.compile.models import CompiledRelationLocation from sqlbuild.executor.audit_results.constants import AUDIT_RESULT_SCHEMA_VERSION from sqlbuild.executor.audit_results.exceptions import AuditResultStorageError from sqlbuild.executor.audit_results.main._write import write_audit_result_records diff --git a/src/sqlbuild/executor/auditing/main/_execute.py b/src/sqlbuild/executor/auditing/main/_execute.py index 6ff96df6ed..716360750f 100644 --- a/src/sqlbuild/executor/auditing/main/_execute.py +++ b/src/sqlbuild/executor/auditing/main/_execute.py @@ -4,10 +4,9 @@ from typing import Any -from sqlbuild._native import AuditPlanEntry, SourceEntry +from sqlbuild._native import AuditPlanEntry, CompiledRelationLocation, SourceEntry from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter from sqlbuild.compiler.auditing.types import AuditRunScope -from sqlbuild.compiler.compile.models import CompiledRelationLocation from sqlbuild.executor.auditing._helpers.execution import execute_audit_impl from sqlbuild.executor.auditing.models import AuditExecutionResult diff --git a/src/sqlbuild/executor/auditing/main/_execute_reporting_errors.py b/src/sqlbuild/executor/auditing/main/_execute_reporting_errors.py index e1233f76d6..b77e8a9093 100644 --- a/src/sqlbuild/executor/auditing/main/_execute_reporting_errors.py +++ b/src/sqlbuild/executor/auditing/main/_execute_reporting_errors.py @@ -4,10 +4,9 @@ from typing import Any -from sqlbuild._native import AuditPlanEntry, SourceEntry +from sqlbuild._native import AuditPlanEntry, CompiledRelationLocation, SourceEntry from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter from sqlbuild.compiler.auditing.types import AuditRunScope -from sqlbuild.compiler.compile.models import CompiledRelationLocation from sqlbuild.executor.auditing._helpers.execution import execute_audit_impl, failed_audit_result from sqlbuild.executor.auditing.models import AuditExecutionResult diff --git a/src/sqlbuild/executor/auditing/models.py b/src/sqlbuild/executor/auditing/models.py index 13ff6176e0..27c26b1b5a 100644 --- a/src/sqlbuild/executor/auditing/models.py +++ b/src/sqlbuild/executor/auditing/models.py @@ -6,7 +6,7 @@ from dataclasses import dataclass from typing import Any -from sqlbuild._native import MeasurementThresholds, SourceEntry +from sqlbuild._native import CompiledRelationLocation, MeasurementThresholds, SourceEntry from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter from sqlbuild.compiler.auditing.types import ( AuditAttachmentKind, @@ -15,7 +15,6 @@ AuditRunScope, AuditSeverity, ) -from sqlbuild.compiler.compile.models import CompiledRelationLocation from sqlbuild.compiler.compile.types import AttachedAuditTargetKind diff --git a/src/sqlbuild/executor/build/_helpers/blocking.py b/src/sqlbuild/executor/build/_helpers/blocking.py index 9006edcf40..7a2aceb1eb 100644 --- a/src/sqlbuild/executor/build/_helpers/blocking.py +++ b/src/sqlbuild/executor/build/_helpers/blocking.py @@ -2,8 +2,7 @@ from __future__ import annotations -from sqlbuild._native import transitive_closure -from sqlbuild.compiler.compile.models import CompiledObjectKey +from sqlbuild._native import CompiledObjectKey, transitive_closure def downstream_blocked_keys( diff --git a/src/sqlbuild/executor/build/_helpers/end_audits.py b/src/sqlbuild/executor/build/_helpers/end_audits.py index 4d2e8e8a07..b49ea7f544 100644 --- a/src/sqlbuild/executor/build/_helpers/end_audits.py +++ b/src/sqlbuild/executor/build/_helpers/end_audits.py @@ -4,10 +4,14 @@ from typing import Any -from sqlbuild._native import AuditPlanEntry, SourceEntry, audit_resource_id +from sqlbuild._native import ( + AuditPlanEntry, + CompiledRelationLocation, + SourceEntry, + audit_resource_id, +) from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter from sqlbuild.compiler.auditing.types import AuditOutcome, AuditRunScope -from sqlbuild.compiler.compile.models import CompiledRelationLocation from sqlbuild.executor.auditing.main._execute_reporting_errors import ( execute_audit_reporting_errors, ) diff --git a/src/sqlbuild/executor/build/_helpers/external_source_loads.py b/src/sqlbuild/executor/build/_helpers/external_source_loads.py index cfbfda20c9..25840ab945 100644 --- a/src/sqlbuild/executor/build/_helpers/external_source_loads.py +++ b/src/sqlbuild/executor/build/_helpers/external_source_loads.py @@ -5,6 +5,7 @@ from collections.abc import Callable from sqlbuild._native import ( + CompiledObjectKey, DiscoveredLoaderFunction, LoaderConnectionMode, PlanOutput, @@ -12,7 +13,6 @@ SourceLoadPlanEntry, ) from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter -from sqlbuild.compiler.compile.models import CompiledObjectKey from sqlbuild.errors.contracts.exceptions import ExecutorInputError from sqlbuild.executor.build._helpers.source_node import execute_build_source_node from sqlbuild.executor.build.models import ( diff --git a/src/sqlbuild/executor/build/_helpers/scheduler.py b/src/sqlbuild/executor/build/_helpers/scheduler.py index 6d136f34c1..ecb87b2763 100644 --- a/src/sqlbuild/executor/build/_helpers/scheduler.py +++ b/src/sqlbuild/executor/build/_helpers/scheduler.py @@ -5,6 +5,7 @@ from collections.abc import Callable, Mapping from sqlbuild._native import ( + CompiledObjectKey, IncrementalMode, MaterializationType, ModelPlanEntry, @@ -17,7 +18,6 @@ ) from sqlbuild.adapter.contract.models import RelationInfo from sqlbuild.adapter.contract.types import TablePromotionMode -from sqlbuild.compiler.compile.models import CompiledObjectKey from sqlbuild.compiler.planner.constants import MICROBATCH_END_SENTINEL, MICROBATCH_START_SENTINEL from sqlbuild.errors.contracts.exceptions import ExecutorInputError from sqlbuild.executor.build.constants import ( diff --git a/src/sqlbuild/executor/build/_helpers/source_audits.py b/src/sqlbuild/executor/build/_helpers/source_audits.py index fdce952fc4..2548cd8384 100644 --- a/src/sqlbuild/executor/build/_helpers/source_audits.py +++ b/src/sqlbuild/executor/build/_helpers/source_audits.py @@ -4,10 +4,9 @@ from typing import Any -from sqlbuild._native import AuditPlanEntry, PlanOutput, audit_resource_id +from sqlbuild._native import AuditPlanEntry, CompiledObjectKey, PlanOutput, audit_resource_id from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter from sqlbuild.compiler.auditing.types import AuditOutcome, AuditRunScope -from sqlbuild.compiler.compile.models import CompiledObjectKey from sqlbuild.compiler.compile.types import CompiledResourceType from sqlbuild.executor.auditing.main._execute_reporting_errors import ( execute_audit_reporting_errors, diff --git a/src/sqlbuild/executor/build/_helpers/source_node.py b/src/sqlbuild/executor/build/_helpers/source_node.py index 78375bd589..84ae9ffec6 100644 --- a/src/sqlbuild/executor/build/_helpers/source_node.py +++ b/src/sqlbuild/executor/build/_helpers/source_node.py @@ -8,6 +8,7 @@ from typing import Any from sqlbuild._native import ( + CompiledObjectKey, CostContext, DiscoveredLoaderFunction, PlanOutput, @@ -16,7 +17,6 @@ ) from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter from sqlbuild.adapter.contract.classes.statement_recorder import StatementRecorder -from sqlbuild.compiler.compile.models import CompiledObjectKey from sqlbuild.executor.build.models import BuildCallbacks, BuildRuntimeParams from sqlbuild.executor.load.main._execute import execute_source_load from sqlbuild.executor.load.models import LoadExecutionResult, LoadRuntimeParams diff --git a/src/sqlbuild/executor/build/classes/build_scheduler.py b/src/sqlbuild/executor/build/classes/build_scheduler.py index dcb64aef78..2d8fad6bcb 100644 --- a/src/sqlbuild/executor/build/classes/build_scheduler.py +++ b/src/sqlbuild/executor/build/classes/build_scheduler.py @@ -17,6 +17,7 @@ from sqlbuild._native import ( AuditPlanEntry, + CompiledObjectKey, CostContext, DiscoveredLoaderFunction, FunctionPlanEntry, @@ -37,7 +38,6 @@ from sqlbuild.adapter.contract.models import RelationInfo from sqlbuild.adapter.contract.types import TablePromotionMode from sqlbuild.compiler.auditing.types import AuditOutcome -from sqlbuild.compiler.compile.models import CompiledObjectKey from sqlbuild.compiler.compile.types import CompiledResourceType from sqlbuild.cost.models import StatementExecutionTelemetry from sqlbuild.diagnostics.main.diagnostics_context import diagnostics_context diff --git a/src/sqlbuild/executor/build/main/_external_source_loads.py b/src/sqlbuild/executor/build/main/_external_source_loads.py index 752cc09978..4c8ce832c3 100644 --- a/src/sqlbuild/executor/build/main/_external_source_loads.py +++ b/src/sqlbuild/executor/build/main/_external_source_loads.py @@ -2,9 +2,8 @@ from __future__ import annotations -from sqlbuild._native import DiscoveredLoaderFunction, PlanOutput +from sqlbuild._native import CompiledObjectKey, DiscoveredLoaderFunction, PlanOutput from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter -from sqlbuild.compiler.compile.models import CompiledObjectKey from sqlbuild.executor.build._helpers.external_source_loads import ( run_external_source_loads_before_connections as _run_external_source_loads_before_connections, ) diff --git a/src/sqlbuild/executor/build/models.py b/src/sqlbuild/executor/build/models.py index 691ab14055..6ef91accb1 100644 --- a/src/sqlbuild/executor/build/models.py +++ b/src/sqlbuild/executor/build/models.py @@ -9,6 +9,8 @@ from sqlbuild._native import ( AuditPlanEntry, + CompiledObjectKey, + CompiledRelationLocation, DiscoveredLoaderFunction, FunctionPlanEntry, ModelPlanEntry, @@ -19,7 +21,6 @@ ) from sqlbuild.adapter.contract.models import LifeCycleEvent, RelationInfo from sqlbuild.adapter.contract.types import TablePromotionMode -from sqlbuild.compiler.compile.models import CompiledObjectKey, CompiledRelationLocation from sqlbuild.cost.models import StatementExecutionTelemetry from sqlbuild.executor.auditing.models import AuditExecutionResult from sqlbuild.executor.build.types import BeforeModelMaterializeCallback, BuildStatus diff --git a/src/sqlbuild/executor/clone/_helpers/execution_items.py b/src/sqlbuild/executor/clone/_helpers/execution_items.py index 47537fa20c..487b5e3694 100644 --- a/src/sqlbuild/executor/clone/_helpers/execution_items.py +++ b/src/sqlbuild/executor/clone/_helpers/execution_items.py @@ -7,6 +7,7 @@ from sqlbuild._native import ( CloneSourcePlanEntry, + CompiledObjectKey, FunctionPlanEntry, MaterializationType, ModelPlanEntry, @@ -15,7 +16,6 @@ from sqlbuild.adapter.contract.classes.statement_recorder import StatementRecorder from sqlbuild.adapter.contract.models import RelationLookup, RetentionRequest from sqlbuild.adapter.contract.types import RetentionScope -from sqlbuild.compiler.compile.models import CompiledObjectKey from sqlbuild.executor.build.models import FunctionExecutionResult from sqlbuild.executor.clone._helpers.operations import clone_relation, recreate_view from sqlbuild.executor.clone._helpers.retention import apply_clone_retention diff --git a/src/sqlbuild/executor/clone/main/execute.py b/src/sqlbuild/executor/clone/main/execute.py index af25a28b55..c660fad021 100644 --- a/src/sqlbuild/executor/clone/main/execute.py +++ b/src/sqlbuild/executor/clone/main/execute.py @@ -2,9 +2,14 @@ from __future__ import annotations -from sqlbuild._native import CloneSourcePlanEntry, FunctionPlanEntry, ModelPlanEntry, SeedPlanEntry +from sqlbuild._native import ( + CloneSourcePlanEntry, + CompiledObjectKey, + FunctionPlanEntry, + ModelPlanEntry, + SeedPlanEntry, +) from sqlbuild.adapter.contract.models import RelationLookup -from sqlbuild.compiler.compile.models import CompiledObjectKey from sqlbuild.executor.clone._helpers.canonical_progress import clone_item_enrichment from sqlbuild.executor.clone._helpers.execution_items import ( execute_clone_destination_item, diff --git a/src/sqlbuild/executor/clone/models.py b/src/sqlbuild/executor/clone/models.py index 19ffdebe91..1ab31e3159 100644 --- a/src/sqlbuild/executor/clone/models.py +++ b/src/sqlbuild/executor/clone/models.py @@ -5,10 +5,16 @@ from dataclasses import dataclass, field from typing import Any -from sqlbuild._native import CloneSourcePlanEntry, FunctionPlanEntry, ModelPlanEntry, SeedPlanEntry +from sqlbuild._native import ( + CloneSourcePlanEntry, + CompiledObjectKey, + CompiledRelationLocation, + FunctionPlanEntry, + ModelPlanEntry, + SeedPlanEntry, +) from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter from sqlbuild.adapter.contract.models import LifeCycleEvent, RetentionRequest -from sqlbuild.compiler.compile.models import CompiledObjectKey, CompiledRelationLocation from sqlbuild.executor.clone.types import CloneAction, CloneItemCallback, CloneStatus diff --git a/src/sqlbuild/executor/migrations/_helpers/old_name_views.py b/src/sqlbuild/executor/migrations/_helpers/old_name_views.py index 34de345f4a..38b7aa6ff0 100644 --- a/src/sqlbuild/executor/migrations/_helpers/old_name_views.py +++ b/src/sqlbuild/executor/migrations/_helpers/old_name_views.py @@ -22,7 +22,6 @@ resolve_qualified_name_parts, ) from sqlbuild.adapter.type_system.main.normalize_relation_type import normalize_relation_type -from sqlbuild.compiler.compile.models import CompiledRelationLocation from sqlbuild.compiler.migrations.constants import COLUMN_MIGRATION_TABLE_NAME from sqlbuild.errors.contracts.exceptions import ExecutorInputError from sqlbuild.executor.migrations._helpers.old_name_facts import ( diff --git a/src/sqlbuild/executor/migrations/_helpers/relation_for_location.py b/src/sqlbuild/executor/migrations/_helpers/relation_for_location.py index 4131202980..7b0ffd93c5 100644 --- a/src/sqlbuild/executor/migrations/_helpers/relation_for_location.py +++ b/src/sqlbuild/executor/migrations/_helpers/relation_for_location.py @@ -2,8 +2,7 @@ from __future__ import annotations -from sqlbuild._native import MigrationRelation -from sqlbuild.compiler.compile.models import CompiledRelationLocation +from sqlbuild._native import CompiledRelationLocation, MigrationRelation def migration_relation_for_location(location: CompiledRelationLocation) -> MigrationRelation: diff --git a/src/sqlbuild/executor/migrations/main/_bound_old_name_view_guard.py b/src/sqlbuild/executor/migrations/main/_bound_old_name_view_guard.py index fe4f97cec8..55379a6de6 100644 --- a/src/sqlbuild/executor/migrations/main/_bound_old_name_view_guard.py +++ b/src/sqlbuild/executor/migrations/main/_bound_old_name_view_guard.py @@ -6,9 +6,8 @@ from functools import partial from typing import Any -from sqlbuild._native import PlanOutput +from sqlbuild._native import CompiledRelationLocation, PlanOutput from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter -from sqlbuild.compiler.compile.models import CompiledRelationLocation from sqlbuild.executor.migrations._helpers.old_name_views import views_reading from sqlbuild.executor.migrations.classes.bound_old_name_views import BoundOldNameViews from sqlbuild.executor.migrations.models import OldNameViewSource diff --git a/src/sqlbuild/executor/migrations/main/_compatibility_view_sql.py b/src/sqlbuild/executor/migrations/main/_compatibility_view_sql.py index 3d2aecb8ba..191dae4901 100644 --- a/src/sqlbuild/executor/migrations/main/_compatibility_view_sql.py +++ b/src/sqlbuild/executor/migrations/main/_compatibility_view_sql.py @@ -4,9 +4,8 @@ from typing import Any -from sqlbuild._native import MigrationRelation, OldNameViewHistory +from sqlbuild._native import CompiledRelationLocation, MigrationRelation, OldNameViewHistory from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter -from sqlbuild.compiler.compile.models import CompiledRelationLocation from sqlbuild.executor.migrations._helpers.old_name_views import render_old_name_view_select from sqlbuild.executor.migrations.models import OldNameViewSource diff --git a/src/sqlbuild/executor/migrations/main/_complete_old_name_views.py b/src/sqlbuild/executor/migrations/main/_complete_old_name_views.py index 63ce40982b..00eaf0ef35 100644 --- a/src/sqlbuild/executor/migrations/main/_complete_old_name_views.py +++ b/src/sqlbuild/executor/migrations/main/_complete_old_name_views.py @@ -4,9 +4,8 @@ from typing import Any -from sqlbuild._native import PlanOutput +from sqlbuild._native import CompiledRelationLocation, PlanOutput from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter -from sqlbuild.compiler.compile.models import CompiledRelationLocation from sqlbuild.executor.migrations._helpers.old_name_views import ( rebind_old_name_views_atomically, run_planned_old_name_steps, diff --git a/src/sqlbuild/executor/migrations/models.py b/src/sqlbuild/executor/migrations/models.py index 23e6b87f43..ac6a9e44f1 100644 --- a/src/sqlbuild/executor/migrations/models.py +++ b/src/sqlbuild/executor/migrations/models.py @@ -4,8 +4,8 @@ from dataclasses import dataclass +from sqlbuild._native import CompiledRelationLocation from sqlbuild.adapter.contract.models import RelationGrant -from sqlbuild.compiler.compile.models import CompiledRelationLocation @dataclass(frozen=True) diff --git a/src/sqlbuild/executor/pipeline/_helpers/scenario.py b/src/sqlbuild/executor/pipeline/_helpers/scenario.py index 9d1736fc0d..47b2649c76 100644 --- a/src/sqlbuild/executor/pipeline/_helpers/scenario.py +++ b/src/sqlbuild/executor/pipeline/_helpers/scenario.py @@ -16,6 +16,7 @@ from typing import Any, ClassVar, Protocol from sqlbuild._native import ( + CompiledSqlScenario, CompilePipelineResult, NativeProgressProjector, ScenarioExecutionPlan, @@ -29,7 +30,6 @@ from sqlbuild._native import build_cli_scenario_plan as build_scenario_plan from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter from sqlbuild.adapter.contract.types import TablePromotionMode -from sqlbuild.compiler.compile.models import CompiledSqlScenario from sqlbuild.diagnostics.main.log_debug_event import log_debug_event from sqlbuild.executor.pipeline._helpers.connections import ( close_connections, diff --git a/src/sqlbuild/executor/pipeline/_helpers/testing.py b/src/sqlbuild/executor/pipeline/_helpers/testing.py index 3bdb0bf93e..0a8fd02066 100644 --- a/src/sqlbuild/executor/pipeline/_helpers/testing.py +++ b/src/sqlbuild/executor/pipeline/_helpers/testing.py @@ -12,14 +12,17 @@ from typing import Any, cast from uuid import uuid4 -from sqlbuild._native import FunctionPlanEntry, PlanOutput, SqlTestPlanEntry, sql_test_resource_id -from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter -from sqlbuild.adapter.contract.classes.statement_recorder import StatementRecorder -from sqlbuild.adapter.contract.models import FunctionInfo -from sqlbuild.compiler.compile.models import ( +from sqlbuild._native import ( CompiledObjectKey, CompiledRelationLocation, + FunctionPlanEntry, + PlanOutput, + SqlTestPlanEntry, + sql_test_resource_id, ) +from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter +from sqlbuild.adapter.contract.classes.statement_recorder import StatementRecorder +from sqlbuild.adapter.contract.models import FunctionInfo from sqlbuild.compiler.compile.types import FunctionLanguage from sqlbuild.executor.build.models import FunctionExecutionResult from sqlbuild.executor.functions.main._execute import execute_function diff --git a/src/sqlbuild/executor/run/_helpers/execution/final_audits.py b/src/sqlbuild/executor/run/_helpers/execution/final_audits.py index 1c9e0dc1ed..7f0ab92c27 100644 --- a/src/sqlbuild/executor/run/_helpers/execution/final_audits.py +++ b/src/sqlbuild/executor/run/_helpers/execution/final_audits.py @@ -4,10 +4,9 @@ from typing import Any -from sqlbuild._native import AuditPlanEntry, Fingerprint, SourceEntry +from sqlbuild._native import AuditPlanEntry, CompiledRelationLocation, Fingerprint, SourceEntry from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter from sqlbuild.compiler.auditing.types import AuditOutcome, AuditRunScope -from sqlbuild.compiler.compile.models import CompiledRelationLocation from sqlbuild.executor.auditing.main._execute import execute_audit from sqlbuild.executor.auditing.models import AuditExecutionResult from sqlbuild.executor.run._helpers.reuse.audit import ( diff --git a/src/sqlbuild/executor/run/_helpers/execution/hooks.py b/src/sqlbuild/executor/run/_helpers/execution/hooks.py index b1cce44223..a83dee9f83 100644 --- a/src/sqlbuild/executor/run/_helpers/execution/hooks.py +++ b/src/sqlbuild/executor/run/_helpers/execution/hooks.py @@ -6,6 +6,7 @@ from typing import Any from sqlbuild._native import ( + CompiledRelationLocation, CostContext, DiscoveredHookFunction, HardCodedRelationOwnerKind, @@ -20,7 +21,6 @@ from sqlbuild.adapter.relations.main.resolve_relation_location_qualified_name import ( resolve_relation_location_qualified_name, ) -from sqlbuild.compiler.compile.models import CompiledRelationLocation from sqlbuild.compiler.fingerprints.constants import NODE_TYPE_HOOK from sqlbuild.compiler.python_nodes.main.identity import build_python_node_identity from sqlbuild.compiler.python_nodes.types import SkipMode diff --git a/src/sqlbuild/executor/run/_helpers/materializations/custom.py b/src/sqlbuild/executor/run/_helpers/materializations/custom.py index bebc9efea5..d12c469e5a 100644 --- a/src/sqlbuild/executor/run/_helpers/materializations/custom.py +++ b/src/sqlbuild/executor/run/_helpers/materializations/custom.py @@ -7,7 +7,13 @@ from dataclasses import replace from typing import Any, cast -from sqlbuild._native import AuditPlanEntry, ModelPlanEntry, PlanReason, SourceEntry +from sqlbuild._native import ( + AuditPlanEntry, + CompiledRelationLocation, + ModelPlanEntry, + PlanReason, + SourceEntry, +) from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter from sqlbuild.adapter.contract.classes.statement_recorder import StatementRecorder from sqlbuild.adapter.contract.models import ColumnInfo, RelationInfo @@ -15,7 +21,6 @@ resolve_relation_location_qualified_name, ) from sqlbuild.compiler.auditing.types import AuditRunScope -from sqlbuild.compiler.compile.models import CompiledRelationLocation from sqlbuild.diagnostics.main.diagnostics_context import diagnostics_context from sqlbuild.errors.contracts.exceptions import ExecutorInputError from sqlbuild.executor.auditing.main._execute import execute_audit diff --git a/src/sqlbuild/executor/run/_helpers/materializations/view.py b/src/sqlbuild/executor/run/_helpers/materializations/view.py index 3e3a682cb3..4d3d1453b6 100644 --- a/src/sqlbuild/executor/run/_helpers/materializations/view.py +++ b/src/sqlbuild/executor/run/_helpers/materializations/view.py @@ -5,14 +5,19 @@ from collections.abc import Mapping from typing import Any -from sqlbuild._native import AuditPlanEntry, DiscoveredHookFunction, ModelPlanEntry, SourceEntry +from sqlbuild._native import ( + AuditPlanEntry, + CompiledRelationLocation, + DiscoveredHookFunction, + ModelPlanEntry, + SourceEntry, +) from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter from sqlbuild.adapter.contract.classes.statement_recorder import StatementRecorder from sqlbuild.adapter.relations.main.resolve_relation_location_qualified_name import ( resolve_relation_location_qualified_name, ) from sqlbuild.compiler.auditing.types import AuditOutcome, AuditRunScope -from sqlbuild.compiler.compile.models import CompiledRelationLocation from sqlbuild.diagnostics.main.diagnostics_context import diagnostics_context from sqlbuild.executor.auditing.main._execute import execute_audit from sqlbuild.executor.auditing.models import AuditExecutionResult diff --git a/src/sqlbuild/executor/run/models.py b/src/sqlbuild/executor/run/models.py index eb0c923a80..598dc135be 100644 --- a/src/sqlbuild/executor/run/models.py +++ b/src/sqlbuild/executor/run/models.py @@ -10,6 +10,7 @@ from sqlbuild._native import ( AuditPlanEntry, + CompiledRelationLocation, CursorBounds, CursorGrain, CursorType, @@ -32,7 +33,6 @@ from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter from sqlbuild.adapter.contract.classes.statement_recorder import StatementRecorder from sqlbuild.adapter.contract.models import LifeCycleEvent, QueryResult -from sqlbuild.compiler.compile.models import CompiledRelationLocation from sqlbuild.compiler.python_nodes.types import SkipMode from sqlbuild.cursor_algebra.exceptions import CursorAlgebraError from sqlbuild.cursor_algebra.types import BoundSentinel, CursorScalar diff --git a/src/sqlbuild/executor/scenario/_helpers/lifecycle/cleanup.py b/src/sqlbuild/executor/scenario/_helpers/lifecycle/cleanup.py index c80973fa69..7988d10312 100644 --- a/src/sqlbuild/executor/scenario/_helpers/lifecycle/cleanup.py +++ b/src/sqlbuild/executor/scenario/_helpers/lifecycle/cleanup.py @@ -8,6 +8,7 @@ from typing import Any from sqlbuild._native import ( + CompiledRelationLocation, MaterializationType, ModelPlanEntry, ScenarioArtifactKind, @@ -25,7 +26,6 @@ resolve_relation_location_qualified_name, ) from sqlbuild.adapter.type_system.main.normalize_relation_type import normalize_relation_type -from sqlbuild.compiler.compile.models import CompiledRelationLocation from sqlbuild.diagnostics.main.log_debug_event import log_debug_event from sqlbuild.executor.run.main._table_targets import resolve_table_targets from sqlbuild.executor.run.main._type_enforcement_relation_name import ( diff --git a/src/sqlbuild/executor/scenario/_helpers/local/execution.py b/src/sqlbuild/executor/scenario/_helpers/local/execution.py index 4d4b5d8e0e..a702cae017 100644 --- a/src/sqlbuild/executor/scenario/_helpers/local/execution.py +++ b/src/sqlbuild/executor/scenario/_helpers/local/execution.py @@ -7,6 +7,7 @@ from typing import Any from sqlbuild._native import ( + CompiledRelationLocation, FunctionPlanEntry, ModelPlanEntry, ScenarioArtifactKind, @@ -22,7 +23,6 @@ from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter from sqlbuild.adapter.contract.classes.statement_recorder import StatementRecorder from sqlbuild.adapter.contract.types import TablePromotionMode -from sqlbuild.compiler.compile.models import CompiledRelationLocation from sqlbuild.compiler.compile.types import FunctionLanguage from sqlbuild.executor.build.models import FunctionExecutionResult from sqlbuild.executor.functions.main._execute import execute_function diff --git a/src/sqlbuild/executor/scenario/classes/prepared_scenario_schemas.py b/src/sqlbuild/executor/scenario/classes/prepared_scenario_schemas.py index da5df38d9b..20d9abd2ae 100644 --- a/src/sqlbuild/executor/scenario/classes/prepared_scenario_schemas.py +++ b/src/sqlbuild/executor/scenario/classes/prepared_scenario_schemas.py @@ -6,10 +6,9 @@ import threading from typing import Any -from sqlbuild._native import ScenarioExecutionPlan +from sqlbuild._native import CompiledRelationLocation, ScenarioExecutionPlan from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter from sqlbuild.adapter.contract.classes.statement_recorder import StatementRecorder -from sqlbuild.compiler.compile.models import CompiledRelationLocation from sqlbuild.diagnostics.main.log_debug_event import log_debug_event _DEBUG_LOGGER: logging.Logger = logging.getLogger("sqlbuild.execution") diff --git a/src/sqlbuild/executor/scenario/models.py b/src/sqlbuild/executor/scenario/models.py index 46019d1e31..22c2fd7251 100644 --- a/src/sqlbuild/executor/scenario/models.py +++ b/src/sqlbuild/executor/scenario/models.py @@ -6,6 +6,7 @@ from pathlib import Path from sqlbuild._native import ( + CompiledRelationLocation, MaterializationType, ScenarioArtifactKind, ScenarioExecutionPlan, @@ -14,7 +15,6 @@ ) from sqlbuild.adapter.contract.models import LifeCycleEvent from sqlbuild.adapter.contract.types import TablePromotionMode -from sqlbuild.compiler.compile.models import CompiledRelationLocation from sqlbuild.executor.build.models import FunctionExecutionResult, SeedExecutionResult from sqlbuild.executor.run.models import ModelExecutionResult from sqlbuild.executor.scenario.classes.prepared_scenario_schemas import PreparedScenarioSchemas diff --git a/src/sqlbuild/microbatches/classes/direct_store.py b/src/sqlbuild/microbatches/classes/direct_store.py index e29016878d..ed3e6f5f9b 100644 --- a/src/sqlbuild/microbatches/classes/direct_store.py +++ b/src/sqlbuild/microbatches/classes/direct_store.py @@ -6,9 +6,8 @@ from dataclasses import replace from typing import Any -from sqlbuild._native import ModelPlanEntry +from sqlbuild._native import CompiledRelationLocation, ModelPlanEntry from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter -from sqlbuild.compiler.compile.models import CompiledRelationLocation from sqlbuild.microbatches._helpers.sql import ( build_existing_event_ids_sql, build_insert_many_sql, diff --git a/src/sqlbuild/rule_engine/_helpers/host/custom_host.py b/src/sqlbuild/rule_engine/_helpers/host/custom_host.py index cdddea35a9..9cbbd16f24 100644 --- a/src/sqlbuild/rule_engine/_helpers/host/custom_host.py +++ b/src/sqlbuild/rule_engine/_helpers/host/custom_host.py @@ -14,8 +14,7 @@ from pathlib import Path from typing import Any, BinaryIO -from sqlbuild._native import HostCancelledError, RulesError, build_rule_catalogue -from sqlbuild.compiler.compile.models import CompiledProject +from sqlbuild._native import CompiledProject, HostCancelledError, RulesError, build_rule_catalogue from sqlbuild.compiler.compile.types import CompiledProjectView from sqlbuild.rule_engine._helpers.engine.fact_replay import fact_key_payload from sqlbuild.rule_engine._helpers.host.custom_evaluation import evaluate_custom_rules diff --git a/src/sqlbuild/rule_engine/_helpers/host/tracked_facts.py b/src/sqlbuild/rule_engine/_helpers/host/tracked_facts.py index f5be531640..ca54c9ee78 100644 --- a/src/sqlbuild/rule_engine/_helpers/host/tracked_facts.py +++ b/src/sqlbuild/rule_engine/_helpers/host/tracked_facts.py @@ -6,8 +6,13 @@ from pathlib import Path from typing import Any -from sqlbuild._native import ConstantDeclaration, EnumDeclaration, FactDigestError -from sqlbuild.compiler.compile.models import CompiledSqlTest, InferredColumn +from sqlbuild._native import ( + CompiledSqlTest, + ConstantDeclaration, + EnumDeclaration, + FactDigestError, + InferredColumn, +) from sqlbuild.compiler.compile.types import CompiledAuditView from sqlbuild.rule_engine._helpers.engine.fact_replay import fact_error_digest, fact_value_digest from sqlbuild.rule_engine.classes.fact_reads import FactReads diff --git a/src/sqlbuild/rule_engine/classes/audit_facts.py b/src/sqlbuild/rule_engine/classes/audit_facts.py index 361b2dce77..0f033460cb 100644 --- a/src/sqlbuild/rule_engine/classes/audit_facts.py +++ b/src/sqlbuild/rule_engine/classes/audit_facts.py @@ -3,9 +3,9 @@ from functools import cached_property from sqlbuild._native import ( + CompiledAudit, compiled_audit_record, ) -from sqlbuild.compiler.compile.models import CompiledAudit from sqlbuild.compiler.compile.types import CompiledProjectView from sqlbuild.rule_engine.models import Model diff --git a/src/sqlbuild/rule_engine/classes/column_facts.py b/src/sqlbuild/rule_engine/classes/column_facts.py index f6b62b392a..72722b74f5 100644 --- a/src/sqlbuild/rule_engine/classes/column_facts.py +++ b/src/sqlbuild/rule_engine/classes/column_facts.py @@ -2,7 +2,7 @@ from functools import cached_property -from sqlbuild.compiler.compile.models import InferredColumn +from sqlbuild._native import InferredColumn from sqlbuild.compiler.compile.types import CompiledModelView, CompiledProjectView from sqlbuild.rule_engine.models import Model diff --git a/src/sqlbuild/rule_engine/classes/model_test_facts.py b/src/sqlbuild/rule_engine/classes/model_test_facts.py index 395bff6d35..6a031a4b05 100644 --- a/src/sqlbuild/rule_engine/classes/model_test_facts.py +++ b/src/sqlbuild/rule_engine/classes/model_test_facts.py @@ -2,7 +2,7 @@ from functools import cached_property -from sqlbuild.compiler.compile.models import CompiledSqlTest +from sqlbuild._native import CompiledSqlTest from sqlbuild.compiler.compile.types import CompiledProjectView, SqlTestMode from sqlbuild.rule_engine.models import Model diff --git a/src/sqlbuild/rule_engine/classes/project_tree.py b/src/sqlbuild/rule_engine/classes/project_tree.py index 9e37c1463b..407dadf1cf 100644 --- a/src/sqlbuild/rule_engine/classes/project_tree.py +++ b/src/sqlbuild/rule_engine/classes/project_tree.py @@ -152,7 +152,7 @@ def _validate_relative(path: str) -> None: def public_model(model: object) -> Model: - from sqlbuild.compiler.compile.models import CompiledModel + from sqlbuild._native import CompiledModel if not isinstance(model, (CompiledModel, _native.NativeCompiledModel)): raise RuleUsageError("model is not a compiled SQL model") diff --git a/tests/e2e/fixtures/waffle_shop/models/marts/_macros/time.py b/tests/e2e/fixtures/waffle_shop/models/marts/_macros/time.py index 041c18185b..06ec0c3445 100644 --- a/tests/e2e/fixtures/waffle_shop/models/marts/_macros/time.py +++ b/tests/e2e/fixtures/waffle_shop/models/marts/_macros/time.py @@ -1,6 +1,6 @@ """Time expression macros.""" -from sqlbuild.compiler.compile.models import MacroContext +from sqlbuild._native import MacroContext def timestamp_trunc(ctx: MacroContext, grain: str, expression: str) -> str: diff --git a/tests/integration/src/sqlbuild/adapters/snowflake/test_table_type_conversion.py b/tests/integration/src/sqlbuild/adapters/snowflake/test_table_type_conversion.py index b5b9186ed8..7a8bb03edc 100644 --- a/tests/integration/src/sqlbuild/adapters/snowflake/test_table_type_conversion.py +++ b/tests/integration/src/sqlbuild/adapters/snowflake/test_table_type_conversion.py @@ -6,9 +6,13 @@ import pytest -from sqlbuild._native import TableType, TableTypePlanEntry, table_type_copy_name +from sqlbuild._native import ( + CompiledRelationLocation, + TableType, + TableTypePlanEntry, + table_type_copy_name, +) from sqlbuild.adapter.contract.models import RelationInfo -from sqlbuild.compiler.compile.models import CompiledRelationLocation from sqlbuild.executor.build._helpers.retention import apply_table_type_conversion from sqlbuild.executor.clone.main._clone_relation_operation import clone_relation_by_names from tests.integration.src.sqlbuild.adapters.snowflake._test_types import ( diff --git a/tests/integration/src/sqlbuild/cli/commands/main/test_rules.py b/tests/integration/src/sqlbuild/cli/commands/main/test_rules.py index 51dad358ea..90b0042362 100644 --- a/tests/integration/src/sqlbuild/cli/commands/main/test_rules.py +++ b/tests/integration/src/sqlbuild/cli/commands/main/test_rules.py @@ -20,12 +20,12 @@ DiscoveredMacroFile, DiscoveredProjectInputs, LintRunResult, + LoadedMacro, SqlLexicalSyntax, active_lint_stop, rules_bulk_cache_path, ) from sqlbuild._native import run_cli as main -from sqlbuild.compiler.compile.models import LoadedMacro from tests.integration.src.sqlbuild.cli.commands.main._test_types import ( CustomHostFailureIntegrationTestCase, CustomHostInterruptIntegrationTestCase, diff --git a/tests/integration/src/sqlbuild/cli/commands/main/test_startup_import_footprint.py b/tests/integration/src/sqlbuild/cli/commands/main/test_startup_import_footprint.py index ac3a1a0ea9..a748006d60 100644 --- a/tests/integration/src/sqlbuild/cli/commands/main/test_startup_import_footprint.py +++ b/tests/integration/src/sqlbuild/cli/commands/main/test_startup_import_footprint.py @@ -41,7 +41,6 @@ "pydantic", "yaml", "sqlbuild.adapter.contract.classes.base_adapter", - "sqlbuild.compiler.compile.models", "sqlbuild.lint", "sqlbuild.rule_engine", "sqlbuild.runtime.compute_logs", @@ -63,7 +62,6 @@ "polyglot_sql", "pydantic", "sqlbuild.adapter.contract.classes.base_adapter", - "sqlbuild.compiler.compile.models", "sqlbuild.compiler.pipeline", ) REUSED_COMPILE_FORBIDDEN: tuple[str, ...] = ( diff --git a/tests/integration/src/sqlbuild/compiler/analysis_session/helpers.py b/tests/integration/src/sqlbuild/compiler/analysis_session/helpers.py index da3677e75a..2986b02dec 100644 --- a/tests/integration/src/sqlbuild/compiler/analysis_session/helpers.py +++ b/tests/integration/src/sqlbuild/compiler/analysis_session/helpers.py @@ -18,9 +18,11 @@ AnalysisCacheContext, CollectionRendering, ColumnLineageMode, + CompactLineageFacts, CompileAdapterContext, CompileInputError, DynamicColumnContractProof, + InferredColumn, ModelSqlAnalysis, NativeModelAnalyses, NativeModelAnalysisRequest, @@ -41,7 +43,6 @@ from sqlbuild.adapter.contract.models import ExpressionInferenceProfile from sqlbuild.adapters.duckdb.classes.duckdb_adapter import DuckDbAdapter from sqlbuild.cli.entry.main.entry import main -from sqlbuild.compiler.compile.models import CompactLineageFacts, InferredColumn from sqlbuild.compiler.compile.types import CompiledProjectView from sqlbuild.compiler.lineage.types import InferredNullability from tests.integration.src.sqlbuild.compiler.analysis_session._test_types import ( diff --git a/tests/integration/src/sqlbuild/compiler/auditing/main/test_rendering.py b/tests/integration/src/sqlbuild/compiler/auditing/main/test_rendering.py index ebd276dc64..bf4b6bb07b 100644 --- a/tests/integration/src/sqlbuild/compiler/auditing/main/test_rendering.py +++ b/tests/integration/src/sqlbuild/compiler/auditing/main/test_rendering.py @@ -6,8 +6,7 @@ import pytest -from sqlbuild._native import SourceEntry, render_audit_sql -from sqlbuild.compiler.compile.models import CompiledRelationLocation +from sqlbuild._native import CompiledRelationLocation, SourceEntry, render_audit_sql from tests.integration.src.sqlbuild.compiler.auditing.main._test_types import ( ExecuteRenderedAuditTestCase, ) diff --git a/tests/integration/src/sqlbuild/compiler/compile/helpers.py b/tests/integration/src/sqlbuild/compiler/compile/helpers.py index da62fbbb46..926cba2c18 100644 --- a/tests/integration/src/sqlbuild/compiler/compile/helpers.py +++ b/tests/integration/src/sqlbuild/compiler/compile/helpers.py @@ -19,6 +19,8 @@ CompileInputError, CompilerStage, DiscoveredProjectInputs, + LoadedMacro, + MacroContext, MacroExpansionResult, ResourceIdentity, ResourceKind, @@ -31,7 +33,6 @@ resolve_effective_collection_rendering, ) from sqlbuild.adapters.duckdb.classes.duckdb_adapter import DuckDbAdapter -from sqlbuild.compiler.compile.models import LoadedMacro, MacroContext from sqlbuild.compiler.macro_bridge.classes.macro_bridge import MacroBridge from sqlbuild.compiler.macro_bridge.constants import ACTIVE_MACRO_BRIDGE diff --git a/tests/integration/src/sqlbuild/compiler/compile/native_model_config/helpers.py b/tests/integration/src/sqlbuild/compiler/compile/native_model_config/helpers.py index e4bd4b8b4d..94cd47b87d 100644 --- a/tests/integration/src/sqlbuild/compiler/compile/native_model_config/helpers.py +++ b/tests/integration/src/sqlbuild/compiler/compile/native_model_config/helpers.py @@ -8,6 +8,7 @@ import sqlbuild._native as _native from sqlbuild._native import ( CompileModelConfig, + CompileSqlReference, DiscoveredSqlModelFile, ModelValidationRequest, NativeModelConfigInputs, @@ -16,7 +17,6 @@ SqlReferenceKind, native_model_config_session, ) -from sqlbuild.compiler.compile.models import CompileSqlReference _ACCEPTED: str = "accepted" diff --git a/tests/integration/src/sqlbuild/compiler/compiled_project/helpers.py b/tests/integration/src/sqlbuild/compiler/compiled_project/helpers.py index ce1fd9e280..72f66866e1 100644 --- a/tests/integration/src/sqlbuild/compiler/compiled_project/helpers.py +++ b/tests/integration/src/sqlbuild/compiler/compiled_project/helpers.py @@ -18,7 +18,9 @@ CollectionRendering, CompileAdapterContext, CompileModelConfig, + CompileModelInput, CompileProjectInputs, + CompilerDiagnostic, DiscoveredProjectInputs, DiscoveredSqlModelFile, LocalConfig, @@ -35,8 +37,6 @@ ) from sqlbuild.adapter.contract.models import ExpressionInferenceProfile from sqlbuild.adapters.duckdb.classes.duckdb_adapter import DuckDbAdapter -from sqlbuild.compiler.compile import models as compile_models -from sqlbuild.compiler.compile.models import CompileModelInput, CompilerDiagnostic from sqlbuild.compiler.compile.types import ( CompiledAuditView, CompiledFunctionView, @@ -120,7 +120,7 @@ def count_legacy_constructions(*, monkeypatch: pytest.MonkeyPatch) -> Counter[st counts: Counter[str] = Counter() for name in LEGACY_PROJECT_TYPES: - model_type: type = getattr(compile_models, name, None) or getattr(_native, name) + model_type: type = getattr(_native, name) monkeypatch.setattr( model_type, "__init__", _counted_init(counts=counts, name=name, model_type=model_type) ) diff --git a/tests/integration/src/sqlbuild/compiler/compiled_project/test_native_compiled_project.py b/tests/integration/src/sqlbuild/compiler/compiled_project/test_native_compiled_project.py index 8505f0d92b..c6db982c6f 100644 --- a/tests/integration/src/sqlbuild/compiler/compiled_project/test_native_compiled_project.py +++ b/tests/integration/src/sqlbuild/compiler/compiled_project/test_native_compiled_project.py @@ -12,13 +12,13 @@ from sqlbuild._native import ( CompileInputError, + InferredColumn, InvalidQualifiedIdentityError, build_compiled_project, discover_project_inputs, ) from sqlbuild.adapters.duckdb.classes.duckdb_adapter import DuckDbAdapter from sqlbuild.cli.entry.main.entry import main -from sqlbuild.compiler.compile.models import InferredColumn from sqlbuild.compiler.compile.types import CompiledProjectView from tests.integration.src.sqlbuild.compiler.compiled_project._test_types import ( ImmediateSyntaxErrorTestCase, diff --git a/tests/integration/src/sqlbuild/compiler/contracts/helpers.py b/tests/integration/src/sqlbuild/compiler/contracts/helpers.py index 1ab2e01904..eef1bc48dd 100644 --- a/tests/integration/src/sqlbuild/compiler/contracts/helpers.py +++ b/tests/integration/src/sqlbuild/compiler/contracts/helpers.py @@ -14,11 +14,13 @@ import sqlbuild._native as native_module from sqlbuild._native import ( ColumnContractMode, + CompilerDiagnostic, ContractPolicy, ContractValidationResult, DynamicColumnContractProof, DynamicColumnFamilyProof, IncrementalMode, + InferredColumn, MaterializationType, SchemaColumn, SchemaDynamicColumnFamily, @@ -31,7 +33,6 @@ ) from sqlbuild.adapter.contract.types import TypeDialect from sqlbuild.adapters.duckdb.classes.duckdb_adapter import DuckDbAdapter -from sqlbuild.compiler.compile.models import CompilerDiagnostic, InferredColumn from sqlbuild.compiler.compile.types import CompiledModelView, CompiledProjectView from sqlbuild.compiler.lineage.types import InferredNullability diff --git a/tests/integration/src/sqlbuild/compiler/explicit_references/helpers.py b/tests/integration/src/sqlbuild/compiler/explicit_references/helpers.py index 1c26a35c58..adfeddc89a 100644 --- a/tests/integration/src/sqlbuild/compiler/explicit_references/helpers.py +++ b/tests/integration/src/sqlbuild/compiler/explicit_references/helpers.py @@ -10,12 +10,13 @@ from sqlbuild._native import ( CollectionRendering, CompileAdapterContext, + CompilerDiagnostic, + LiteralSqlRelation, assemble_project, build_compile_inputs, discover_project_inputs, ) from sqlbuild.adapters.duckdb.classes.duckdb_adapter import DuckDbAdapter -from sqlbuild.compiler.compile.models import CompilerDiagnostic, LiteralSqlRelation from sqlbuild.compiler.compile.types import CompiledProjectView _ADAPTER_CONTEXT: CompileAdapterContext = CompileAdapterContext( diff --git a/tests/integration/src/sqlbuild/compiler/pipeline/test_main.py b/tests/integration/src/sqlbuild/compiler/pipeline/test_main.py index ee4de4919f..61d8458840 100644 --- a/tests/integration/src/sqlbuild/compiler/pipeline/test_main.py +++ b/tests/integration/src/sqlbuild/compiler/pipeline/test_main.py @@ -10,6 +10,7 @@ from sqlbuild._native import ( CompilePipelineResult, DiscoveredProjectInputs, + InferredColumn, ModelPlanEntry, RulesError, build_compiled_project, @@ -19,7 +20,6 @@ ) from sqlbuild.adapters.duckdb.classes.duckdb_adapter import DuckDbAdapter from sqlbuild.adapters.snowflake.classes.snowflake_adapter import SnowflakeAdapter -from sqlbuild.compiler.compile.models import InferredColumn from sqlbuild.compiler.compile.types import CompiledProjectView from sqlbuild.compiler.lineage.types import InferredNullability from tests.integration.src.sqlbuild.compiler.pipeline._test_types import ( @@ -313,7 +313,7 @@ def test_given_rule_finding_when_running_shared_compile_pipeline_then_planning_i 'ENUM (name order_status, members (ACTIVE "active", PAUSED "paused"));\n' ), "models/_macros/policy.py": ( - "from sqlbuild.compiler.compile.models import MacroContext\n\n\n" + "from sqlbuild._native import MacroContext\n\n\n" "def policy_columns(ctx: MacroContext) -> str:\n" " minimum = ctx.constants['minimum_quantity']\n" " region_count = len(ctx.constants['regions'])\n" diff --git a/tests/integration/src/sqlbuild/compiler/pipeline/test_native_binding_catalog.py b/tests/integration/src/sqlbuild/compiler/pipeline/test_native_binding_catalog.py index fec1e83cdd..ad43c91fb6 100644 --- a/tests/integration/src/sqlbuild/compiler/pipeline/test_native_binding_catalog.py +++ b/tests/integration/src/sqlbuild/compiler/pipeline/test_native_binding_catalog.py @@ -8,6 +8,8 @@ import sqlbuild._native as sql_analysis_models from sqlbuild._native import ( + CompiledObjectKey, + CompilerDiagnostic, build_compiled_project, discover_project_inputs, source_binding_diagnostics, @@ -15,7 +17,6 @@ from sqlbuild._native import run_cli as main from sqlbuild.adapter.contract.models import ColumnInfo, ExpressionInferenceProfile from sqlbuild.adapters.snowflake.classes.snowflake_adapter import SnowflakeAdapter -from sqlbuild.compiler.compile.models import CompiledObjectKey, CompilerDiagnostic from sqlbuild.compiler.compile.types import CompiledProjectView from tests.integration.src.sqlbuild.compiler.pipeline._test_types import ( NativeCatalogCase, diff --git a/tests/integration/src/sqlbuild/compiler/pipeline/test_semantic_triage.py b/tests/integration/src/sqlbuild/compiler/pipeline/test_semantic_triage.py index f6d98e54f3..298e7d7a45 100644 --- a/tests/integration/src/sqlbuild/compiler/pipeline/test_semantic_triage.py +++ b/tests/integration/src/sqlbuild/compiler/pipeline/test_semantic_triage.py @@ -9,6 +9,7 @@ import sqlbuild._native as native from sqlbuild._native import ( + CompilerDiagnostic, build_compiled_project, discover_project_inputs, source_binding_diagnostics, @@ -16,7 +17,6 @@ from sqlbuild._native import run_cli as main from sqlbuild.adapter.contract.models import ColumnInfo, ExpressionInferenceProfile from sqlbuild.adapters.snowflake.classes.snowflake_adapter import SnowflakeAdapter -from sqlbuild.compiler.compile.models import CompilerDiagnostic from sqlbuild.compiler.compile.types import CompiledProjectView from tests.integration.src.sqlbuild.compiler.pipeline._test_types import ( FunctionArgumentTypeCase, diff --git a/tests/integration/src/sqlbuild/compiler/planner/_helpers/_test_types.py b/tests/integration/src/sqlbuild/compiler/planner/_helpers/_test_types.py index f0976db814..9f9f5861ca 100644 --- a/tests/integration/src/sqlbuild/compiler/planner/_helpers/_test_types.py +++ b/tests/integration/src/sqlbuild/compiler/planner/_helpers/_test_types.py @@ -1,12 +1,13 @@ from dataclasses import dataclass, field from sqlbuild._native import ( + CompiledObjectKey, + CompiledRelationLocation, CursorSnapshotScope, Fingerprint, ModelCursorSnapshot, SourceEntry, ) -from sqlbuild.compiler.compile.models import CompiledObjectKey, CompiledRelationLocation from sqlbuild.cursor_algebra.types import CursorScalar diff --git a/tests/integration/src/sqlbuild/compiler/planner/_helpers/helpers.py b/tests/integration/src/sqlbuild/compiler/planner/_helpers/helpers.py index 8014e137ce..40739c6815 100644 --- a/tests/integration/src/sqlbuild/compiler/planner/_helpers/helpers.py +++ b/tests/integration/src/sqlbuild/compiler/planner/_helpers/helpers.py @@ -10,9 +10,14 @@ from typing import Any, ClassVar from sqlbuild._native import ( + CompiledModel, + CompiledObjectKey, + CompiledProject, + CompiledRelationLocation, CompiledSeed, CompiledSource, CompileModelConfig, + CompileSqlReference, DiscoveredSchemaFile, DiscoveredSeedFile, DiscoveredSourceFile, @@ -29,13 +34,6 @@ from sqlbuild.adapter.contract.models import ColumnInfo from sqlbuild.adapter.relations.main.open_inspection_catalog import open_inspection_catalog from sqlbuild.adapters.duckdb.classes.duckdb_adapter import DuckDbAdapter -from sqlbuild.compiler.compile.models import ( - CompiledModel, - CompiledObjectKey, - CompiledProject, - CompiledRelationLocation, - CompileSqlReference, -) from sqlbuild.compiler.compile.types import ( CompiledModelView, CompiledProjectView, diff --git a/tests/integration/src/sqlbuild/compiler/planner/_helpers/resolve/helpers.py b/tests/integration/src/sqlbuild/compiler/planner/_helpers/resolve/helpers.py index bfccee908b..8a7992d288 100644 --- a/tests/integration/src/sqlbuild/compiler/planner/_helpers/resolve/helpers.py +++ b/tests/integration/src/sqlbuild/compiler/planner/_helpers/resolve/helpers.py @@ -11,7 +11,11 @@ BackfillResult, CollectionRendering, CompileAdapterContext, + CompiledModel, + CompiledObjectKey, + CompiledRelationLocation, CompileModelConfig, + CompileSqlReference, CursorOverridePair, ModelCursorSnapshot, ModelPlanContext, @@ -25,12 +29,6 @@ ) from sqlbuild.adapter.contract.models import ColumnInfo, RelationInfo from sqlbuild.adapters.duckdb.classes.duckdb_adapter import DuckDbAdapter -from sqlbuild.compiler.compile.models import ( - CompiledModel, - CompiledObjectKey, - CompiledRelationLocation, - CompileSqlReference, -) from sqlbuild.compiler.compile.types import ( CompiledModelView, CompiledProjectView, diff --git a/tests/integration/src/sqlbuild/compiler/planner/_helpers/resolve/test_resolve.py b/tests/integration/src/sqlbuild/compiler/planner/_helpers/resolve/test_resolve.py index d85c979d49..b8a6a90c92 100644 --- a/tests/integration/src/sqlbuild/compiler/planner/_helpers/resolve/test_resolve.py +++ b/tests/integration/src/sqlbuild/compiler/planner/_helpers/resolve/test_resolve.py @@ -7,6 +7,7 @@ import pytest from sqlbuild._native import ( + CompiledRelationLocation, CursorType, ModelCursorSnapshot, PlannerInputError, @@ -15,9 +16,6 @@ WarehouseSnapshot, ) from sqlbuild.adapter.contract.models import ColumnInfo -from sqlbuild.compiler.compile.models import ( - CompiledRelationLocation, -) from sqlbuild.compiler.compile.types import CompiledModelView from tests.integration.src.sqlbuild.compiler.planner._helpers.resolve._test_types import ( ResolveAndExecuteTestCase, diff --git a/tests/integration/src/sqlbuild/compiler/planner/_helpers/sql_test_assembly/helpers.py b/tests/integration/src/sqlbuild/compiler/planner/_helpers/sql_test_assembly/helpers.py index 61687bce9d..4a542a445f 100644 --- a/tests/integration/src/sqlbuild/compiler/planner/_helpers/sql_test_assembly/helpers.py +++ b/tests/integration/src/sqlbuild/compiler/planner/_helpers/sql_test_assembly/helpers.py @@ -5,20 +5,18 @@ from pathlib import Path from sqlbuild._native import ( + CompiledModel, CompiledModelSqlTestPayload, + CompiledObjectKey, + CompiledProject, + CompiledRelationLocation, + CompiledSqlTest, CompileModelConfig, CompileSqlTestCte, DiscoveredSqlTestBlock, DiscoveredSqlTestFile, SqlReferenceKind, ) -from sqlbuild.compiler.compile.models import ( - CompiledModel, - CompiledObjectKey, - CompiledProject, - CompiledRelationLocation, - CompiledSqlTest, -) from sqlbuild.compiler.compile.types import ( CompiledModelView, CompiledProjectView, diff --git a/tests/integration/src/sqlbuild/compiler/planner/_helpers/sql_test_assembly/test_execute_chain.py b/tests/integration/src/sqlbuild/compiler/planner/_helpers/sql_test_assembly/test_execute_chain.py index 870d4dbdd2..f174224260 100644 --- a/tests/integration/src/sqlbuild/compiler/planner/_helpers/sql_test_assembly/test_execute_chain.py +++ b/tests/integration/src/sqlbuild/compiler/planner/_helpers/sql_test_assembly/test_execute_chain.py @@ -9,6 +9,9 @@ from sqlbuild._native import ( ChainStep, CompiledDirectLogicSqlTestPayload, + CompiledObjectKey, + CompiledProject, + CompiledSqlTest, CompileSqlTestCte, DiscoveredSqlTestBlock, DiscoveredSqlTestFile, @@ -17,7 +20,6 @@ discover_project_inputs, ) from sqlbuild.adapters.duckdb.classes.duckdb_adapter import DuckDbAdapter -from sqlbuild.compiler.compile.models import CompiledObjectKey, CompiledProject, CompiledSqlTest from sqlbuild.compiler.compile.types import CompiledProjectView, CompiledResourceType, SqlTestMode from sqlbuild.executor.testing.main._execute import execute_sql_test from sqlbuild.executor.testing.models import SqlTestExecutionResult diff --git a/tests/integration/src/sqlbuild/compiler/planner/_helpers/test_audit_entry.py b/tests/integration/src/sqlbuild/compiler/planner/_helpers/test_audit_entry.py index 8c0e94e6a4..6c223c9ada 100644 --- a/tests/integration/src/sqlbuild/compiler/planner/_helpers/test_audit_entry.py +++ b/tests/integration/src/sqlbuild/compiler/planner/_helpers/test_audit_entry.py @@ -7,17 +7,15 @@ from sqlbuild._native import ( AuditPlanEntry, + CompiledAudit, + CompiledObjectKey, + CompiledRelationLocation, DiscoveredAuditBlock, DiscoveredAuditFile, SourceEntry, plan_audit, ) from sqlbuild.adapters.duckdb.classes.duckdb_adapter import DuckDbAdapter -from sqlbuild.compiler.compile.models import ( - CompiledAudit, - CompiledObjectKey, - CompiledRelationLocation, -) from sqlbuild.compiler.compile.types import CompiledAuditView, CompiledResourceType from tests.integration.src.sqlbuild.compiler.planner._helpers._test_types import ( ExecuteAuditTestCase, diff --git a/tests/integration/src/sqlbuild/compiler/planner/_helpers/test_warehouse_cursor_reads.py b/tests/integration/src/sqlbuild/compiler/planner/_helpers/test_warehouse_cursor_reads.py index 35ceca3c1c..475af48f3e 100644 --- a/tests/integration/src/sqlbuild/compiler/planner/_helpers/test_warehouse_cursor_reads.py +++ b/tests/integration/src/sqlbuild/compiler/planner/_helpers/test_warehouse_cursor_reads.py @@ -6,6 +6,7 @@ import pytest from sqlbuild._native import ( + CompiledObjectKey, CursorSnapshotScope, ModelCursorSnapshot, PlannerInputError, @@ -14,7 +15,6 @@ ) from sqlbuild.adapter.relations.main.open_inspection_catalog import open_inspection_catalog from sqlbuild.adapters.duckdb.classes.duckdb_adapter import DuckDbAdapter -from sqlbuild.compiler.compile.models import CompiledObjectKey from sqlbuild.compiler.compile.types import CompiledProjectView, CompiledResourceType from tests.integration.src.sqlbuild.compiler.planner._helpers._test_types import ( ConcurrentCursorReadsTestCase, diff --git a/tests/integration/src/sqlbuild/compiler/planner/_helpers/test_warehouse_snapshot.py b/tests/integration/src/sqlbuild/compiler/planner/_helpers/test_warehouse_snapshot.py index 13acc78eb7..b82f927619 100644 --- a/tests/integration/src/sqlbuild/compiler/planner/_helpers/test_warehouse_snapshot.py +++ b/tests/integration/src/sqlbuild/compiler/planner/_helpers/test_warehouse_snapshot.py @@ -7,6 +7,8 @@ import pytest from sqlbuild._native import ( + CompiledObjectKey, + CompiledRelationLocation, CursorOverrides, CursorSnapshotScope, Fingerprint, @@ -24,10 +26,6 @@ ) from sqlbuild.adapter.contract.models import ColumnInfo from sqlbuild.adapters.duckdb.classes.duckdb_adapter import DuckDbAdapter -from sqlbuild.compiler.compile.models import ( - CompiledObjectKey, - CompiledRelationLocation, -) from sqlbuild.compiler.compile.types import ( CompiledModelView, CompiledProjectView, diff --git a/tests/integration/src/sqlbuild/compiler/planner/main/helpers.py b/tests/integration/src/sqlbuild/compiler/planner/main/helpers.py index 2e22794e93..803a661c0b 100644 --- a/tests/integration/src/sqlbuild/compiler/planner/main/helpers.py +++ b/tests/integration/src/sqlbuild/compiler/planner/main/helpers.py @@ -14,11 +14,16 @@ from sqlbuild._native import ( CompiledFunction, + CompiledModel, + CompiledObjectKey, + CompiledProject, + CompiledRelationLocation, CompiledSeed, CompiledSource, CompileModelConfig, CompilePipelineOptions, CompilePipelineResult, + CompileSqlReference, DeferralInputs, DiscoveredProjectInputs, DiscoveredSchemaFile, @@ -45,13 +50,6 @@ ) from sqlbuild.adapter.contract.models import RelationInfo, RelationReadProbe from sqlbuild.adapters.duckdb.classes.duckdb_adapter import DuckDbAdapter -from sqlbuild.compiler.compile.models import ( - CompiledModel, - CompiledObjectKey, - CompiledProject, - CompiledRelationLocation, - CompileSqlReference, -) from sqlbuild.compiler.compile.types import ( CompiledFunctionView, CompiledModelView, diff --git a/tests/integration/src/sqlbuild/compiler/scopes/helpers.py b/tests/integration/src/sqlbuild/compiler/scopes/helpers.py index 3560a1468d..19d13a53b1 100644 --- a/tests/integration/src/sqlbuild/compiler/scopes/helpers.py +++ b/tests/integration/src/sqlbuild/compiler/scopes/helpers.py @@ -19,12 +19,14 @@ DeclarationIdentity, DeclarationKind, DeclarationRecord, + DeclarationResolutionContext, DeclarationScopeBuild, DeclarationScopeResolver, DeclarationVisibility, DiscoveredProjectInputs, DiscoveryError, EnumDeclaration, + LoadedMacro, ResourceIdentity, ResourceKind, ScopeIndex, @@ -41,7 +43,6 @@ ) from sqlbuild.adapters.duckdb.classes.duckdb_adapter import DuckDbAdapter from sqlbuild.compiler.compile._helpers.render.user_macros import load_project_macros -from sqlbuild.compiler.compile.models import DeclarationResolutionContext, LoadedMacro from tests.integration.src.sqlbuild.compiler.helpers import GoldenEntry, golden_entry CURRENT_PATH_COMPONENT: str = "." diff --git a/tests/integration/src/sqlbuild/compiler/semantic_checks/helpers.py b/tests/integration/src/sqlbuild/compiler/semantic_checks/helpers.py index d236fd3411..54f83d4884 100644 --- a/tests/integration/src/sqlbuild/compiler/semantic_checks/helpers.py +++ b/tests/integration/src/sqlbuild/compiler/semantic_checks/helpers.py @@ -13,10 +13,14 @@ import pytest import sqlbuild._native as native_module -from sqlbuild._native import build_project_graph, discover_project_inputs, project_with +from sqlbuild._native import ( + CompilerDiagnostic, + build_project_graph, + discover_project_inputs, + project_with, +) from sqlbuild.adapter.contract.models import ExpressionInferenceProfile from sqlbuild.adapters.duckdb.classes.duckdb_adapter import DuckDbAdapter -from sqlbuild.compiler.compile.models import CompilerDiagnostic from sqlbuild.compiler.compile.types import CompiledModelView, CompiledProjectView from tests.integration.src.sqlbuild.compiler.analysis_session.helpers import ( generated_analysis_files, diff --git a/tests/integration/src/sqlbuild/compiler/sql_test_glue/helpers.py b/tests/integration/src/sqlbuild/compiler/sql_test_glue/helpers.py index ee3e91c7ba..c835c7305b 100644 --- a/tests/integration/src/sqlbuild/compiler/sql_test_glue/helpers.py +++ b/tests/integration/src/sqlbuild/compiler/sql_test_glue/helpers.py @@ -14,15 +14,13 @@ import sqlbuild._native as native_module from sqlbuild._native import ( + CompiledSqlTest, NativeSqlTestAssembly, NativeSqlTestFailure, build_project_graph, discover_project_inputs, ) from sqlbuild.adapters.duckdb.classes.duckdb_adapter import DuckDbAdapter -from sqlbuild.compiler.compile.models import ( - CompiledSqlTest, -) from sqlbuild.compiler.compile.types import CompiledProjectView from tests.integration.src.sqlbuild.compiler.helpers import GoldenEntry, golden_entry diff --git a/tests/integration/src/sqlbuild/executor/build/concurrent/helpers.py b/tests/integration/src/sqlbuild/executor/build/concurrent/helpers.py index 63bd1a4673..3426afc360 100644 --- a/tests/integration/src/sqlbuild/executor/build/concurrent/helpers.py +++ b/tests/integration/src/sqlbuild/executor/build/concurrent/helpers.py @@ -215,7 +215,7 @@ def extract_upstream_model_deps( ) -> dict[str, frozenset[str]]: """Extract upstream model dependency names for each model in the plan.""" - from sqlbuild.compiler.compile.models import CompiledObjectKey + from sqlbuild._native import CompiledObjectKey from sqlbuild.compiler.compile.types import CompiledResourceType dependencies_by_resource_type: dict[object, dict[str, frozenset[str]]] = { diff --git a/tests/integration/src/sqlbuild/executor/build/custom/helpers.py b/tests/integration/src/sqlbuild/executor/build/custom/helpers.py index 7b6b004f0e..4982244da4 100644 --- a/tests/integration/src/sqlbuild/executor/build/custom/helpers.py +++ b/tests/integration/src/sqlbuild/executor/build/custom/helpers.py @@ -8,6 +8,8 @@ from sqlbuild._native import ( AuditPlanEntry, + CompiledObjectKey, + CompiledRelationLocation, DiscoveredHookFunction, MaterializationType, ModelPlanEntry, @@ -21,10 +23,6 @@ AuditRunScope, AuditSeverity, ) -from sqlbuild.compiler.compile.models import ( - CompiledObjectKey, - CompiledRelationLocation, -) from sqlbuild.compiler.compile.types import CompiledResourceType from sqlbuild.executor.auditing.models import AuditExecutionResult from sqlbuild.executor.custom.models import ( diff --git a/tests/integration/src/sqlbuild/executor/build/custom/test_advanced_custom_materialization.py b/tests/integration/src/sqlbuild/executor/build/custom/test_advanced_custom_materialization.py index 0ab10600c6..0eab5b6f8d 100644 --- a/tests/integration/src/sqlbuild/executor/build/custom/test_advanced_custom_materialization.py +++ b/tests/integration/src/sqlbuild/executor/build/custom/test_advanced_custom_materialization.py @@ -9,11 +9,10 @@ import duckdb import pytest -from sqlbuild._native import AuditPlanEntry, ModelPlanEntry, PlanReason +from sqlbuild._native import AuditPlanEntry, CompiledRelationLocation, ModelPlanEntry, PlanReason from sqlbuild.adapter.contract.models import RelationInfo from sqlbuild.adapter.contract.types import LifeCycleEventKind from sqlbuild.adapters.duckdb.classes.duckdb_adapter import DuckDbAdapter -from sqlbuild.compiler.compile.models import CompiledRelationLocation from sqlbuild.executor.build.models import BuildExecutionResult from sqlbuild.executor.build.types import BuildStatus from sqlbuild.executor.custom.models import MaterializationContext, MaterializationResult diff --git a/tests/integration/src/sqlbuild/executor/build/custom/test_custom_materialization.py b/tests/integration/src/sqlbuild/executor/build/custom/test_custom_materialization.py index c88578b9e0..7879a17227 100644 --- a/tests/integration/src/sqlbuild/executor/build/custom/test_custom_materialization.py +++ b/tests/integration/src/sqlbuild/executor/build/custom/test_custom_materialization.py @@ -11,6 +11,7 @@ from sqlbuild._native import ( AuditPlanEntry, + CompiledRelationLocation, DiscoveredHookFunction, ModelPlanEntry, PlanReason, @@ -19,7 +20,6 @@ ) from sqlbuild.adapters.duckdb.classes.duckdb_adapter import DuckDbAdapter from sqlbuild.compiler.auditing.types import AuditOutcome -from sqlbuild.compiler.compile.models import CompiledRelationLocation from sqlbuild.executor.custom.models import ( MaterializationContext, MaterializationResult, diff --git a/tests/integration/src/sqlbuild/executor/diff/test_query_artifact_cleanup.py b/tests/integration/src/sqlbuild/executor/diff/test_query_artifact_cleanup.py index 707a8d34e2..3f37924d40 100644 --- a/tests/integration/src/sqlbuild/executor/diff/test_query_artifact_cleanup.py +++ b/tests/integration/src/sqlbuild/executor/diff/test_query_artifact_cleanup.py @@ -18,7 +18,6 @@ from sqlbuild.adapter.contract.classes.statement_recorder import StatementRecorder from sqlbuild.adapter.contract.exceptions import AdapterUserError from sqlbuild.adapters.duckdb.classes.duckdb_adapter import DuckDbAdapter -from sqlbuild.compiler.compile.models import CompiledProject from sqlbuild.compiler.compile.types import CompiledProjectView from sqlbuild.compiler.fingerprints.constants import NODE_TYPE_QUERY_DIFF_ARTIFACT from sqlbuild.executor.diff.classes.query_artifact_lifecycle import QueryDiffArtifactLifecycle diff --git a/tests/integration/src/sqlbuild/executor/migrations/_helpers/helpers.py b/tests/integration/src/sqlbuild/executor/migrations/_helpers/helpers.py index 4674c6ce68..2ba1a0d788 100644 --- a/tests/integration/src/sqlbuild/executor/migrations/_helpers/helpers.py +++ b/tests/integration/src/sqlbuild/executor/migrations/_helpers/helpers.py @@ -3,6 +3,7 @@ from __future__ import annotations from sqlbuild._native import ( + CompiledRelationLocation, MigrationCompatibility, MigrationDecision, MigrationDiscovery, @@ -11,7 +12,6 @@ from sqlbuild.adapter.contract.models import MigrationStagePlan from sqlbuild.adapter.contract.types import MigrationTransfer from sqlbuild.adapters.duckdb.classes.duckdb_adapter import DuckDbAdapter -from sqlbuild.compiler.compile.models import CompiledRelationLocation from sqlbuild.executor.migrations.models import MigrationArtifactNames diff --git a/tests/integration/src/sqlbuild/executor/run/helpers.py b/tests/integration/src/sqlbuild/executor/run/helpers.py index c1ce088ee5..58662db772 100644 --- a/tests/integration/src/sqlbuild/executor/run/helpers.py +++ b/tests/integration/src/sqlbuild/executor/run/helpers.py @@ -9,6 +9,8 @@ from sqlbuild._native import ( AuditPlanEntry, + CompiledObjectKey, + CompiledRelationLocation, DiscoveredHookFunction, MaterializationType, ModelPlanEntry, @@ -22,10 +24,6 @@ AuditRunScope, AuditSeverity, ) -from sqlbuild.compiler.compile.models import ( - CompiledObjectKey, - CompiledRelationLocation, -) from sqlbuild.compiler.compile.types import CompiledResourceType from sqlbuild.executor.run.main._execute import execute_table_entry from sqlbuild.executor.run.models import ( diff --git a/tests/integration/src/sqlbuild/executor/run/incremental/helpers.py b/tests/integration/src/sqlbuild/executor/run/incremental/helpers.py index 8c1d50fae5..4a0f56e9b0 100644 --- a/tests/integration/src/sqlbuild/executor/run/incremental/helpers.py +++ b/tests/integration/src/sqlbuild/executor/run/incremental/helpers.py @@ -9,6 +9,8 @@ from sqlbuild._native import ( AuditPlanEntry, + CompiledObjectKey, + CompiledRelationLocation, CursorBounds, CursorInputRelation, DiscoveredHookFunction, @@ -26,10 +28,6 @@ AuditRunScope, AuditSeverity, ) -from sqlbuild.compiler.compile.models import ( - CompiledObjectKey, - CompiledRelationLocation, -) from sqlbuild.compiler.compile.types import CompiledResourceType from sqlbuild.executor.run._helpers.materializations.incremental import execute_incremental_entry from sqlbuild.executor.run.models import ( diff --git a/tests/integration/src/sqlbuild/executor/run/microbatch/helpers.py b/tests/integration/src/sqlbuild/executor/run/microbatch/helpers.py index 067260ee38..20064a7020 100644 --- a/tests/integration/src/sqlbuild/executor/run/microbatch/helpers.py +++ b/tests/integration/src/sqlbuild/executor/run/microbatch/helpers.py @@ -9,6 +9,8 @@ from sqlbuild._native import ( AuditPlanEntry, + CompiledObjectKey, + CompiledRelationLocation, CursorBounds, CursorInputRelation, DiscoveredHookFunction, @@ -24,10 +26,6 @@ AuditAttachmentKind, AuditSeverity, ) -from sqlbuild.compiler.compile.models import ( - CompiledObjectKey, - CompiledRelationLocation, -) from sqlbuild.compiler.compile.types import CompiledResourceType from sqlbuild.compiler.planner.constants import ( MICROBATCH_END_SENTINEL, diff --git a/tests/integration/src/sqlbuild/executor/run/view/helpers.py b/tests/integration/src/sqlbuild/executor/run/view/helpers.py index 703de1354c..25b4a0e41c 100644 --- a/tests/integration/src/sqlbuild/executor/run/view/helpers.py +++ b/tests/integration/src/sqlbuild/executor/run/view/helpers.py @@ -8,6 +8,8 @@ from sqlbuild._native import ( AuditPlanEntry, + CompiledObjectKey, + CompiledRelationLocation, DiscoveredHookFunction, MaterializationType, ModelPlanEntry, @@ -20,10 +22,6 @@ AuditRunScope, AuditSeverity, ) -from sqlbuild.compiler.compile.models import ( - CompiledObjectKey, - CompiledRelationLocation, -) from sqlbuild.compiler.compile.types import CompiledResourceType from sqlbuild.executor.run._helpers.materializations.view import execute_view_entry from sqlbuild.executor.run.models import ( diff --git a/tests/integration/src/sqlbuild/executor/scenario/helpers.py b/tests/integration/src/sqlbuild/executor/scenario/helpers.py index 3e8aae2e24..d8e679f0c1 100644 --- a/tests/integration/src/sqlbuild/executor/scenario/helpers.py +++ b/tests/integration/src/sqlbuild/executor/scenario/helpers.py @@ -4,6 +4,8 @@ from typing import Any from sqlbuild._native import ( + CompiledObjectKey, + CompiledRelationLocation, DiscoveredHookFunction, MaterializationType, ModelPlanEntry, @@ -18,10 +20,6 @@ ScenarioRelationMap, ScenarioRelationPlan, ) -from sqlbuild.compiler.compile.models import ( - CompiledObjectKey, - CompiledRelationLocation, -) from sqlbuild.compiler.compile.types import CompiledResourceType from sqlbuild.executor.run.models import HookContext diff --git a/tests/integration/src/sqlbuild/executor/scenario/test_capture_types.py b/tests/integration/src/sqlbuild/executor/scenario/test_capture_types.py index 0ef295f0ca..fd7dd93469 100644 --- a/tests/integration/src/sqlbuild/executor/scenario/test_capture_types.py +++ b/tests/integration/src/sqlbuild/executor/scenario/test_capture_types.py @@ -9,9 +9,8 @@ import duckdb import pytest -from sqlbuild._native import ScenarioArtifactKind +from sqlbuild._native import CompiledRelationLocation, ScenarioArtifactKind from sqlbuild.adapters.duckdb.classes.duckdb_adapter import DuckDbAdapter -from sqlbuild.compiler.compile.models import CompiledRelationLocation from sqlbuild.executor.scenario._helpers.snapshots.core import read_scenario_snapshot_manifest from sqlbuild.executor.scenario.main._capture import execute_scenario_snapshot_capture from sqlbuild.executor.scenario.models import ( diff --git a/tests/integration/src/sqlbuild/executor/scenario/test_fixture_lifecycle.py b/tests/integration/src/sqlbuild/executor/scenario/test_fixture_lifecycle.py index 07fe247849..1c5a05a9fc 100644 --- a/tests/integration/src/sqlbuild/executor/scenario/test_fixture_lifecycle.py +++ b/tests/integration/src/sqlbuild/executor/scenario/test_fixture_lifecycle.py @@ -6,6 +6,8 @@ import pytest from sqlbuild._native import ( + CompiledObjectKey, + CompiledRelationLocation, DiscoveredHookFunction, PythonHookEntry, ScenarioExecutionPlan, @@ -15,10 +17,6 @@ from sqlbuild.adapter.contract.models import ColumnInfo from sqlbuild.adapter.contract.types import TablePromotionMode from sqlbuild.adapters.duckdb.classes.duckdb_adapter import DuckDbAdapter -from sqlbuild.compiler.compile.models import ( - CompiledObjectKey, - CompiledRelationLocation, -) from sqlbuild.compiler.compile.types import CompiledResourceType from sqlbuild.executor.build.models import SeedExecutionResult from sqlbuild.executor.run.models import ModelExecutionResult diff --git a/tests/integration/src/sqlbuild/executor/testing/helpers.py b/tests/integration/src/sqlbuild/executor/testing/helpers.py index 86553af302..6fcc3ecd2c 100644 --- a/tests/integration/src/sqlbuild/executor/testing/helpers.py +++ b/tests/integration/src/sqlbuild/executor/testing/helpers.py @@ -8,12 +8,12 @@ from sqlbuild._native import ( ChainStep, + CompiledObjectKey, CompilePipelineResult, SqlTestPlanEntry, render_sql_test_comparison_sql, ) from sqlbuild.adapters.duckdb.classes.duckdb_adapter import DuckDbAdapter -from sqlbuild.compiler.compile.models import CompiledObjectKey from sqlbuild.compiler.compile.types import CompiledResourceType from sqlbuild.executor.testing.main._execute import execute_sql_test from sqlbuild.executor.testing.models import SqlTestExecutionResult diff --git a/tests/integration/src/sqlbuild/rule_engine/host_payload/helpers.py b/tests/integration/src/sqlbuild/rule_engine/host_payload/helpers.py index 8419916287..193acaf6ff 100644 --- a/tests/integration/src/sqlbuild/rule_engine/host_payload/helpers.py +++ b/tests/integration/src/sqlbuild/rule_engine/host_payload/helpers.py @@ -6,10 +6,14 @@ import pytest -import sqlbuild.compiler.compile.models as compile_models -from sqlbuild._native import build_compiled_project, discover_project_inputs +import sqlbuild._native as compile_models +from sqlbuild._native import ( + CompiledModel, + CompiledProject, + build_compiled_project, + discover_project_inputs, +) from sqlbuild.adapters.duckdb.classes.duckdb_adapter import DuckDbAdapter -from sqlbuild.compiler.compile.models import CompiledModel, CompiledProject from sqlbuild.compiler.compile.types import CompiledProjectView from sqlbuild.rule_engine._helpers.engine import fact_replay from sqlbuild.rule_engine._helpers.engine.fact_replay import ( diff --git a/tests/integration/src/sqlbuild/rule_engine/host_payload/test_host_payload_facts.py b/tests/integration/src/sqlbuild/rule_engine/host_payload/test_host_payload_facts.py index 5088abeffd..de14299b49 100644 --- a/tests/integration/src/sqlbuild/rule_engine/host_payload/test_host_payload_facts.py +++ b/tests/integration/src/sqlbuild/rule_engine/host_payload/test_host_payload_facts.py @@ -5,10 +5,11 @@ import pytest from sqlbuild._native import ( + CompiledModel, + CompiledProject, compiled_project_record, custom_host_payload, ) -from sqlbuild.compiler.compile.models import CompiledModel, CompiledProject from sqlbuild.compiler.compile.types import CompiledProjectView from sqlbuild.rule_engine.classes.project_tree import public_model from sqlbuild.rule_engine.types import FactKey diff --git a/tests/unit/scripts/compiler_differential/_helpers/coverage/helpers.py b/tests/unit/scripts/compiler_differential/_helpers/coverage/helpers.py index 62cb3cd70c..6507e94d9c 100644 --- a/tests/unit/scripts/compiler_differential/_helpers/coverage/helpers.py +++ b/tests/unit/scripts/compiler_differential/_helpers/coverage/helpers.py @@ -8,10 +8,10 @@ from sqlbuild._native import ( STAGE_CAPTURE_OMITTED_ATTRIBUTES, + CompiledProject, CompileProjectInputs, DiscoveredProjectInputs, ) -from sqlbuild.compiler.compile.models import CompiledProject _VALUE_KIND: str = "sqlbuild.sql_values.types:SqlValueKind" diff --git a/tests/unit/src/sqlbuild/cli/commands/_helpers/build_planning/helpers.py b/tests/unit/src/sqlbuild/cli/commands/_helpers/build_planning/helpers.py index e20a2f17b8..e8940a0343 100644 --- a/tests/unit/src/sqlbuild/cli/commands/_helpers/build_planning/helpers.py +++ b/tests/unit/src/sqlbuild/cli/commands/_helpers/build_planning/helpers.py @@ -3,6 +3,8 @@ from pathlib import Path from sqlbuild._native import ( + CompiledObjectKey, + CompiledRelationLocation, MaterializationType, ModelPlanEntry, PlanAction, @@ -15,7 +17,6 @@ ) from sqlbuild.adapter.contract.models import RetentionRequest from sqlbuild.adapter.contract.types import RetentionScope -from sqlbuild.compiler.compile.models import CompiledObjectKey, CompiledRelationLocation from sqlbuild.compiler.compile.types import CompiledResourceType diff --git a/tests/unit/src/sqlbuild/cli/commands/_helpers/check/test_planning.py b/tests/unit/src/sqlbuild/cli/commands/_helpers/check/test_planning.py index 08a3765a6f..0e9b24c3ef 100644 --- a/tests/unit/src/sqlbuild/cli/commands/_helpers/check/test_planning.py +++ b/tests/unit/src/sqlbuild/cli/commands/_helpers/check/test_planning.py @@ -8,6 +8,8 @@ from sqlbuild._native import ( CliUserError, + CompiledProject, + CompiledRelationLocation, CompilePipelineResult, PlanOutput, PythonSqlRunLifecyclePlan, @@ -16,7 +18,6 @@ ) from sqlbuild.adapter.contract.models import RelationLookup from sqlbuild.adapter.relations.main import relation_lookup as relation_lookup_module -from sqlbuild.compiler.compile.models import CompiledProject, CompiledRelationLocation from sqlbuild.executor.python_nodes.main import read_side as read_side_module from sqlbuild.executor.python_nodes.models import PythonNodeExecutionResult from sqlbuild.python_nodes.models import SqlResourceRef diff --git a/tests/unit/src/sqlbuild/cli/commands/_helpers/helpers.py b/tests/unit/src/sqlbuild/cli/commands/_helpers/helpers.py index 341dd52cfc..5c431c8e83 100644 --- a/tests/unit/src/sqlbuild/cli/commands/_helpers/helpers.py +++ b/tests/unit/src/sqlbuild/cli/commands/_helpers/helpers.py @@ -5,6 +5,8 @@ from sqlbuild._native import ( ChainStep, + CompiledObjectKey, + CompiledRelationLocation, CostContext, MaterializationType, ModelPlanEntry, @@ -17,10 +19,6 @@ ) from sqlbuild.adapter.contract.types import FrameworkType from sqlbuild.adapters.duckdb.classes.duckdb_adapter import DuckDbAdapter -from sqlbuild.compiler.compile.models import ( - CompiledObjectKey, - CompiledRelationLocation, -) from sqlbuild.compiler.compile.types import CompiledResourceType from sqlbuild.cost.models import CostResourceContext diff --git a/tests/unit/src/sqlbuild/cli/commands/main/build/_test_types.py b/tests/unit/src/sqlbuild/cli/commands/main/build/_test_types.py index 5f7f39c3e8..bfd5c1aa64 100644 --- a/tests/unit/src/sqlbuild/cli/commands/main/build/_test_types.py +++ b/tests/unit/src/sqlbuild/cli/commands/main/build/_test_types.py @@ -2,7 +2,7 @@ from dataclasses import dataclass -from sqlbuild.compiler.compile.models import CompiledObjectKey +from sqlbuild._native import CompiledObjectKey @dataclass(frozen=True) diff --git a/tests/unit/src/sqlbuild/cli/commands/main/build/helpers.py b/tests/unit/src/sqlbuild/cli/commands/main/build/helpers.py index 458f05fd04..1f72c96b35 100644 --- a/tests/unit/src/sqlbuild/cli/commands/main/build/helpers.py +++ b/tests/unit/src/sqlbuild/cli/commands/main/build/helpers.py @@ -1,6 +1,6 @@ from __future__ import annotations -from sqlbuild.compiler.compile.models import CompiledObjectKey +from sqlbuild._native import CompiledObjectKey from sqlbuild.compiler.compile.types import CompiledResourceType diff --git a/tests/unit/src/sqlbuild/cli/commands/main/build/test_defer_clone.py b/tests/unit/src/sqlbuild/cli/commands/main/build/test_defer_clone.py index 1ef2d91a94..3b87f9458a 100644 --- a/tests/unit/src/sqlbuild/cli/commands/main/build/test_defer_clone.py +++ b/tests/unit/src/sqlbuild/cli/commands/main/build/test_defer_clone.py @@ -4,11 +4,13 @@ import pytest -from sqlbuild._native import CompileModelConfig, PlannerScope, resolve_defer_clone_selection -from sqlbuild.compiler.compile.models import ( +from sqlbuild._native import ( CompiledModel, CompiledObjectKey, CompiledRelationLocation, + CompileModelConfig, + PlannerScope, + resolve_defer_clone_selection, ) from sqlbuild.compiler.compile.types import CompiledModelView, CompiledResourceType from tests.unit.src.sqlbuild.cli.commands.main.build._test_types import ( diff --git a/tests/unit/src/sqlbuild/cli/commands/main/compile/helpers.py b/tests/unit/src/sqlbuild/cli/commands/main/compile/helpers.py index 8b0b01301f..e0e8a179d9 100644 --- a/tests/unit/src/sqlbuild/cli/commands/main/compile/helpers.py +++ b/tests/unit/src/sqlbuild/cli/commands/main/compile/helpers.py @@ -9,6 +9,10 @@ AuditPlanEntry, ChainStep, CompiledFunction, + CompiledModel, + CompiledObjectKey, + CompiledProject, + CompiledRelationLocation, CompileModelConfig, FunctionArgument, FunctionPlanEntry, @@ -33,12 +37,6 @@ AuditRunScope, AuditSeverity, ) -from sqlbuild.compiler.compile.models import ( - CompiledModel, - CompiledObjectKey, - CompiledProject, - CompiledRelationLocation, -) from sqlbuild.compiler.compile.types import ( CompiledModelView, CompiledProjectView, diff --git a/tests/unit/src/sqlbuild/cli/commands/main/compile/test_compile_output.py b/tests/unit/src/sqlbuild/cli/commands/main/compile/test_compile_output.py index b5447a3436..c25a082aee 100644 --- a/tests/unit/src/sqlbuild/cli/commands/main/compile/test_compile_output.py +++ b/tests/unit/src/sqlbuild/cli/commands/main/compile/test_compile_output.py @@ -7,6 +7,7 @@ import pytest from sqlbuild._native import ( + CompiledObjectKey, CompileLineageMode, CompileModelConfig, ProjectColumnLineage, @@ -18,7 +19,6 @@ format_compile_text, project_with, ) -from sqlbuild.compiler.compile.models import CompiledObjectKey from sqlbuild.compiler.compile.types import CompiledModelView, CompiledResourceType from tests.unit.src.sqlbuild.cli.commands.main.compile._test_types import ( CompileJsonExecutionLayersTestCase, diff --git a/tests/unit/src/sqlbuild/cli/commands/main/compile/test_compile_target_writer.py b/tests/unit/src/sqlbuild/cli/commands/main/compile/test_compile_target_writer.py index 7a0da29847..908f900469 100644 --- a/tests/unit/src/sqlbuild/cli/commands/main/compile/test_compile_target_writer.py +++ b/tests/unit/src/sqlbuild/cli/commands/main/compile/test_compile_target_writer.py @@ -12,6 +12,7 @@ import sqlbuild._native as plan_artifacts_module from sqlbuild._native import ( ChainStep, + CompiledSqlTest, DiagnosticPhase, PlanOutput, SqlTestPlanEntry, @@ -24,7 +25,6 @@ write_static_compile_target, ) from sqlbuild.adapters.duckdb.classes.duckdb_adapter import DuckDbAdapter -from sqlbuild.compiler.compile.models import CompiledSqlTest from sqlbuild.compiler.compile.types import ( CompiledModelView, CompiledProjectView, diff --git a/tests/unit/src/sqlbuild/cli/commands/main/execution/helpers.py b/tests/unit/src/sqlbuild/cli/commands/main/execution/helpers.py index 5e8e51b653..18a48bbc06 100644 --- a/tests/unit/src/sqlbuild/cli/commands/main/execution/helpers.py +++ b/tests/unit/src/sqlbuild/cli/commands/main/execution/helpers.py @@ -11,13 +11,14 @@ from sqlbuild import _native as native from sqlbuild._native import ( + CompiledObjectKey, + CompiledSqlScenario, DiscoveredProjectInputs, DiscoveredSqlScenarioFile, LocalConfig, ProjectConfig, ) from sqlbuild.adapter.discovery.main import resolve_adapter as adapters_module -from sqlbuild.compiler.compile.models import CompiledObjectKey, CompiledSqlScenario from sqlbuild.compiler.compile.types import CompiledResourceType from sqlbuild.diagnostics.main import process_resource_reporting as resource_reporting_module from sqlbuild.executor.pipeline.main import run as executor_run_module diff --git a/tests/unit/src/sqlbuild/cli/commands/shared/_helpers/helpers.py b/tests/unit/src/sqlbuild/cli/commands/shared/_helpers/helpers.py index 61da8acb40..b9554f7905 100644 --- a/tests/unit/src/sqlbuild/cli/commands/shared/_helpers/helpers.py +++ b/tests/unit/src/sqlbuild/cli/commands/shared/_helpers/helpers.py @@ -7,6 +7,8 @@ from sqlbuild._native import ( ChainStep, + CompiledObjectKey, + CompiledRelationLocation, MaterializationType, ModelPlanEntry, PlanAction, @@ -21,7 +23,6 @@ AuditRunScope, AuditSeverity, ) -from sqlbuild.compiler.compile.models import CompiledObjectKey, CompiledRelationLocation from sqlbuild.compiler.compile.types import CompiledResourceType from sqlbuild.executor.auditing.models import AuditExecutionResult from sqlbuild.executor.testing.models import SqlTestExecutionResult, StepResult diff --git a/tests/unit/src/sqlbuild/cli/compile_reuse/main/test_attempt_import_footprint.py b/tests/unit/src/sqlbuild/cli/compile_reuse/main/test_attempt_import_footprint.py index 24853fe94b..65d94956f7 100644 --- a/tests/unit/src/sqlbuild/cli/compile_reuse/main/test_attempt_import_footprint.py +++ b/tests/unit/src/sqlbuild/cli/compile_reuse/main/test_attempt_import_footprint.py @@ -14,7 +14,7 @@ ) from tests.unit.src.sqlbuild.cli.compile_reuse.main.helpers import compile_in_fresh_process -_FULL_COMPILE_MODULES: tuple[str, ...] = ("sqlbuild.compiler.compile.models",) +_FULL_COMPILE_MODULES: tuple[str, ...] = ("sqlbuild.compiler.compile.types",) @pytest.mark.parametrize( diff --git a/tests/unit/src/sqlbuild/cli/output/main/plan/helpers.py b/tests/unit/src/sqlbuild/cli/output/main/plan/helpers.py index 4cf7b16f6d..de19758d14 100644 --- a/tests/unit/src/sqlbuild/cli/output/main/plan/helpers.py +++ b/tests/unit/src/sqlbuild/cli/output/main/plan/helpers.py @@ -8,6 +8,8 @@ BackfillAction, BackfillResult, ColumnRenameHint, + CompiledObjectKey, + CompiledRelationLocation, CursorBounds, CursorInputRelation, DiscoveredProviderUsage, @@ -30,10 +32,6 @@ TableTypePlanEntry, WarningSeverity, ) -from sqlbuild.compiler.compile.models import ( - CompiledObjectKey, - CompiledRelationLocation, -) from sqlbuild.compiler.compile.types import CompiledResourceType, FunctionLanguage diff --git a/tests/unit/src/sqlbuild/cli/progress/classes/test_build_sql_test_projection.py b/tests/unit/src/sqlbuild/cli/progress/classes/test_build_sql_test_projection.py index 69887593a2..ba2277d229 100644 --- a/tests/unit/src/sqlbuild/cli/progress/classes/test_build_sql_test_projection.py +++ b/tests/unit/src/sqlbuild/cli/progress/classes/test_build_sql_test_projection.py @@ -10,13 +10,13 @@ from sqlbuild._native import ( BuildProgressCallbacks, ChainStep, + CompiledObjectKey, NativeProgressProjector, PlanOutput, SqlTestPlanEntry, TerminalEventIndex, terminal_event_index_scope, ) -from sqlbuild.compiler.compile.models import CompiledObjectKey from sqlbuild.compiler.compile.types import CompiledResourceType from sqlbuild.executor.run.models import ModelExecutionResult from sqlbuild.executor.scheduling.types import ExecutionStatus diff --git a/tests/unit/src/sqlbuild/compiler/auditing/main/helpers.py b/tests/unit/src/sqlbuild/compiler/auditing/main/helpers.py index 2be628ed6d..5d7f7b34ca 100644 --- a/tests/unit/src/sqlbuild/compiler/auditing/main/helpers.py +++ b/tests/unit/src/sqlbuild/compiler/auditing/main/helpers.py @@ -7,7 +7,14 @@ from types import MappingProxyType from typing import cast -from sqlbuild._native import AuditPlanEntry, SchemaAuditInstance, SourceEntry, parse_audit_instances +from sqlbuild._native import ( + AuditPlanEntry, + CompiledObjectKey, + CompiledRelationLocation, + SchemaAuditInstance, + SourceEntry, + parse_audit_instances, +) from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter from sqlbuild.adapter.discovery.main.builtins import builtin_adapter_classes from sqlbuild.compiler.auditing.types import ( @@ -15,7 +22,6 @@ AuditRunScope, AuditSeverity, ) -from sqlbuild.compiler.compile.models import CompiledObjectKey, CompiledRelationLocation from sqlbuild.compiler.compile.types import CompiledResourceType diff --git a/tests/unit/src/sqlbuild/compiler/auditing/main/test_rendering.py b/tests/unit/src/sqlbuild/compiler/auditing/main/test_rendering.py index d155968bc5..5abc5566aa 100644 --- a/tests/unit/src/sqlbuild/compiler/auditing/main/test_rendering.py +++ b/tests/unit/src/sqlbuild/compiler/auditing/main/test_rendering.py @@ -4,9 +4,8 @@ import pytest -from sqlbuild._native import SourceEntry, render_audit_sql +from sqlbuild._native import CompiledRelationLocation, SourceEntry, render_audit_sql from sqlbuild.adapter.contract.types import BuiltinAdapter -from sqlbuild.compiler.compile.models import CompiledRelationLocation from tests.unit.src.sqlbuild.compiler.auditing.main._test_types import ( RenderAuditSqlTestCase, ) diff --git a/tests/unit/src/sqlbuild/compiler/compile/_helpers/_test_types.py b/tests/unit/src/sqlbuild/compiler/compile/_helpers/_test_types.py index d6b6f8ccd0..d3d7488e73 100644 --- a/tests/unit/src/sqlbuild/compiler/compile/_helpers/_test_types.py +++ b/tests/unit/src/sqlbuild/compiler/compile/_helpers/_test_types.py @@ -7,7 +7,10 @@ from sqlbuild._native import ( CompiledLineageColumnFact, + CompiledObjectKey, + CompileSqlReference, ExpansionSpan, + InferredColumn, ResolvedTimeTravelRetention, SourceLocation, SqlLexicalSyntax, @@ -16,7 +19,6 @@ UsageRecord, ) from sqlbuild.adapter.contract.models import ExpressionInferenceProfile -from sqlbuild.compiler.compile.models import CompiledObjectKey, CompileSqlReference, InferredColumn from sqlbuild.compiler.compile.types import AttachedAuditTargetKind from sqlbuild.compiler.lineage.types import ( InferredNullability, diff --git a/tests/unit/src/sqlbuild/compiler/compile/_helpers/helpers.py b/tests/unit/src/sqlbuild/compiler/compile/_helpers/helpers.py index 4807eb8696..65690b902f 100644 --- a/tests/unit/src/sqlbuild/compiler/compile/_helpers/helpers.py +++ b/tests/unit/src/sqlbuild/compiler/compile/_helpers/helpers.py @@ -14,15 +14,24 @@ CompileAdapterContext, CompiledDirectLogicSqlTestPayload, CompiledLineageColumnFact, + CompiledLineageSourceFact, CompiledModelSqlTestPayload, + CompiledObjectKey, + CompiledRelationLocation, + CompiledSqlTest, + CompilerDiagnostic, + CompileSqlReference, DeclarationExpansionContext, DeclarationIdentity, DeclarationKind, DeclarationRecord, + DeclarationResolutionContext, DeclarationRuntimeProjection, DeclarationScopeResolver, DiscoveredMacroFile, DiscoveredProjectInputs, + LoadedMacro, + MacroContext, MacroMetadata, NativeColumnQuery, NativeQueryColumns, @@ -54,17 +63,6 @@ from sqlbuild.adapters.snowflake.classes.snowflake_adapter import SnowflakeAdapter from sqlbuild.adapters.sqlserver.classes.sqlserver_adapter import SqlServerAdapter from sqlbuild.compiler.compile._helpers.render.user_macros import load_project_macros -from sqlbuild.compiler.compile.models import ( - CompiledLineageSourceFact, - CompiledObjectKey, - CompiledRelationLocation, - CompiledSqlTest, - CompilerDiagnostic, - CompileSqlReference, - DeclarationResolutionContext, - LoadedMacro, - MacroContext, -) from sqlbuild.compiler.compile.types import ( CompiledModelView, CompiledProjectView, diff --git a/tests/unit/src/sqlbuild/compiler/compile/_helpers/render/test_cursor_intrinsics.py b/tests/unit/src/sqlbuild/compiler/compile/_helpers/render/test_cursor_intrinsics.py index c1264614e0..0f38530fe4 100644 --- a/tests/unit/src/sqlbuild/compiler/compile/_helpers/render/test_cursor_intrinsics.py +++ b/tests/unit/src/sqlbuild/compiler/compile/_helpers/render/test_cursor_intrinsics.py @@ -4,6 +4,7 @@ from sqlbuild._native import ( CompileInputError, + InferredColumn, NativeColumnQuery, cursor_intrinsics_analysis_sql, infer_native_query_columns, @@ -11,7 +12,6 @@ validated_model_cursor_intrinsics, ) from sqlbuild.adapter.contract.models import ExpressionInferenceProfile -from sqlbuild.compiler.compile.models import InferredColumn from sqlbuild.compiler.lineage.types import InferredNullability from tests.unit.src.sqlbuild.compiler.compile._helpers.render._test_types import ( CursorIntrinsicAnalysisTestCase, diff --git a/tests/unit/src/sqlbuild/compiler/compile/_helpers/test_assembly.py b/tests/unit/src/sqlbuild/compiler/compile/_helpers/test_assembly.py index 9d06c7448f..cf62a2121f 100644 --- a/tests/unit/src/sqlbuild/compiler/compile/_helpers/test_assembly.py +++ b/tests/unit/src/sqlbuild/compiler/compile/_helpers/test_assembly.py @@ -7,6 +7,7 @@ import pytest from sqlbuild._native import ( + CompiledObjectKey, DiscoveredProjectInputs, SqlHookEntry, assemble_project, @@ -14,9 +15,6 @@ discover_project_inputs, ) from sqlbuild.adapter.contract.models import ExpressionInferenceProfile -from sqlbuild.compiler.compile.models import ( - CompiledObjectKey, -) from sqlbuild.compiler.compile.types import ( AttachedAuditTargetKind, CompiledProjectView, diff --git a/tests/unit/src/sqlbuild/compiler/compile/_helpers/test_expansion_spans.py b/tests/unit/src/sqlbuild/compiler/compile/_helpers/test_expansion_spans.py index 33ee290dfc..6f2bd96993 100644 --- a/tests/unit/src/sqlbuild/compiler/compile/_helpers/test_expansion_spans.py +++ b/tests/unit/src/sqlbuild/compiler/compile/_helpers/test_expansion_spans.py @@ -8,11 +8,12 @@ from sqlbuild._native import ( ExpansionSpan, + LoadedMacro, + MacroContext, MappedOffset, expand_authored_sql_with_spans, map_expanded_offset, ) -from sqlbuild.compiler.compile.models import LoadedMacro, MacroContext from tests.unit.src.sqlbuild.compiler.compile._helpers._test_types import ExpandWithSpansTestCase from tests.unit.src.sqlbuild.compiler.compile._helpers.helpers import ( DUCKDB_COMPILE_ADAPTER_CONTEXT, diff --git a/tests/unit/src/sqlbuild/compiler/compile/_helpers/test_macros.py b/tests/unit/src/sqlbuild/compiler/compile/_helpers/test_macros.py index f8032528dd..e9e012828f 100644 --- a/tests/unit/src/sqlbuild/compiler/compile/_helpers/test_macros.py +++ b/tests/unit/src/sqlbuild/compiler/compile/_helpers/test_macros.py @@ -7,16 +7,14 @@ from sqlbuild._native import ( DiscoveredMacroFile, + LoadedMacro, + MacroContext, ScopeKind, expand_sql_macros, find_macro_call_names, ) from sqlbuild.compiler.compile._helpers.render import user_macros as macros_module from sqlbuild.compiler.compile._helpers.render.user_macros import load_project_macros -from sqlbuild.compiler.compile.models import ( - LoadedMacro, - MacroContext, -) from tests.unit.src.sqlbuild.compiler.compile._helpers._test_types import ( ExpandSqlMacrosErrorTestCase, ExpandSqlMacrosTestCase, diff --git a/tests/unit/src/sqlbuild/compiler/compile/_helpers/test_model_schemas.py b/tests/unit/src/sqlbuild/compiler/compile/_helpers/test_model_schemas.py index 3b1a67f4a6..f0e3257417 100644 --- a/tests/unit/src/sqlbuild/compiler/compile/_helpers/test_model_schemas.py +++ b/tests/unit/src/sqlbuild/compiler/compile/_helpers/test_model_schemas.py @@ -9,6 +9,7 @@ from sqlbuild._native import ( CompileInputError, + CompilerDiagnostic, DiscoveredProjectInputs, SchemaAuditInstance, SchemaModelEntry, @@ -20,9 +21,6 @@ evaluate_model_contracts, ) from sqlbuild.adapters.duckdb.classes.duckdb_adapter import DuckDbAdapter -from sqlbuild.compiler.compile.models import ( - CompilerDiagnostic, -) from sqlbuild.compiler.compile.types import CompiledModelView, CompiledProjectView from tests.unit.src.sqlbuild.compiler.compile._helpers._test_types import ( DynamicModelSchemaTestCase, diff --git a/tests/unit/src/sqlbuild/compiler/compile/_helpers/test_parameterized_sql_tests.py b/tests/unit/src/sqlbuild/compiler/compile/_helpers/test_parameterized_sql_tests.py index 695580993a..719fc001eb 100644 --- a/tests/unit/src/sqlbuild/compiler/compile/_helpers/test_parameterized_sql_tests.py +++ b/tests/unit/src/sqlbuild/compiler/compile/_helpers/test_parameterized_sql_tests.py @@ -8,6 +8,8 @@ import pytest from sqlbuild._native import ( + CompiledObjectKey, + CompiledSqlTest, CompileInputError, SqlLogicalType, SqlValue, @@ -17,10 +19,6 @@ ) from sqlbuild.adapters.bigquery.classes.bigquery_adapter import BigQueryAdapter from sqlbuild.adapters.sqlserver.classes.sqlserver_adapter import SqlServerAdapter -from sqlbuild.compiler.compile.models import ( - CompiledObjectKey, - CompiledSqlTest, -) from sqlbuild.compiler.compile.types import ( CompiledProjectView, CompiledResourceType, diff --git a/tests/unit/src/sqlbuild/compiler/compile/_helpers/test_rebind_declaration_scope.py b/tests/unit/src/sqlbuild/compiler/compile/_helpers/test_rebind_declaration_scope.py index 2f18c6002c..edf999d9d1 100644 --- a/tests/unit/src/sqlbuild/compiler/compile/_helpers/test_rebind_declaration_scope.py +++ b/tests/unit/src/sqlbuild/compiler/compile/_helpers/test_rebind_declaration_scope.py @@ -12,13 +12,13 @@ DeclarationKind, DeclarationScopeBuild, DiscoveredProjectInputs, + LoadedMacro, SqlLexicalSyntax, build_declaration_scope, discover_project_inputs, rebind_declaration_scope, ) from sqlbuild.compiler.compile._helpers.render.user_macros import load_project_macros -from sqlbuild.compiler.compile.models import LoadedMacro from tests.unit.src.sqlbuild.compiler.compile._helpers._test_types import ( RebindDeclarationScopeTestCase, ) diff --git a/tests/unit/src/sqlbuild/compiler/compile/_helpers/test_required_columns.py b/tests/unit/src/sqlbuild/compiler/compile/_helpers/test_required_columns.py index f782666a02..7c3ffd22c2 100644 --- a/tests/unit/src/sqlbuild/compiler/compile/_helpers/test_required_columns.py +++ b/tests/unit/src/sqlbuild/compiler/compile/_helpers/test_required_columns.py @@ -4,8 +4,12 @@ import pytest -from sqlbuild._native import SqlReferenceKind, analyze_resolved_column_reads -from sqlbuild.compiler.compile.models import CompiledLineageSourceFact, CompileSqlReference +from sqlbuild._native import ( + CompiledLineageSourceFact, + CompileSqlReference, + SqlReferenceKind, + analyze_resolved_column_reads, +) from sqlbuild.compiler.compile.types import CompiledResourceType from tests.unit.src.sqlbuild.compiler.compile._helpers._test_types import ( LongUnionRequiredColumnsTestCase, diff --git a/tests/unit/src/sqlbuild/compiler/compile/_helpers/test_sql_analysis_columns.py b/tests/unit/src/sqlbuild/compiler/compile/_helpers/test_sql_analysis_columns.py index 283ac59d53..4ef9a72518 100644 --- a/tests/unit/src/sqlbuild/compiler/compile/_helpers/test_sql_analysis_columns.py +++ b/tests/unit/src/sqlbuild/compiler/compile/_helpers/test_sql_analysis_columns.py @@ -6,6 +6,9 @@ ColumnLineageConfidence, ColumnTransformKind, CompiledLineageColumnFact, + CompiledLineageSourceFact, + CompileSqlReference, + InferredColumn, NativeColumnQuery, NativeQueryColumns, SqlReferenceKind, @@ -13,11 +16,6 @@ substitute_placeholder_defaults, ) from sqlbuild.adapter.contract.models import ExpressionInferenceProfile -from sqlbuild.compiler.compile.models import ( - CompiledLineageSourceFact, - CompileSqlReference, - InferredColumn, -) from sqlbuild.compiler.compile.types import CompiledResourceType from sqlbuild.compiler.lineage.types import InferredNullability from tests.unit.src.sqlbuild.compiler.compile._helpers._test_types import ( diff --git a/tests/unit/src/sqlbuild/compiler/compile/_helpers/test_sql_references.py b/tests/unit/src/sqlbuild/compiler/compile/_helpers/test_sql_references.py index 0d14eaad84..1e4c1265e5 100644 --- a/tests/unit/src/sqlbuild/compiler/compile/_helpers/test_sql_references.py +++ b/tests/unit/src/sqlbuild/compiler/compile/_helpers/test_sql_references.py @@ -8,6 +8,7 @@ from sqlbuild._native import ( CompileInputError, + CompileSqlReference, SqlLexicalSyntax, SqlReferenceKind, collect_compile_diagnostics, @@ -18,7 +19,6 @@ from sqlbuild.adapters.duckdb.classes.duckdb_adapter import DuckDbAdapter from sqlbuild.adapters.postgres.classes.postgres_adapter import PostgresAdapter from sqlbuild.adapters.snowflake.classes.snowflake_adapter import SnowflakeAdapter -from sqlbuild.compiler.compile.models import CompileSqlReference from tests.unit.src.sqlbuild.compiler.compile._helpers._test_types import ( CollectedReferenceCallSyntaxTestCase, DialectSqlScanTestCase, diff --git a/tests/unit/src/sqlbuild/compiler/compile/_helpers/test_typed_macro_references.py b/tests/unit/src/sqlbuild/compiler/compile/_helpers/test_typed_macro_references.py index d5927cab1a..fa0b2e5a9b 100644 --- a/tests/unit/src/sqlbuild/compiler/compile/_helpers/test_typed_macro_references.py +++ b/tests/unit/src/sqlbuild/compiler/compile/_helpers/test_typed_macro_references.py @@ -6,8 +6,7 @@ import pytest -from sqlbuild._native import CompileInputError -from sqlbuild.compiler.compile.models import CompilerDiagnostic +from sqlbuild._native import CompileInputError, CompilerDiagnostic from sqlbuild.compiler.compile.types import CompiledModelView, CompiledProjectView from tests.unit.src.sqlbuild.compiler.compile._helpers._test_types import ( MacroGeneratedReferenceCollectionTestCase, diff --git a/tests/unit/src/sqlbuild/compiler/contracts/helpers.py b/tests/unit/src/sqlbuild/compiler/contracts/helpers.py index b038b893ce..61471d09ce 100644 --- a/tests/unit/src/sqlbuild/compiler/contracts/helpers.py +++ b/tests/unit/src/sqlbuild/compiler/contracts/helpers.py @@ -5,8 +5,13 @@ from sqlbuild._native import ( ColumnContractMode, + CompiledModel, + CompiledObjectKey, + CompiledProject, + CompiledRelationLocation, CompileModelConfig, DynamicColumnContractProof, + InferredColumn, SchemaAuditInstance, SchemaColumn, SchemaDynamicColumnFamily, @@ -14,13 +19,6 @@ SettingsConfig, SourceLocation, ) -from sqlbuild.compiler.compile.models import ( - CompiledModel, - CompiledObjectKey, - CompiledProject, - CompiledRelationLocation, - InferredColumn, -) from sqlbuild.compiler.compile.types import CompiledProjectView, CompiledResourceType from sqlbuild.compiler.lineage.types import InferredNullability diff --git a/tests/unit/src/sqlbuild/compiler/contracts/test_promotion_conflicts.py b/tests/unit/src/sqlbuild/compiler/contracts/test_promotion_conflicts.py index 6e9d014531..f8c848a6b0 100644 --- a/tests/unit/src/sqlbuild/compiler/contracts/test_promotion_conflicts.py +++ b/tests/unit/src/sqlbuild/compiler/contracts/test_promotion_conflicts.py @@ -2,9 +2,8 @@ import pytest -from sqlbuild._native import promotion_conflict_diagnostics +from sqlbuild._native import CompilerDiagnostic, promotion_conflict_diagnostics from sqlbuild.adapter.contract.types import TablePromotionMode -from sqlbuild.compiler.compile.models import CompilerDiagnostic from tests.unit.src.sqlbuild.compiler.contracts._test_types import PromotionConflictTestCase from tests.unit.src.sqlbuild.compiler.contracts.helpers import make_promotion_project diff --git a/tests/unit/src/sqlbuild/compiler/dag/main/helpers.py b/tests/unit/src/sqlbuild/compiler/dag/main/helpers.py index d68ba67363..f6a4f748c2 100644 --- a/tests/unit/src/sqlbuild/compiler/dag/main/helpers.py +++ b/tests/unit/src/sqlbuild/compiler/dag/main/helpers.py @@ -3,9 +3,16 @@ from pathlib import Path from sqlbuild._native import ( + CompiledAudit, CompiledFunction, + CompiledModel, + CompiledObjectKey, + CompiledProject, + CompiledRelationLocation, CompiledSeed, CompiledSource, + CompiledSqlScenario, + CompiledSqlTest, CompileModelConfig, DiscoveredAuditBlock, DiscoveredAuditFile, @@ -25,15 +32,6 @@ SourceEntry, ) from sqlbuild.compiler.auditing.types import AuditSeverity -from sqlbuild.compiler.compile.models import ( - CompiledAudit, - CompiledModel, - CompiledObjectKey, - CompiledProject, - CompiledRelationLocation, - CompiledSqlScenario, - CompiledSqlTest, -) from sqlbuild.compiler.compile.types import ( AttachedAuditTargetKind, CompiledProjectView, diff --git a/tests/unit/src/sqlbuild/compiler/dag/main/test_build.py b/tests/unit/src/sqlbuild/compiler/dag/main/test_build.py index d0c62a82e5..14303e8a88 100644 --- a/tests/unit/src/sqlbuild/compiler/dag/main/test_build.py +++ b/tests/unit/src/sqlbuild/compiler/dag/main/test_build.py @@ -9,6 +9,7 @@ import pytest from sqlbuild._native import ( + CompiledObjectKey, DiscoveredProjectInputs, ProjectGraph, assemble_project, @@ -18,9 +19,6 @@ project_lineage_views, project_with, ) -from sqlbuild.compiler.compile.models import ( - CompiledObjectKey, -) from sqlbuild.compiler.compile.types import CompiledProjectView from sqlbuild.compiler.dag.types import NodeKind from tests.unit.src.sqlbuild.compiler.compile._helpers.helpers import ( diff --git a/tests/unit/src/sqlbuild/compiler/frontier/helpers.py b/tests/unit/src/sqlbuild/compiler/frontier/helpers.py index 192e943531..26ada14e85 100644 --- a/tests/unit/src/sqlbuild/compiler/frontier/helpers.py +++ b/tests/unit/src/sqlbuild/compiler/frontier/helpers.py @@ -7,8 +7,7 @@ from pathlib import PurePosixPath from typing import Any, cast -from sqlbuild._native import BindingCatalog, DeclarationScopeBuild -from sqlbuild.compiler.compile.models import CompactLineageFacts +from sqlbuild._native import BindingCatalog, CompactLineageFacts, DeclarationScopeBuild HELPERS_MODULE: str = "tests.unit.src.sqlbuild.compiler.frontier.helpers" diff --git a/tests/unit/src/sqlbuild/compiler/graph/helpers.py b/tests/unit/src/sqlbuild/compiler/graph/helpers.py index 25ac641513..7bfa72887f 100644 --- a/tests/unit/src/sqlbuild/compiler/graph/helpers.py +++ b/tests/unit/src/sqlbuild/compiler/graph/helpers.py @@ -4,8 +4,7 @@ from collections.abc import Iterable, Mapping -from sqlbuild._native import ProjectGraph, native_graph_from_views -from sqlbuild.compiler.compile.models import CompiledObjectKey +from sqlbuild._native import CompiledObjectKey, ProjectGraph, native_graph_from_views from sqlbuild.compiler.compile.types import CompiledProjectView diff --git a/tests/unit/src/sqlbuild/compiler/lineage/helpers.py b/tests/unit/src/sqlbuild/compiler/lineage/helpers.py index be57e440aa..4d55aeb1a8 100644 --- a/tests/unit/src/sqlbuild/compiler/lineage/helpers.py +++ b/tests/unit/src/sqlbuild/compiler/lineage/helpers.py @@ -5,22 +5,20 @@ import sqlbuild._native as native from sqlbuild._native import ( ColumnLineageMode, + CompiledModel, + CompiledObjectKey, + CompiledProject, + CompiledRelationLocation, CompiledSeed, CompileModelConfig, DiscoveredSchemaFile, DiscoveredSeedFile, + InferredColumn, ProjectColumnLineage, SchemaColumn, SchemaSeedEntry, SettingsConfig, ) -from sqlbuild.compiler.compile.models import ( - CompiledModel, - CompiledObjectKey, - CompiledProject, - CompiledRelationLocation, - InferredColumn, -) from sqlbuild.compiler.compile.types import ( CompiledModelView, CompiledProjectView, diff --git a/tests/unit/src/sqlbuild/compiler/lineage/test_columns.py b/tests/unit/src/sqlbuild/compiler/lineage/test_columns.py index 2c0d24efa4..3337fea7b2 100644 --- a/tests/unit/src/sqlbuild/compiler/lineage/test_columns.py +++ b/tests/unit/src/sqlbuild/compiler/lineage/test_columns.py @@ -10,11 +10,11 @@ ColumnLineageMode, ColumnTransformKind, CompiledLineageColumnFact, + CompiledLineageSourceFact, ModelColumnLineage, ProjectColumnLineage, compiled_model_record, ) -from sqlbuild.compiler.compile.models import CompiledLineageSourceFact from sqlbuild.compiler.compile.types import ( CompiledModelView, CompiledProjectView, diff --git a/tests/unit/src/sqlbuild/compiler/macro_bridge/_helpers/_test_types.py b/tests/unit/src/sqlbuild/compiler/macro_bridge/_helpers/_test_types.py index 9d68bb05c8..441d58fe03 100644 --- a/tests/unit/src/sqlbuild/compiler/macro_bridge/_helpers/_test_types.py +++ b/tests/unit/src/sqlbuild/compiler/macro_bridge/_helpers/_test_types.py @@ -4,7 +4,7 @@ from dataclasses import dataclass from pathlib import Path -from sqlbuild.compiler.compile.models import DeclarationResolutionContext, MacroContext +from sqlbuild._native import DeclarationResolutionContext, MacroContext @dataclass(frozen=True) diff --git a/tests/unit/src/sqlbuild/compiler/macro_bridge/_helpers/helpers.py b/tests/unit/src/sqlbuild/compiler/macro_bridge/_helpers/helpers.py index 2e798912c4..cfbc6d9a38 100644 --- a/tests/unit/src/sqlbuild/compiler/macro_bridge/_helpers/helpers.py +++ b/tests/unit/src/sqlbuild/compiler/macro_bridge/_helpers/helpers.py @@ -14,8 +14,15 @@ from pathlib import Path import sqlbuild._native as _native -from sqlbuild._native import ConstantDeclaration, SqlLogicalType, SqlValue, SqlValueKind -from sqlbuild.compiler.compile.models import DeclarationResolutionContext, LoadedMacro, MacroContext +from sqlbuild._native import ( + ConstantDeclaration, + DeclarationResolutionContext, + LoadedMacro, + MacroContext, + SqlLogicalType, + SqlValue, + SqlValueKind, +) from sqlbuild.compiler.macro_bridge._helpers.store_environment import ( project_fingerprint, store_environment, diff --git a/tests/unit/src/sqlbuild/compiler/manifest/_test_types.py b/tests/unit/src/sqlbuild/compiler/manifest/_test_types.py index 8aee802a07..de632b3402 100644 --- a/tests/unit/src/sqlbuild/compiler/manifest/_test_types.py +++ b/tests/unit/src/sqlbuild/compiler/manifest/_test_types.py @@ -1,9 +1,6 @@ from dataclasses import dataclass, field -from sqlbuild._native import ModelPlanEntry, PlanOutput -from sqlbuild.compiler.compile.models import ( - CompiledObjectKey, -) +from sqlbuild._native import CompiledObjectKey, ModelPlanEntry, PlanOutput from sqlbuild.compiler.compile.types import CompiledModelView, CompiledProjectView diff --git a/tests/unit/src/sqlbuild/compiler/manifest/helpers.py b/tests/unit/src/sqlbuild/compiler/manifest/helpers.py index 9dd29ea9ec..5d68f33411 100644 --- a/tests/unit/src/sqlbuild/compiler/manifest/helpers.py +++ b/tests/unit/src/sqlbuild/compiler/manifest/helpers.py @@ -9,6 +9,10 @@ from sqlbuild._native import ( AuditPlanEntry, ChainStep, + CompiledModel, + CompiledObjectKey, + CompiledProject, + CompiledRelationLocation, CompiledSeed, CompiledSource, CompileModelConfig, @@ -16,6 +20,7 @@ DiscoveredSeedFile, DiscoveredSourceFile, DiscoveredSqlHookFile, + LoadedMacro, MaterializationType, ModelPlanEntry, PlanAction, @@ -35,13 +40,6 @@ AuditRunScope, AuditSeverity, ) -from sqlbuild.compiler.compile.models import ( - CompiledModel, - CompiledObjectKey, - CompiledProject, - CompiledRelationLocation, - LoadedMacro, -) from sqlbuild.compiler.compile.types import ( CompiledModelView, CompiledProjectView, diff --git a/tests/unit/src/sqlbuild/compiler/manifest/test_main.py b/tests/unit/src/sqlbuild/compiler/manifest/test_main.py index 3a096c978d..4c58d21fcc 100644 --- a/tests/unit/src/sqlbuild/compiler/manifest/test_main.py +++ b/tests/unit/src/sqlbuild/compiler/manifest/test_main.py @@ -11,10 +11,12 @@ from sqlbuild._native import ( AuditPlanEntry, ChainStep, + CompiledObjectKey, DeclarationIdentity, DeclarationKind, DeclarationRecord, DiscoveredSqlHookFile, + LoadedMacro, MacroMetadata, OwnershipRoot, PlanOutput, @@ -28,10 +30,6 @@ build_manifest, project_with, ) -from sqlbuild.compiler.compile.models import ( - CompiledObjectKey, - LoadedMacro, -) from sqlbuild.compiler.compile.types import ( CompiledModelView, CompiledProjectView, diff --git a/tests/unit/src/sqlbuild/compiler/pipeline/_helpers/helpers.py b/tests/unit/src/sqlbuild/compiler/pipeline/_helpers/helpers.py index f135f86c8d..0271fefc1c 100644 --- a/tests/unit/src/sqlbuild/compiler/pipeline/_helpers/helpers.py +++ b/tests/unit/src/sqlbuild/compiler/pipeline/_helpers/helpers.py @@ -5,12 +5,13 @@ from datetime import datetime from pathlib import Path -from sqlbuild._native import CompileModelConfig, Fingerprint -from sqlbuild.compiler.compile.models import ( +from sqlbuild._native import ( CompiledModel, CompiledObjectKey, CompiledProject, CompiledRelationLocation, + CompileModelConfig, + Fingerprint, ) from sqlbuild.compiler.compile.types import ( CompiledModelView, diff --git a/tests/unit/src/sqlbuild/compiler/pipeline/_helpers/target_validation/_test_types.py b/tests/unit/src/sqlbuild/compiler/pipeline/_helpers/target_validation/_test_types.py index 0c27390378..30fbe439c9 100644 --- a/tests/unit/src/sqlbuild/compiler/pipeline/_helpers/target_validation/_test_types.py +++ b/tests/unit/src/sqlbuild/compiler/pipeline/_helpers/target_validation/_test_types.py @@ -1,8 +1,7 @@ from dataclasses import dataclass -from sqlbuild._native import LocalConfig, ProjectConfig +from sqlbuild._native import CompiledRelationLocation, LocalConfig, ProjectConfig from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter -from sqlbuild.compiler.compile.models import CompiledRelationLocation @dataclass(frozen=True) diff --git a/tests/unit/src/sqlbuild/compiler/pipeline/_helpers/target_validation/helpers.py b/tests/unit/src/sqlbuild/compiler/pipeline/_helpers/target_validation/helpers.py index b52740a712..85eb362f4b 100644 --- a/tests/unit/src/sqlbuild/compiler/pipeline/_helpers/target_validation/helpers.py +++ b/tests/unit/src/sqlbuild/compiler/pipeline/_helpers/target_validation/helpers.py @@ -5,14 +5,14 @@ from pathlib import Path from typing import Any, ClassVar -from sqlbuild._native import CompileModelConfig -from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter -from sqlbuild.compiler.compile.models import ( +from sqlbuild._native import ( CompiledModel, CompiledObjectKey, CompiledProject, CompiledRelationLocation, + CompileModelConfig, ) +from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter from sqlbuild.compiler.compile.types import CompiledProjectView, CompiledResourceType diff --git a/tests/unit/src/sqlbuild/compiler/pipeline/_helpers/target_validation/test_main.py b/tests/unit/src/sqlbuild/compiler/pipeline/_helpers/target_validation/test_main.py index 60a2e667ee..c7a0cdfd5e 100644 --- a/tests/unit/src/sqlbuild/compiler/pipeline/_helpers/target_validation/test_main.py +++ b/tests/unit/src/sqlbuild/compiler/pipeline/_helpers/target_validation/test_main.py @@ -5,6 +5,7 @@ import pytest from sqlbuild._native import ( + CompiledRelationLocation, DiscoveredProjectInputs, DiscoveredSourceFile, LocalConfig, @@ -25,9 +26,6 @@ from sqlbuild.adapters.postgres.classes.postgres_adapter import PostgresAdapter from sqlbuild.adapters.snowflake.classes.snowflake_adapter import SnowflakeAdapter from sqlbuild.adapters.sqlserver.classes.sqlserver_adapter import SqlServerAdapter -from sqlbuild.compiler.compile.models import ( - CompiledRelationLocation, -) from sqlbuild.compiler.compile.types import CompiledProjectView from tests.unit.src.sqlbuild.compiler.pipeline._helpers.target_validation._test_types import ( ValidateManagedLoaderConnectionTestCase, diff --git a/tests/unit/src/sqlbuild/compiler/pipeline/_helpers/test_deferred_locations.py b/tests/unit/src/sqlbuild/compiler/pipeline/_helpers/test_deferred_locations.py index 5a06eb78a2..ed79bf8e16 100644 --- a/tests/unit/src/sqlbuild/compiler/pipeline/_helpers/test_deferred_locations.py +++ b/tests/unit/src/sqlbuild/compiler/pipeline/_helpers/test_deferred_locations.py @@ -4,12 +4,9 @@ import pytest -from sqlbuild._native import TargetConfig, build_deferred_locations +from sqlbuild._native import CompiledRelationLocation, TargetConfig, build_deferred_locations from sqlbuild.adapters.bigquery.classes.bigquery_adapter import BigQueryAdapter from sqlbuild.adapters.duckdb.classes.duckdb_adapter import DuckDbAdapter -from sqlbuild.compiler.compile.models import ( - CompiledRelationLocation, -) from sqlbuild.compiler.compile.types import CompiledProjectView from tests.unit.src.sqlbuild.compiler.pipeline._helpers._test_types import ( DeferredTargetTestCase, diff --git a/tests/unit/src/sqlbuild/compiler/pipeline/main/helpers.py b/tests/unit/src/sqlbuild/compiler/pipeline/main/helpers.py index fb08e8786b..4437f1965b 100644 --- a/tests/unit/src/sqlbuild/compiler/pipeline/main/helpers.py +++ b/tests/unit/src/sqlbuild/compiler/pipeline/main/helpers.py @@ -4,6 +4,9 @@ from typing import Any, ClassVar from sqlbuild._native import ( + CompiledObjectKey, + CompiledProject, + CompiledRelationLocation, CompiledSource, DiscoveredSourceFile, MaterializationType, @@ -14,11 +17,6 @@ SourceEntry, ) from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter -from sqlbuild.compiler.compile.models import ( - CompiledObjectKey, - CompiledProject, - CompiledRelationLocation, -) from sqlbuild.compiler.compile.types import CompiledProjectView, CompiledResourceType diff --git a/tests/unit/src/sqlbuild/compiler/pipeline/main/test_compile_phase_timings.py b/tests/unit/src/sqlbuild/compiler/pipeline/main/test_compile_phase_timings.py index 4663874013..0a41ca55d2 100644 --- a/tests/unit/src/sqlbuild/compiler/pipeline/main/test_compile_phase_timings.py +++ b/tests/unit/src/sqlbuild/compiler/pipeline/main/test_compile_phase_timings.py @@ -11,13 +11,13 @@ import sqlbuild._native as _native from sqlbuild._native import ( BuildPhaseTimingTracker, + CompiledProject, CompilePipelineOptions, CompilePipelineResult, PartialBuildPhaseTimings, PlanOutput, RulesRunResult, ) -from sqlbuild.compiler.compile.models import CompiledProject from sqlbuild.compiler.compile.types import CompiledProjectView from sqlbuild.compiler.pipeline._helpers import materializations from sqlbuild.runtime.contracts.models import ConnectionHooks diff --git a/tests/unit/src/sqlbuild/compiler/planner/_helpers/_test_types.py b/tests/unit/src/sqlbuild/compiler/planner/_helpers/_test_types.py index 68862d8592..be95ea7350 100644 --- a/tests/unit/src/sqlbuild/compiler/planner/_helpers/_test_types.py +++ b/tests/unit/src/sqlbuild/compiler/planner/_helpers/_test_types.py @@ -5,8 +5,11 @@ from sqlbuild._native import ( BackfillAction, ChangeKind, + CompiledObjectKey, + CompileSqlReference, CursorBounds, Fingerprint, + InferredColumn, MaterializationType, MissingUpstream, OnSchemaChange, @@ -30,11 +33,6 @@ AuditAttachmentKind, AuditRunScope, ) -from sqlbuild.compiler.compile.models import ( - CompiledObjectKey, - CompileSqlReference, - InferredColumn, -) from sqlbuild.compiler.compile.types import AttachedAuditTargetKind, CompiledProjectView from tests.unit.src.sqlbuild.compiler.planner._helpers.sql_test_assembly._test_types import ( PlanTestChainTestCase as PlanTestChainTestCase, diff --git a/tests/unit/src/sqlbuild/compiler/planner/_helpers/changes/_test_helpers.py b/tests/unit/src/sqlbuild/compiler/planner/_helpers/changes/_test_helpers.py index f4bf072891..b8e04c4b3a 100644 --- a/tests/unit/src/sqlbuild/compiler/planner/_helpers/changes/_test_helpers.py +++ b/tests/unit/src/sqlbuild/compiler/planner/_helpers/changes/_test_helpers.py @@ -10,6 +10,10 @@ from sqlbuild._native import ( ChangeKind, CompiledFunction, + CompiledModel, + CompiledObjectKey, + CompiledProject, + CompiledRelationLocation, CompileModelConfig, Fingerprint, FunctionArgument, @@ -22,12 +26,6 @@ compute_query_hash, ) from sqlbuild.adapter.contract.models import ColumnInfo, RelationInfo -from sqlbuild.compiler.compile.models import ( - CompiledModel, - CompiledObjectKey, - CompiledProject, - CompiledRelationLocation, -) from sqlbuild.compiler.compile.types import ( CompiledFunctionView, CompiledModelView, diff --git a/tests/unit/src/sqlbuild/compiler/planner/_helpers/changes/_test_types.py b/tests/unit/src/sqlbuild/compiler/planner/_helpers/changes/_test_types.py index 7d87bf60a9..1e6be88af8 100644 --- a/tests/unit/src/sqlbuild/compiler/planner/_helpers/changes/_test_types.py +++ b/tests/unit/src/sqlbuild/compiler/planner/_helpers/changes/_test_types.py @@ -4,6 +4,7 @@ BackfillAction, BackfillResult, ChangeKind, + InferredColumn, PlanAction, PlanReason, ReferenceRename, @@ -11,7 +12,6 @@ SchemaFinding, ) from sqlbuild.adapter.contract.models import ColumnInfo -from sqlbuild.compiler.compile.models import InferredColumn @dataclass(frozen=True) diff --git a/tests/unit/src/sqlbuild/compiler/planner/_helpers/changes/test_detect.py b/tests/unit/src/sqlbuild/compiler/planner/_helpers/changes/test_detect.py index 9c2bde4352..966b99d644 100644 --- a/tests/unit/src/sqlbuild/compiler/planner/_helpers/changes/test_detect.py +++ b/tests/unit/src/sqlbuild/compiler/planner/_helpers/changes/test_detect.py @@ -11,6 +11,7 @@ ChangeDetectionResult, ChangeKind, Fingerprint, + InferredColumn, PlanAction, PlannerChangeResults, PlannerScope, @@ -28,7 +29,6 @@ get_materialization_type, resolve_model_plan_action, ) -from sqlbuild.compiler.compile.models import InferredColumn from sqlbuild.compiler.compile.types import CompiledModelView from tests.unit.src.sqlbuild.compiler.planner._helpers.changes._test_helpers import ( build_function_caller_metadata_case, diff --git a/tests/unit/src/sqlbuild/compiler/planner/_helpers/changes/test_schema.py b/tests/unit/src/sqlbuild/compiler/planner/_helpers/changes/test_schema.py index b0e45d6a94..b24c1e522a 100644 --- a/tests/unit/src/sqlbuild/compiler/planner/_helpers/changes/test_schema.py +++ b/tests/unit/src/sqlbuild/compiler/planner/_helpers/changes/test_schema.py @@ -3,6 +3,7 @@ import pytest from sqlbuild._native import ( + InferredColumn, SchemaChangeKind, SchemaColumnSource, SchemaDynamicColumnFamily, @@ -10,7 +11,6 @@ detect_schema_changes, ) from sqlbuild.adapter.contract.models import ColumnInfo -from sqlbuild.compiler.compile.models import InferredColumn from tests.unit.src.sqlbuild.compiler.planner._helpers.changes._test_types import ( DetectSchemaChangesTestCase, ) diff --git a/tests/unit/src/sqlbuild/compiler/planner/_helpers/conftest.py b/tests/unit/src/sqlbuild/compiler/planner/_helpers/conftest.py index c4efc9f391..0a6994b40e 100644 --- a/tests/unit/src/sqlbuild/compiler/planner/_helpers/conftest.py +++ b/tests/unit/src/sqlbuild/compiler/planner/_helpers/conftest.py @@ -3,13 +3,11 @@ import pytest from sqlbuild._native import ( + CompiledObjectKey, build_downstream_deps, build_execution_upstream_deps, project_lineage_views, ) -from sqlbuild.compiler.compile.models import ( - CompiledObjectKey, -) from sqlbuild.compiler.compile.types import CompiledProjectView from tests.unit.src.sqlbuild.compiler.planner._helpers.helpers import ( build_test_project, diff --git a/tests/unit/src/sqlbuild/compiler/planner/_helpers/helpers.py b/tests/unit/src/sqlbuild/compiler/planner/_helpers/helpers.py index 7cedd317a1..7e8ae4ca2b 100644 --- a/tests/unit/src/sqlbuild/compiler/planner/_helpers/helpers.py +++ b/tests/unit/src/sqlbuild/compiler/planner/_helpers/helpers.py @@ -16,13 +16,21 @@ BackfillAction, BackfillResult, ChangeDetectionResult, + CompiledAudit, CompiledDirectLogicSqlTestPayload, CompiledFunction, + CompiledModel, CompiledModelSqlTestPayload, + CompiledObjectKey, + CompiledProject, + CompiledRelationLocation, CompiledSeed, CompiledSource, + CompiledSqlScenario, + CompiledSqlTest, CompileModelConfig, CompilePipelineResult, + CompileSqlReference, CompileSqlScenarioCte, CompileSqlTestCte, DefaultsConfig, @@ -63,16 +71,6 @@ from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter from sqlbuild.adapter.contract.models import RelationInfo from sqlbuild.compiler.auditing.types import AuditSeverity -from sqlbuild.compiler.compile.models import ( - CompiledAudit, - CompiledModel, - CompiledObjectKey, - CompiledProject, - CompiledRelationLocation, - CompiledSqlScenario, - CompiledSqlTest, - CompileSqlReference, -) from sqlbuild.compiler.compile.types import ( AttachedAuditTargetKind, CompiledAuditView, diff --git a/tests/unit/src/sqlbuild/compiler/planner/_helpers/migrations/helpers.py b/tests/unit/src/sqlbuild/compiler/planner/_helpers/migrations/helpers.py index 80c999e576..10daace3fc 100644 --- a/tests/unit/src/sqlbuild/compiler/planner/_helpers/migrations/helpers.py +++ b/tests/unit/src/sqlbuild/compiler/planner/_helpers/migrations/helpers.py @@ -7,13 +7,13 @@ from typing import TYPE_CHECKING from sqlbuild._native import ( + CompiledRelationLocation, OldNameViewAction, OldNameViewPlanEntry, QueryShape, build_migration_fingerprint, parse_query_shape, ) -from sqlbuild.compiler.compile.models import CompiledRelationLocation if TYPE_CHECKING: from sqlbuild._native import InputColumns diff --git a/tests/unit/src/sqlbuild/compiler/planner/_helpers/migrations/test_column_migration_candidates.py b/tests/unit/src/sqlbuild/compiler/planner/_helpers/migrations/test_column_migration_candidates.py index c0f9de84d6..cd0c4d1278 100644 --- a/tests/unit/src/sqlbuild/compiler/planner/_helpers/migrations/test_column_migration_candidates.py +++ b/tests/unit/src/sqlbuild/compiler/planner/_helpers/migrations/test_column_migration_candidates.py @@ -9,6 +9,10 @@ import pytest from sqlbuild._native import ( + CompiledModel, + CompiledObjectKey, + CompiledProject, + CompiledRelationLocation, CompileModelConfig, DeferralInputs, Fingerprint, @@ -21,12 +25,6 @@ plan_column_migrations, ) from sqlbuild.adapter.contract.models import ColumnInfo, RelationInfo -from sqlbuild.compiler.compile.models import ( - CompiledModel, - CompiledObjectKey, - CompiledProject, - CompiledRelationLocation, -) from sqlbuild.compiler.compile.types import CompiledModelView, CompiledResourceType from tests.unit.src.sqlbuild.compiler.planner._helpers.migrations._test_types import ( UnfingerprintableCandidateTestCase, diff --git a/tests/unit/src/sqlbuild/compiler/planner/_helpers/planning/helpers.py b/tests/unit/src/sqlbuild/compiler/planner/_helpers/planning/helpers.py index 79f6e51659..aa4ed448c9 100644 --- a/tests/unit/src/sqlbuild/compiler/planner/_helpers/planning/helpers.py +++ b/tests/unit/src/sqlbuild/compiler/planner/_helpers/planning/helpers.py @@ -5,6 +5,10 @@ from unittest.mock import Mock from sqlbuild._native import ( + CompiledModel, + CompiledObjectKey, + CompiledProject, + CompiledRelationLocation, CompileModelConfig, LocalConfig, PlannerRelationsContext, @@ -21,12 +25,6 @@ ) from sqlbuild.adapter.contract.classes.retention_adapter import RetentionAdapterMixin from sqlbuild.adapter.contract.models import RelationInfo -from sqlbuild.compiler.compile.models import ( - CompiledModel, - CompiledObjectKey, - CompiledProject, - CompiledRelationLocation, -) from sqlbuild.compiler.compile.types import ( CompiledModelView, CompiledProjectView, diff --git a/tests/unit/src/sqlbuild/compiler/planner/_helpers/resolve/helpers.py b/tests/unit/src/sqlbuild/compiler/planner/_helpers/resolve/helpers.py index 82f2acdab3..88f507dd64 100644 --- a/tests/unit/src/sqlbuild/compiler/planner/_helpers/resolve/helpers.py +++ b/tests/unit/src/sqlbuild/compiler/planner/_helpers/resolve/helpers.py @@ -5,15 +5,13 @@ from pathlib import Path from sqlbuild._native import ( - CompileModelConfig, - ModelPlanContext, -) -from sqlbuild.adapters.duckdb.classes.duckdb_adapter import DuckDbAdapter -from sqlbuild.compiler.compile.models import ( CompiledModel, CompiledObjectKey, CompiledRelationLocation, + CompileModelConfig, + ModelPlanContext, ) +from sqlbuild.adapters.duckdb.classes.duckdb_adapter import DuckDbAdapter from sqlbuild.compiler.compile.types import CompiledModelView, CompiledResourceType diff --git a/tests/unit/src/sqlbuild/compiler/planner/_helpers/resolve/test_refs.py b/tests/unit/src/sqlbuild/compiler/planner/_helpers/resolve/test_refs.py index e47fe652dc..28b25f0c39 100644 --- a/tests/unit/src/sqlbuild/compiler/planner/_helpers/resolve/test_refs.py +++ b/tests/unit/src/sqlbuild/compiler/planner/_helpers/resolve/test_refs.py @@ -5,6 +5,8 @@ import pytest from sqlbuild._native import ( + CompiledObjectKey, + CompiledRelationLocation, CursorBounds, DbtCompileReferenceResolver, DbtManifestIndex, @@ -17,10 +19,6 @@ ) from sqlbuild.adapter.contract.types import CursorKind from sqlbuild.adapters.duckdb.classes.duckdb_adapter import DuckDbAdapter -from sqlbuild.compiler.compile.models import ( - CompiledObjectKey, - CompiledRelationLocation, -) from sqlbuild.compiler.compile.types import CompiledResourceType from tests.unit.src.sqlbuild.compiler.planner._helpers.resolve._test_types import ( ApplyDeferredTargetsTestCase, diff --git a/tests/unit/src/sqlbuild/compiler/planner/_helpers/sql_test_assembly/helpers.py b/tests/unit/src/sqlbuild/compiler/planner/_helpers/sql_test_assembly/helpers.py index e750b392f4..6f04361318 100644 --- a/tests/unit/src/sqlbuild/compiler/planner/_helpers/sql_test_assembly/helpers.py +++ b/tests/unit/src/sqlbuild/compiler/planner/_helpers/sql_test_assembly/helpers.py @@ -9,11 +9,18 @@ from sqlbuild._native import ( CompiledFunction, + CompiledModel, CompiledModelSqlTestPayload, + CompiledObjectKey, + CompiledProject, + CompiledRelationLocation, + CompiledSqlTest, CompileModelConfig, CompileSqlTestCte, DiscoveredSqlTestBlock, DiscoveredSqlTestFile, + LoadedMacro, + MacroContext, NativeSqlTestPlan, PlanWarning, SqlLexicalSyntax, @@ -31,15 +38,6 @@ from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter from sqlbuild.adapters.duckdb.classes.duckdb_adapter import DuckDbAdapter from sqlbuild.adapters.snowflake.classes.snowflake_adapter import SnowflakeAdapter -from sqlbuild.compiler.compile.models import ( - CompiledModel, - CompiledObjectKey, - CompiledProject, - CompiledRelationLocation, - CompiledSqlTest, - LoadedMacro, - MacroContext, -) from sqlbuild.compiler.compile.types import ( CompiledFunctionView, CompiledModelView, diff --git a/tests/unit/src/sqlbuild/compiler/planner/_helpers/sql_test_assembly/test_plan_test.py b/tests/unit/src/sqlbuild/compiler/planner/_helpers/sql_test_assembly/test_plan_test.py index a450ff53b0..7d89e3ffbb 100644 --- a/tests/unit/src/sqlbuild/compiler/planner/_helpers/sql_test_assembly/test_plan_test.py +++ b/tests/unit/src/sqlbuild/compiler/planner/_helpers/sql_test_assembly/test_plan_test.py @@ -10,6 +10,10 @@ CompiledDirectLogicSqlTestPayload, CompiledFunction, CompiledModelSqlTestPayload, + CompiledObjectKey, + CompiledProject, + CompiledRelationLocation, + CompiledSqlTest, CompileInputError, CompileSqlTestCte, DiscoveredSqlTestBlock, @@ -35,12 +39,6 @@ from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter from sqlbuild.adapters.duckdb.classes.duckdb_adapter import DuckDbAdapter from sqlbuild.adapters.sqlserver.classes.sqlserver_adapter import SqlServerAdapter -from sqlbuild.compiler.compile.models import ( - CompiledObjectKey, - CompiledProject, - CompiledRelationLocation, - CompiledSqlTest, -) from sqlbuild.compiler.compile.types import CompiledProjectView, CompiledResourceType, SqlTestMode from tests.unit.src.sqlbuild.compiler.planner._helpers.sql_test_assembly._test_types import ( AssertionChainCteErrorTestCase, diff --git a/tests/unit/src/sqlbuild/compiler/planner/_helpers/test_audit_entry.py b/tests/unit/src/sqlbuild/compiler/planner/_helpers/test_audit_entry.py index 7f7a371ac7..5af837bb96 100644 --- a/tests/unit/src/sqlbuild/compiler/planner/_helpers/test_audit_entry.py +++ b/tests/unit/src/sqlbuild/compiler/planner/_helpers/test_audit_entry.py @@ -2,13 +2,16 @@ import pytest -from sqlbuild._native import AuditPlanEntry, SourceEntry, SqlReferenceKind, plan_audit -from sqlbuild.compiler.auditing.types import AuditAttachmentKind, AuditRunScope -from sqlbuild.compiler.compile.models import ( +from sqlbuild._native import ( + AuditPlanEntry, CompiledObjectKey, CompiledRelationLocation, CompileSqlReference, + SourceEntry, + SqlReferenceKind, + plan_audit, ) +from sqlbuild.compiler.auditing.types import AuditAttachmentKind, AuditRunScope from sqlbuild.compiler.compile.types import AttachedAuditTargetKind, CompiledAuditView from tests.unit.src.sqlbuild.compiler.planner._helpers._test_types import ( PlanAttachedAuditTestCase, diff --git a/tests/unit/src/sqlbuild/compiler/planner/_helpers/test_audit_scheduling.py b/tests/unit/src/sqlbuild/compiler/planner/_helpers/test_audit_scheduling.py index 5c1a2b9f75..5378bf5e98 100644 --- a/tests/unit/src/sqlbuild/compiler/planner/_helpers/test_audit_scheduling.py +++ b/tests/unit/src/sqlbuild/compiler/planner/_helpers/test_audit_scheduling.py @@ -5,6 +5,8 @@ import pytest from sqlbuild._native import ( + CompiledObjectKey, + CompileSqlReference, MaterializationType, SqlReferenceKind, reads_outside_target_lineage, @@ -15,10 +17,6 @@ AuditAttachmentKind, AuditRunScope, ) -from sqlbuild.compiler.compile.models import ( - CompiledObjectKey, - CompileSqlReference, -) from sqlbuild.compiler.compile.types import ( AttachedAuditTargetKind, CompiledAuditView, diff --git a/tests/unit/src/sqlbuild/compiler/planner/_helpers/test_fixture_planning_work.py b/tests/unit/src/sqlbuild/compiler/planner/_helpers/test_fixture_planning_work.py index c13b233337..5da157a363 100644 --- a/tests/unit/src/sqlbuild/compiler/planner/_helpers/test_fixture_planning_work.py +++ b/tests/unit/src/sqlbuild/compiler/planner/_helpers/test_fixture_planning_work.py @@ -8,6 +8,9 @@ import sqlbuild._native as sql_test_assembly from sqlbuild import _native as fixture_completion from sqlbuild._native import ( + CompiledObjectKey, + CompiledSqlTest, + CompileSqlReference, FixtureColumnMetadata, FixtureRelationMetadata, PlanWarning, @@ -21,11 +24,6 @@ compiled_model_record, project_with, ) -from sqlbuild.compiler.compile.models import ( - CompiledObjectKey, - CompiledSqlTest, - CompileSqlReference, -) from sqlbuild.compiler.compile.types import ( CompiledModelView, CompiledProjectView, diff --git a/tests/unit/src/sqlbuild/compiler/planner/_helpers/test_graph.py b/tests/unit/src/sqlbuild/compiler/planner/_helpers/test_graph.py index 85a57d11bb..af86317720 100644 --- a/tests/unit/src/sqlbuild/compiler/planner/_helpers/test_graph.py +++ b/tests/unit/src/sqlbuild/compiler/planner/_helpers/test_graph.py @@ -3,13 +3,13 @@ import pytest from sqlbuild._native import ( + CompiledObjectKey, build_downstream_deps, build_execution_edge_origins, build_execution_upstream_deps, topologically_order_keys, transitive_closure, ) -from sqlbuild.compiler.compile.models import CompiledObjectKey from tests.unit.src.sqlbuild.compiler.planner._helpers._test_types import ( BuildDownstreamDepsTestCase, BuildUpstreamDepsTestCase, diff --git a/tests/unit/src/sqlbuild/compiler/planner/_helpers/test_microbatch_limit.py b/tests/unit/src/sqlbuild/compiler/planner/_helpers/test_microbatch_limit.py index 667c2526f5..de3d39f92b 100644 --- a/tests/unit/src/sqlbuild/compiler/planner/_helpers/test_microbatch_limit.py +++ b/tests/unit/src/sqlbuild/compiler/planner/_helpers/test_microbatch_limit.py @@ -5,6 +5,8 @@ import pytest from sqlbuild._native import ( + CompiledObjectKey, + CompiledRelationLocation, CompileInputError, CursorBounds, CursorInputRelation, @@ -20,10 +22,6 @@ apply_microbatch_limit, ) from sqlbuild._native import cursor_sentinel_to_token as sentinel_to_token -from sqlbuild.compiler.compile.models import ( - CompiledObjectKey, - CompiledRelationLocation, -) from sqlbuild.compiler.compile.types import CompiledResourceType from tests.unit.src.sqlbuild.compiler.planner._helpers._test_types import ( ConfiguredMicrobatchPolicyTestCase, diff --git a/tests/unit/src/sqlbuild/compiler/planner/_helpers/test_model_closure.py b/tests/unit/src/sqlbuild/compiler/planner/_helpers/test_model_closure.py index 923a00170c..f78652a500 100644 --- a/tests/unit/src/sqlbuild/compiler/planner/_helpers/test_model_closure.py +++ b/tests/unit/src/sqlbuild/compiler/planner/_helpers/test_model_closure.py @@ -2,8 +2,7 @@ import pytest -from sqlbuild._native import build_downstream_model_name_closure -from sqlbuild.compiler.compile.models import CompiledObjectKey +from sqlbuild._native import CompiledObjectKey, build_downstream_model_name_closure from sqlbuild.compiler.compile.types import CompiledResourceType from tests.unit.src.sqlbuild.compiler.planner._helpers._test_types import ModelClosureTestCase diff --git a/tests/unit/src/sqlbuild/compiler/planner/_helpers/test_plan_entry_cursor_overrides.py b/tests/unit/src/sqlbuild/compiler/planner/_helpers/test_plan_entry_cursor_overrides.py index c433a0e83a..d8b006fa4b 100644 --- a/tests/unit/src/sqlbuild/compiler/planner/_helpers/test_plan_entry_cursor_overrides.py +++ b/tests/unit/src/sqlbuild/compiler/planner/_helpers/test_plan_entry_cursor_overrides.py @@ -10,6 +10,9 @@ from sqlbuild._native import ( BackfillAction, BackfillResult, + CompiledModel, + CompiledObjectKey, + CompiledRelationLocation, CompileModelConfig, CursorBounds, CursorInputRelation, @@ -19,11 +22,6 @@ compute_plan_cursor_bounds, ) from sqlbuild._native import cursor_sentinel_to_token as sentinel_to_token -from sqlbuild.compiler.compile.models import ( - CompiledModel, - CompiledObjectKey, - CompiledRelationLocation, -) from sqlbuild.compiler.compile.types import CompiledModelView, CompiledResourceType from sqlbuild.compiler.planner.constants import MICROBATCH_END_SENTINEL, MICROBATCH_START_SENTINEL from tests.unit.src.sqlbuild.compiler.planner._helpers._test_types import ( diff --git a/tests/unit/src/sqlbuild/compiler/planner/_helpers/test_scenario_cli.py b/tests/unit/src/sqlbuild/compiler/planner/_helpers/test_scenario_cli.py index c42aac231c..d96a4723c5 100644 --- a/tests/unit/src/sqlbuild/compiler/planner/_helpers/test_scenario_cli.py +++ b/tests/unit/src/sqlbuild/compiler/planner/_helpers/test_scenario_cli.py @@ -6,13 +6,13 @@ import pytest from sqlbuild._native import ( + CompiledSqlScenario, CompilePipelineResult, ScenarioArtifactName, ScenarioExecutionPlan, SqlLexicalSyntax, build_cli_scenario_plan, ) -from sqlbuild.compiler.compile.models import CompiledSqlScenario from tests.unit.src.sqlbuild.compiler.planner._helpers._test_types import ( ScenarioCliPlanIdentifierLimitTestCase, ) diff --git a/tests/unit/src/sqlbuild/compiler/planner/_helpers/test_scenario_graph.py b/tests/unit/src/sqlbuild/compiler/planner/_helpers/test_scenario_graph.py index aa955bd78a..060193c5c8 100644 --- a/tests/unit/src/sqlbuild/compiler/planner/_helpers/test_scenario_graph.py +++ b/tests/unit/src/sqlbuild/compiler/planner/_helpers/test_scenario_graph.py @@ -5,14 +5,12 @@ import pytest from sqlbuild._native import ( + CompiledObjectKey, + CompiledSqlScenario, ScenarioGraphPlan, WarningSeverity, plan_scenario_graph, ) -from sqlbuild.compiler.compile.models import ( - CompiledObjectKey, - CompiledSqlScenario, -) from sqlbuild.compiler.compile.types import CompiledResourceType from tests.unit.src.sqlbuild.compiler.planner._helpers._test_types import ( PlanScenarioGraphErrorTestCase, diff --git a/tests/unit/src/sqlbuild/compiler/planner/_helpers/test_scenario_relations.py b/tests/unit/src/sqlbuild/compiler/planner/_helpers/test_scenario_relations.py index 16ad72384b..8f824c76bf 100644 --- a/tests/unit/src/sqlbuild/compiler/planner/_helpers/test_scenario_relations.py +++ b/tests/unit/src/sqlbuild/compiler/planner/_helpers/test_scenario_relations.py @@ -7,6 +7,7 @@ import pytest from sqlbuild._native import ( + CompiledSqlScenario, CompileSqlScenarioCte, DiscoveredHookFunction, PlannerInputError, @@ -27,7 +28,6 @@ from sqlbuild.adapters.postgres.classes.postgres_adapter import PostgresAdapter from sqlbuild.adapters.snowflake.classes.snowflake_adapter import SnowflakeAdapter from sqlbuild.adapters.sqlserver.classes.sqlserver_adapter import SqlServerAdapter -from sqlbuild.compiler.compile.models import CompiledSqlScenario from sqlbuild.compiler.compile.types import CompiledProjectView from tests.unit.src.sqlbuild.compiler.planner._helpers._test_types import ( ScenarioCheckSqlResolutionErrorTestCase, diff --git a/tests/unit/src/sqlbuild/compiler/planner/_helpers/test_selection_staleness.py b/tests/unit/src/sqlbuild/compiler/planner/_helpers/test_selection_staleness.py index 627793ba5e..5f982bfe37 100644 --- a/tests/unit/src/sqlbuild/compiler/planner/_helpers/test_selection_staleness.py +++ b/tests/unit/src/sqlbuild/compiler/planner/_helpers/test_selection_staleness.py @@ -8,6 +8,8 @@ from sqlbuild._native import ( ChangeDetectionResult, ChangeKind, + CompiledObjectKey, + CompiledProject, DirectModelVersionIdentities, DirectSourceFreshnessPlanningResult, Fingerprint, @@ -24,7 +26,6 @@ compiled_model_record, resolve_planner_scopes, ) -from sqlbuild.compiler.compile.models import CompiledObjectKey, CompiledProject from sqlbuild.compiler.compile.types import ( CompiledModelView, CompiledProjectView, diff --git a/tests/unit/src/sqlbuild/compiler/planner/_helpers/test_selectors.py b/tests/unit/src/sqlbuild/compiler/planner/_helpers/test_selectors.py index a190291669..aee3733314 100644 --- a/tests/unit/src/sqlbuild/compiler/planner/_helpers/test_selectors.py +++ b/tests/unit/src/sqlbuild/compiler/planner/_helpers/test_selectors.py @@ -3,6 +3,7 @@ import pytest from sqlbuild._native import ( + CompiledObjectKey, ParsedSelector, PathSelector, SelectorKind, @@ -11,9 +12,6 @@ project_lineage_views, resolve_project_selectors, ) -from sqlbuild.compiler.compile.models import ( - CompiledObjectKey, -) from sqlbuild.compiler.compile.types import CompiledProjectView from tests.unit.src.sqlbuild.compiler.planner._helpers._test_types import ( BuildPathIndexTestCase, diff --git a/tests/unit/src/sqlbuild/compiler/planner/_helpers/test_source_columns.py b/tests/unit/src/sqlbuild/compiler/planner/_helpers/test_source_columns.py index 7549f16ccb..3c1c5c141e 100644 --- a/tests/unit/src/sqlbuild/compiler/planner/_helpers/test_source_columns.py +++ b/tests/unit/src/sqlbuild/compiler/planner/_helpers/test_source_columns.py @@ -7,6 +7,8 @@ import pytest from sqlbuild._native import ( + CompiledObjectKey, + CompiledProject, CursorInputRelation, PlannerInputError, PlannerRelationsContext, @@ -22,10 +24,6 @@ from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter from sqlbuild.adapter.contract.models import ColumnInfo, RelationInfo from sqlbuild.adapters.duckdb.classes.duckdb_adapter import DuckDbAdapter -from sqlbuild.compiler.compile.models import ( - CompiledObjectKey, - CompiledProject, -) from sqlbuild.compiler.compile.types import ( CompiledFunctionView, CompiledModelView, diff --git a/tests/unit/src/sqlbuild/compiler/planner/_helpers/test_source_freshness.py b/tests/unit/src/sqlbuild/compiler/planner/_helpers/test_source_freshness.py index cf12bac7fb..47e077aed9 100644 --- a/tests/unit/src/sqlbuild/compiler/planner/_helpers/test_source_freshness.py +++ b/tests/unit/src/sqlbuild/compiler/planner/_helpers/test_source_freshness.py @@ -5,6 +5,7 @@ import pytest from sqlbuild._native import ( + CompiledProject, DirectSourceFreshnessPlanningResult, PlannerRelationsContext, PlannerScope, @@ -15,7 +16,6 @@ build_planner_source_freshness_result, ) from sqlbuild.adapters.duckdb.classes.duckdb_adapter import DuckDbAdapter -from sqlbuild.compiler.compile.models import CompiledProject from tests.unit.src.sqlbuild.compiler.planner._helpers._test_types import ( PlannerSourceFreshnessReadMapTestCase, ) diff --git a/tests/unit/src/sqlbuild/compiler/planner/_helpers/test_source_load_nodes.py b/tests/unit/src/sqlbuild/compiler/planner/_helpers/test_source_load_nodes.py index 46bb990b65..fa112e1b48 100644 --- a/tests/unit/src/sqlbuild/compiler/planner/_helpers/test_source_load_nodes.py +++ b/tests/unit/src/sqlbuild/compiler/planner/_helpers/test_source_load_nodes.py @@ -8,6 +8,10 @@ import pytest from sqlbuild._native import ( + CompiledModel, + CompiledObjectKey, + CompiledProject, + CompiledRelationLocation, CompileModelConfig, DiscoveredLoaderFunction, SourceEntry, @@ -21,12 +25,6 @@ project_with, upstream_loader_dependency_names, ) -from sqlbuild.compiler.compile.models import ( - CompiledModel, - CompiledObjectKey, - CompiledProject, - CompiledRelationLocation, -) from sqlbuild.compiler.compile.types import ( CompiledModelView, CompiledProjectView, diff --git a/tests/unit/src/sqlbuild/compiler/planner/_helpers/warehouse/helpers.py b/tests/unit/src/sqlbuild/compiler/planner/_helpers/warehouse/helpers.py index c23d5c4d3d..9172b003b0 100644 --- a/tests/unit/src/sqlbuild/compiler/planner/_helpers/warehouse/helpers.py +++ b/tests/unit/src/sqlbuild/compiler/planner/_helpers/warehouse/helpers.py @@ -7,6 +7,8 @@ from typing import cast from sqlbuild._native import ( + CompiledObjectKey, + CompiledProject, CompiledSource, DiscoveredSourceFile, PlannerRelationsContext, @@ -16,7 +18,6 @@ ) from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter from sqlbuild.adapter.contract.models import ColumnInfo -from sqlbuild.compiler.compile.models import CompiledObjectKey, CompiledProject from sqlbuild.compiler.compile.types import CompiledResourceType, CompiledSourceView diff --git a/tests/unit/src/sqlbuild/compiler/planner/classes/test_sql_test_planning.py b/tests/unit/src/sqlbuild/compiler/planner/classes/test_sql_test_planning.py index a90e7a64a9..e58cae2ad0 100644 --- a/tests/unit/src/sqlbuild/compiler/planner/classes/test_sql_test_planning.py +++ b/tests/unit/src/sqlbuild/compiler/planner/classes/test_sql_test_planning.py @@ -7,6 +7,7 @@ import sqlbuild._native as native_module from sqlbuild._native import ( + CompiledObjectKey, PlannedSqlTests, PlannerInputError, PlanWarning, @@ -14,7 +15,6 @@ WarningSeverity, start_sql_test_planning, ) -from sqlbuild.compiler.compile.models import CompiledObjectKey from sqlbuild.compiler.compile.types import CompiledResourceType from tests.unit.src.sqlbuild.compiler.planner.classes._test_types import ( BackgroundSqlTestPlanningTestCase, diff --git a/tests/unit/src/sqlbuild/compiler/planner/main/execution/test_display_plan.py b/tests/unit/src/sqlbuild/compiler/planner/main/execution/test_display_plan.py index 3bfe502d29..0b5ca170fb 100644 --- a/tests/unit/src/sqlbuild/compiler/planner/main/execution/test_display_plan.py +++ b/tests/unit/src/sqlbuild/compiler/planner/main/execution/test_display_plan.py @@ -7,6 +7,10 @@ import pytest from sqlbuild._native import ( + CompiledModel, + CompiledObjectKey, + CompiledProject, + CompiledRelationLocation, CompileModelConfig, ModelPlanEntry, PlanAction, @@ -15,12 +19,6 @@ build_display_only_sqlbuild_plan, format_plan, ) -from sqlbuild.compiler.compile.models import ( - CompiledModel, - CompiledObjectKey, - CompiledProject, - CompiledRelationLocation, -) from sqlbuild.compiler.compile.types import CompiledModelView, CompiledResourceType from tests.unit.src.sqlbuild.compiler.planner.main.execution._test_types import ( DisplayFullRefreshTestCase, diff --git a/tests/unit/src/sqlbuild/compiler/planner/main/helpers.py b/tests/unit/src/sqlbuild/compiler/planner/main/helpers.py index f158c2aec8..1af69db657 100644 --- a/tests/unit/src/sqlbuild/compiler/planner/main/helpers.py +++ b/tests/unit/src/sqlbuild/compiler/planner/main/helpers.py @@ -6,6 +6,10 @@ from typing import Any from sqlbuild._native import ( + CompiledModel, + CompiledObjectKey, + CompiledProject, + CompiledRelationLocation, CompileModelConfig, DeferralInputs, Fingerprint, @@ -23,12 +27,6 @@ write_fingerprint, ) from sqlbuild.adapters.duckdb.classes.duckdb_adapter import DuckDbAdapter -from sqlbuild.compiler.compile.models import ( - CompiledModel, - CompiledObjectKey, - CompiledProject, - CompiledRelationLocation, -) from sqlbuild.compiler.compile.types import ( CompiledModelView, CompiledProjectView, diff --git a/tests/unit/src/sqlbuild/compiler/planner/main/identity/helpers.py b/tests/unit/src/sqlbuild/compiler/planner/main/identity/helpers.py index 179ef8ee7a..41b5526012 100644 --- a/tests/unit/src/sqlbuild/compiler/planner/main/identity/helpers.py +++ b/tests/unit/src/sqlbuild/compiler/planner/main/identity/helpers.py @@ -5,6 +5,9 @@ from sqlbuild._native import ( CompiledFunction, + CompiledModel, + CompiledObjectKey, + CompiledRelationLocation, CompileModelConfig, DirectModelVersionIdentities, FunctionArgument, @@ -18,11 +21,6 @@ build_version_identity_metadata_json, compiled_model_record, ) -from sqlbuild.compiler.compile.models import ( - CompiledModel, - CompiledObjectKey, - CompiledRelationLocation, -) from sqlbuild.compiler.compile.types import ( CompiledModelView, CompiledResourceType, diff --git a/tests/unit/src/sqlbuild/compiler/planner/main/identity/test_version_identity_function_hashes.py b/tests/unit/src/sqlbuild/compiler/planner/main/identity/test_version_identity_function_hashes.py index 065326faee..3cc6ca8e30 100644 --- a/tests/unit/src/sqlbuild/compiler/planner/main/identity/test_version_identity_function_hashes.py +++ b/tests/unit/src/sqlbuild/compiler/planner/main/identity/test_version_identity_function_hashes.py @@ -6,11 +6,11 @@ from sqlbuild._native import ( CompiledFunction, + CompiledObjectKey, DirectModelVersionIdentities, FunctionReturnColumn, build_function_local_hashes, ) -from sqlbuild.compiler.compile.models import CompiledObjectKey from sqlbuild.compiler.compile.types import CompiledResourceType from tests.unit.src.sqlbuild.compiler.planner._helpers.helpers import build_compiled_function from tests.unit.src.sqlbuild.compiler.planner.main.identity._test_types import ( diff --git a/tests/unit/src/sqlbuild/compiler/planner/main/pre_build/helpers.py b/tests/unit/src/sqlbuild/compiler/planner/main/pre_build/helpers.py index 45a0cfa718..d3322ed84d 100644 --- a/tests/unit/src/sqlbuild/compiler/planner/main/pre_build/helpers.py +++ b/tests/unit/src/sqlbuild/compiler/planner/main/pre_build/helpers.py @@ -5,6 +5,7 @@ from sqlbuild._native import ( ColumnMigrationDecision, ColumnMigrationPlanEntry, + CompiledRelationLocation, DisplayOptions, MigrationCompatibility, MigrationDecision, @@ -15,7 +16,6 @@ format_pre_build_work, ) from sqlbuild.adapter.contract.types import MigrationTransfer -from sqlbuild.compiler.compile.models import CompiledRelationLocation def migration_entry( diff --git a/tests/unit/src/sqlbuild/compiler/planner/main/test_execution.py b/tests/unit/src/sqlbuild/compiler/planner/main/test_execution.py index af817c7f81..fadd90f751 100644 --- a/tests/unit/src/sqlbuild/compiler/planner/main/test_execution.py +++ b/tests/unit/src/sqlbuild/compiler/planner/main/test_execution.py @@ -6,6 +6,7 @@ import pytest from sqlbuild._native import ( + CompiledProject, DiscoveredHookFunction, ModelPlanEntry, PlanAction, @@ -15,7 +16,6 @@ project_with, ) from sqlbuild.adapters.duckdb.classes.duckdb_adapter import DuckDbAdapter -from sqlbuild.compiler.compile.models import CompiledProject from sqlbuild.compiler.compile.types import CompiledProjectView from tests.unit.src.sqlbuild.compiler.planner.main._test_types import ( DirectSourceFreshnessPlanOutputTestCase, diff --git a/tests/unit/src/sqlbuild/compiler/python_nodes/_helpers/helpers.py b/tests/unit/src/sqlbuild/compiler/python_nodes/_helpers/helpers.py index 7966f84e3e..fb2b8b4c07 100644 --- a/tests/unit/src/sqlbuild/compiler/python_nodes/_helpers/helpers.py +++ b/tests/unit/src/sqlbuild/compiler/python_nodes/_helpers/helpers.py @@ -11,6 +11,10 @@ from typing import Any, cast from sqlbuild._native import ( + CompiledModel, + CompiledObjectKey, + CompiledProject, + CompiledRelationLocation, CompiledSource, CompileModelConfig, DiscoveredAssetFunction, @@ -31,12 +35,6 @@ compiled_graph_keys, resolve_python_sql_run_selection, ) -from sqlbuild.compiler.compile.models import ( - CompiledModel, - CompiledObjectKey, - CompiledProject, - CompiledRelationLocation, -) from sqlbuild.compiler.compile.types import ( CompiledModelView, CompiledProjectView, diff --git a/tests/unit/src/sqlbuild/compiler/python_nodes/_helpers/test_run_lifecycle.py b/tests/unit/src/sqlbuild/compiler/python_nodes/_helpers/test_run_lifecycle.py index 35854c6f9f..4b7d565d33 100644 --- a/tests/unit/src/sqlbuild/compiler/python_nodes/_helpers/test_run_lifecycle.py +++ b/tests/unit/src/sqlbuild/compiler/python_nodes/_helpers/test_run_lifecycle.py @@ -5,12 +5,12 @@ import pytest from sqlbuild._native import ( + CompiledObjectKey, PythonNodeGraph, PythonSqlRunLifecyclePlan, PythonSqlRunSelection, build_python_sql_run_lifecycle, ) -from sqlbuild.compiler.compile.models import CompiledObjectKey from sqlbuild.compiler.compile.types import CompiledResourceType from tests.unit.src.sqlbuild.compiler.python_nodes._helpers._test_types import ( PythonSqlRunLifecycleTestCase, diff --git a/tests/unit/src/sqlbuild/compiler/python_nodes/_helpers/test_unified_selectors.py b/tests/unit/src/sqlbuild/compiler/python_nodes/_helpers/test_unified_selectors.py index 4f3f0dad32..8dd3cc597c 100644 --- a/tests/unit/src/sqlbuild/compiler/python_nodes/_helpers/test_unified_selectors.py +++ b/tests/unit/src/sqlbuild/compiler/python_nodes/_helpers/test_unified_selectors.py @@ -5,12 +5,12 @@ import pytest from sqlbuild._native import ( + CompiledObjectKey, ProjectGraph, PythonNodeGraph, PythonSqlRunSelection, resolve_project_selectors, ) -from sqlbuild.compiler.compile.models import CompiledObjectKey from sqlbuild.refs import seed from tests.unit.src.sqlbuild.compiler.python_nodes._helpers._test_types import ( PythonSqlSelectorErrorTestCase, diff --git a/tests/unit/src/sqlbuild/compiler/scopes/test_builder_and_visibility.py b/tests/unit/src/sqlbuild/compiler/scopes/test_builder_and_visibility.py index e52bbf58f7..5817aefeb3 100644 --- a/tests/unit/src/sqlbuild/compiler/scopes/test_builder_and_visibility.py +++ b/tests/unit/src/sqlbuild/compiler/scopes/test_builder_and_visibility.py @@ -25,6 +25,7 @@ EnumDeclaration, EnumMember, InaccessibleReason, + LoadedMacro, OwnershipRoot, ResourceIdentity, ResourceKind, @@ -45,7 +46,6 @@ resolve_scope_visibility, validate_scope_index, ) -from sqlbuild.compiler.compile.models import LoadedMacro from tests.unit.src.sqlbuild.compiler.scopes._test_types import ( ExpectedBooleanCase, ResourceExpectation, diff --git a/tests/unit/src/sqlbuild/compiler/source_freshness/main/helpers.py b/tests/unit/src/sqlbuild/compiler/source_freshness/main/helpers.py index e581a5d9d0..b68477b7af 100644 --- a/tests/unit/src/sqlbuild/compiler/source_freshness/main/helpers.py +++ b/tests/unit/src/sqlbuild/compiler/source_freshness/main/helpers.py @@ -6,6 +6,7 @@ from typing import Any from sqlbuild._native import ( + CompiledObjectKey, SourceEntry, SourceFreshnessConfig, SourceFreshnessIdentity, @@ -25,7 +26,6 @@ from sqlbuild.adapter.contract.types import FrameworkType from sqlbuild.adapter.relations.main.relation_lookup import build_relation_lookup from sqlbuild.adapters.duckdb.classes.duckdb_adapter import DuckDbAdapter -from sqlbuild.compiler.compile.models import CompiledObjectKey from sqlbuild.compiler.compile.types import CompiledResourceType from sqlbuild.compiler.source_freshness.constants import SOURCE_FRESHNESS_TABLE_NAME diff --git a/tests/unit/src/sqlbuild/executor/auditing/main/helpers.py b/tests/unit/src/sqlbuild/executor/auditing/main/helpers.py index 1ec5540435..9820c3f510 100644 --- a/tests/unit/src/sqlbuild/executor/auditing/main/helpers.py +++ b/tests/unit/src/sqlbuild/executor/auditing/main/helpers.py @@ -6,7 +6,12 @@ from types import SimpleNamespace from typing import Any, cast -from sqlbuild._native import AuditPlanEntry, MeasurementThresholdBound, MeasurementThresholds +from sqlbuild._native import ( + AuditPlanEntry, + CompiledObjectKey, + MeasurementThresholdBound, + MeasurementThresholds, +) from sqlbuild.compiler.auditing.types import ( AuditAttachmentKind, AuditEvaluationMode, @@ -15,7 +20,6 @@ AuditSeverity, ThresholdOperator, ) -from sqlbuild.compiler.compile.models import CompiledObjectKey from sqlbuild.compiler.compile.types import CompiledResourceType from sqlbuild.executor.auditing.main._execute import execute_audit from sqlbuild.executor.auditing.main._execute_reporting_errors import ( diff --git a/tests/unit/src/sqlbuild/executor/auditing/main/test_project_results.py b/tests/unit/src/sqlbuild/executor/auditing/main/test_project_results.py index 0e256cf66b..36fd549e5c 100644 --- a/tests/unit/src/sqlbuild/executor/auditing/main/test_project_results.py +++ b/tests/unit/src/sqlbuild/executor/auditing/main/test_project_results.py @@ -9,9 +9,8 @@ import pytest import sqlbuild.executor.auditing._helpers.result_projection as projection_module -from sqlbuild._native import PlanOutput +from sqlbuild._native import CompiledRelationLocation, PlanOutput from sqlbuild.compiler.auditing.types import AuditOutcome -from sqlbuild.compiler.compile.models import CompiledRelationLocation from sqlbuild.executor.audit_results.exceptions import AuditResultStorageError from sqlbuild.executor.audit_results.models import AuditResultRecord from sqlbuild.executor.auditing.main._project_results import project_audit_result_batch diff --git a/tests/unit/src/sqlbuild/executor/build/_helpers/helpers.py b/tests/unit/src/sqlbuild/executor/build/_helpers/helpers.py index eb4d31cf74..c5144c16ce 100644 --- a/tests/unit/src/sqlbuild/executor/build/_helpers/helpers.py +++ b/tests/unit/src/sqlbuild/executor/build/_helpers/helpers.py @@ -7,6 +7,8 @@ from unittest.mock import Mock from sqlbuild._native import ( + CompiledObjectKey, + CompiledRelationLocation, DiscoveredLoaderFunction, MaterializationType, ModelPlanEntry, @@ -27,10 +29,6 @@ AuditRunScope, AuditSeverity, ) -from sqlbuild.compiler.compile.models import ( - CompiledObjectKey, - CompiledRelationLocation, -) from sqlbuild.compiler.compile.types import CompiledResourceType from sqlbuild.executor.auditing.models import AuditExecutionResult diff --git a/tests/unit/src/sqlbuild/executor/build/_helpers/test_blocking.py b/tests/unit/src/sqlbuild/executor/build/_helpers/test_blocking.py index e8bf977cb7..385244587c 100644 --- a/tests/unit/src/sqlbuild/executor/build/_helpers/test_blocking.py +++ b/tests/unit/src/sqlbuild/executor/build/_helpers/test_blocking.py @@ -2,7 +2,7 @@ import pytest -from sqlbuild.compiler.compile.models import CompiledObjectKey +from sqlbuild._native import CompiledObjectKey from sqlbuild.executor.build._helpers.blocking import downstream_blocked_keys from tests.unit.src.sqlbuild.executor.build._helpers._test_types import ( DownstreamBlockedKeysTestCase, diff --git a/tests/unit/src/sqlbuild/executor/build/_helpers/test_external_source_loads.py b/tests/unit/src/sqlbuild/executor/build/_helpers/test_external_source_loads.py index f5f7a729b8..d50712544f 100644 --- a/tests/unit/src/sqlbuild/executor/build/_helpers/test_external_source_loads.py +++ b/tests/unit/src/sqlbuild/executor/build/_helpers/test_external_source_loads.py @@ -7,13 +7,13 @@ import pytest from sqlbuild._native import ( + CompiledObjectKey, DiscoveredLoaderFunction, LoaderConnectionMode, PlanOutput, SourceEntry, SourceLoadPlanEntry, ) -from sqlbuild.compiler.compile.models import CompiledObjectKey from sqlbuild.compiler.compile.types import CompiledResourceType from sqlbuild.executor.build._helpers.external_source_loads import ( run_external_source_loads_before_connections, diff --git a/tests/unit/src/sqlbuild/executor/build/_helpers/test_indexes.py b/tests/unit/src/sqlbuild/executor/build/_helpers/test_indexes.py index 41c85adcf4..fb682d78a3 100644 --- a/tests/unit/src/sqlbuild/executor/build/_helpers/test_indexes.py +++ b/tests/unit/src/sqlbuild/executor/build/_helpers/test_indexes.py @@ -2,13 +2,12 @@ import pytest -from sqlbuild._native import AuditPlanEntry, PlanOutput +from sqlbuild._native import AuditPlanEntry, CompiledObjectKey, PlanOutput from sqlbuild.compiler.auditing.types import ( AuditAttachmentKind, AuditRunScope, AuditSeverity, ) -from sqlbuild.compiler.compile.models import CompiledObjectKey from sqlbuild.compiler.compile.types import AttachedAuditTargetKind, CompiledResourceType from sqlbuild.executor.build._helpers.indexes import build_execution_indexes from sqlbuild.executor.build.models import BuildIndexes diff --git a/tests/unit/src/sqlbuild/executor/build/_helpers/test_retention.py b/tests/unit/src/sqlbuild/executor/build/_helpers/test_retention.py index e8d7554c65..349623b741 100644 --- a/tests/unit/src/sqlbuild/executor/build/_helpers/test_retention.py +++ b/tests/unit/src/sqlbuild/executor/build/_helpers/test_retention.py @@ -6,6 +6,7 @@ import pytest from sqlbuild._native import ( + CompiledRelationLocation, IncrementalMode, MaterializationType, ModelPlanEntry, @@ -23,7 +24,6 @@ RetentionState, ) from sqlbuild.adapter.contract.types import RetentionChangePhase, RetentionScope -from sqlbuild.compiler.compile.models import CompiledRelationLocation from sqlbuild.errors.contracts.exceptions import ExecutorInputError from sqlbuild.executor.build._helpers.retention import ( apply_retention_phase, diff --git a/tests/unit/src/sqlbuild/executor/build/_helpers/test_scheduler.py b/tests/unit/src/sqlbuild/executor/build/_helpers/test_scheduler.py index ff91e03455..45fa8be653 100644 --- a/tests/unit/src/sqlbuild/executor/build/_helpers/test_scheduler.py +++ b/tests/unit/src/sqlbuild/executor/build/_helpers/test_scheduler.py @@ -10,6 +10,7 @@ import pytest from sqlbuild._native import ( + CompiledObjectKey, DiscoveredHookFunction, DiscoveredLoaderFunction, ModelPlanEntry, @@ -24,7 +25,6 @@ ) from sqlbuild.adapter.contract.types import TablePromotionMode from sqlbuild.adapters.duckdb.classes.duckdb_adapter import DuckDbAdapter -from sqlbuild.compiler.compile.models import CompiledObjectKey from sqlbuild.compiler.compile.types import CompiledResourceType from sqlbuild.executor.build.main._execute import execute_build_plan from sqlbuild.executor.build.models import ( diff --git a/tests/unit/src/sqlbuild/executor/build/_helpers/test_source_node.py b/tests/unit/src/sqlbuild/executor/build/_helpers/test_source_node.py index 89774f9c08..669a32dc4c 100644 --- a/tests/unit/src/sqlbuild/executor/build/_helpers/test_source_node.py +++ b/tests/unit/src/sqlbuild/executor/build/_helpers/test_source_node.py @@ -6,9 +6,8 @@ import pytest -from sqlbuild._native import PlanOutput +from sqlbuild._native import CompiledObjectKey, PlanOutput from sqlbuild.adapters.duckdb.classes.duckdb_adapter import DuckDbAdapter -from sqlbuild.compiler.compile.models import CompiledObjectKey from sqlbuild.compiler.compile.types import CompiledResourceType from sqlbuild.executor.build._helpers.source_node import execute_build_source_node from sqlbuild.executor.build.models import BuildCallbacks, BuildRuntimeParams diff --git a/tests/unit/src/sqlbuild/executor/build/classes/test_build_scheduler_diagnostics.py b/tests/unit/src/sqlbuild/executor/build/classes/test_build_scheduler_diagnostics.py index bc2f4770a1..dae1f86f97 100644 --- a/tests/unit/src/sqlbuild/executor/build/classes/test_build_scheduler_diagnostics.py +++ b/tests/unit/src/sqlbuild/executor/build/classes/test_build_scheduler_diagnostics.py @@ -7,7 +7,7 @@ import pytest -from sqlbuild.compiler.compile.models import CompiledObjectKey +from sqlbuild._native import CompiledObjectKey from sqlbuild.compiler.compile.types import CompiledResourceType from sqlbuild.executor.build.classes.build_scheduler import BuildScheduler from sqlbuild.executor.build.models import BuildCallbacks, SchedulerState diff --git a/tests/unit/src/sqlbuild/executor/build/classes/test_build_scheduler_identity.py b/tests/unit/src/sqlbuild/executor/build/classes/test_build_scheduler_identity.py index 0b9f8fe5b4..c2c9626b84 100644 --- a/tests/unit/src/sqlbuild/executor/build/classes/test_build_scheduler_identity.py +++ b/tests/unit/src/sqlbuild/executor/build/classes/test_build_scheduler_identity.py @@ -9,7 +9,7 @@ import pytest -from sqlbuild.compiler.compile.models import CompiledObjectKey +from sqlbuild._native import CompiledObjectKey from sqlbuild.compiler.compile.types import CompiledResourceType from sqlbuild.executor.build.classes.build_scheduler import BuildScheduler from sqlbuild.executor.build.models import NodeCompletion diff --git a/tests/unit/src/sqlbuild/executor/build/classes/test_build_scheduler_test_order.py b/tests/unit/src/sqlbuild/executor/build/classes/test_build_scheduler_test_order.py index 079f3980e8..fdd2c9506c 100644 --- a/tests/unit/src/sqlbuild/executor/build/classes/test_build_scheduler_test_order.py +++ b/tests/unit/src/sqlbuild/executor/build/classes/test_build_scheduler_test_order.py @@ -4,8 +4,7 @@ import pytest -from sqlbuild._native import PlanOutput, SqlTestPlanEntry -from sqlbuild.compiler.compile.models import CompiledObjectKey +from sqlbuild._native import CompiledObjectKey, PlanOutput, SqlTestPlanEntry from sqlbuild.compiler.compile.types import CompiledResourceType from sqlbuild.executor.build.classes.build_scheduler import _test_result_authored_order from sqlbuild.executor.testing.models import SqlTestExecutionResult diff --git a/tests/unit/src/sqlbuild/executor/clone/_helpers/fingerprinting/helpers.py b/tests/unit/src/sqlbuild/executor/clone/_helpers/fingerprinting/helpers.py index 90a4f6e6ab..81e2534a69 100644 --- a/tests/unit/src/sqlbuild/executor/clone/_helpers/fingerprinting/helpers.py +++ b/tests/unit/src/sqlbuild/executor/clone/_helpers/fingerprinting/helpers.py @@ -6,6 +6,8 @@ import pytest from sqlbuild._native import ( + CompiledObjectKey, + CompiledRelationLocation, Fingerprint, FingerprintSet, MaterializationType, @@ -16,7 +18,6 @@ SeedPlanEntry, ) from sqlbuild.adapter.contract.models import RelationInfo -from sqlbuild.compiler.compile.models import CompiledObjectKey, CompiledRelationLocation from sqlbuild.compiler.compile.types import CompiledResourceType from sqlbuild.compiler.fingerprints.constants import FINGERPRINT_TABLE_NAME diff --git a/tests/unit/src/sqlbuild/executor/clone/_helpers/helpers.py b/tests/unit/src/sqlbuild/executor/clone/_helpers/helpers.py index f56d257012..cf53cd8e3b 100644 --- a/tests/unit/src/sqlbuild/executor/clone/_helpers/helpers.py +++ b/tests/unit/src/sqlbuild/executor/clone/_helpers/helpers.py @@ -4,6 +4,10 @@ from typing import Any, ClassVar from sqlbuild._native import ( + CompiledModel, + CompiledObjectKey, + CompiledProject, + CompiledRelationLocation, CompileModelConfig, FunctionPlanEntry, MaterializationType, @@ -15,12 +19,6 @@ ) from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter from sqlbuild.adapter.contract.models import RelationInfo -from sqlbuild.compiler.compile.models import ( - CompiledModel, - CompiledObjectKey, - CompiledProject, - CompiledRelationLocation, -) from sqlbuild.compiler.compile.types import ( CompiledModelView, CompiledProjectView, diff --git a/tests/unit/src/sqlbuild/executor/janitor/main/helpers.py b/tests/unit/src/sqlbuild/executor/janitor/main/helpers.py index d0a5035533..5e164641ad 100644 --- a/tests/unit/src/sqlbuild/executor/janitor/main/helpers.py +++ b/tests/unit/src/sqlbuild/executor/janitor/main/helpers.py @@ -10,6 +10,10 @@ from typing import Any, ClassVar, cast from sqlbuild._native import ( + CompiledModel, + CompiledObjectKey, + CompiledProject, + CompiledRelationLocation, CompiledSeed, CompiledSource, CompileModelConfig, @@ -28,12 +32,6 @@ SchemaDiffResult, ) from sqlbuild.adapter.contract.types import TablePromotionMode -from sqlbuild.compiler.compile.models import ( - CompiledModel, - CompiledObjectKey, - CompiledProject, - CompiledRelationLocation, -) from sqlbuild.compiler.compile.types import ( CompiledProjectView, CompiledResourceType, diff --git a/tests/unit/src/sqlbuild/executor/migrations/_helpers/helpers.py b/tests/unit/src/sqlbuild/executor/migrations/_helpers/helpers.py index d670813f52..d73f04aaf8 100644 --- a/tests/unit/src/sqlbuild/executor/migrations/_helpers/helpers.py +++ b/tests/unit/src/sqlbuild/executor/migrations/_helpers/helpers.py @@ -12,6 +12,7 @@ ColumnMigrationDecision, ColumnMigrationEvent, ColumnMigrationPlanEntry, + CompiledRelationLocation, MigrationDiscovery, MigrationRelation, OldNameViewAction, @@ -20,7 +21,6 @@ ) from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter from sqlbuild.adapters.duckdb.classes.duckdb_adapter import DuckDbAdapter -from sqlbuild.compiler.compile.models import CompiledRelationLocation RELATION: str = "main.fct_orders" diff --git a/tests/unit/src/sqlbuild/executor/pipeline/_helpers/helpers.py b/tests/unit/src/sqlbuild/executor/pipeline/_helpers/helpers.py index b2bb95334a..1ed9f9368a 100644 --- a/tests/unit/src/sqlbuild/executor/pipeline/_helpers/helpers.py +++ b/tests/unit/src/sqlbuild/executor/pipeline/_helpers/helpers.py @@ -10,6 +10,10 @@ from sqlbuild._native import ( AuditPlanEntry, + CompiledObjectKey, + CompiledProject, + CompiledRelationLocation, + CompiledSqlScenario, CompilePipelineResult, DiscoveredSqlScenarioFile, PlanOutput, @@ -30,12 +34,6 @@ AuditRunScope, AuditSeverity, ) -from sqlbuild.compiler.compile.models import ( - CompiledObjectKey, - CompiledProject, - CompiledRelationLocation, - CompiledSqlScenario, -) from sqlbuild.compiler.compile.types import CompiledResourceType from sqlbuild.errors.contracts.exceptions import ExecutorInputError from sqlbuild.executor.auditing.models import AuditExecutionResult diff --git a/tests/unit/src/sqlbuild/executor/pipeline/_helpers/test_auditing.py b/tests/unit/src/sqlbuild/executor/pipeline/_helpers/test_auditing.py index bc19af1caa..09d1b648da 100644 --- a/tests/unit/src/sqlbuild/executor/pipeline/_helpers/test_auditing.py +++ b/tests/unit/src/sqlbuild/executor/pipeline/_helpers/test_auditing.py @@ -21,6 +21,7 @@ from sqlbuild._native import ( AuditPlanEntry, AuditProgressReporter, + CompiledObjectKey, CostContext, PlanOutput, StatementLedgerEntry, @@ -36,7 +37,6 @@ AuditRunScope, AuditSeverity, ) -from sqlbuild.compiler.compile.models import CompiledObjectKey from sqlbuild.compiler.compile.types import AttachedAuditTargetKind, CompiledResourceType from sqlbuild.cost.models import CostResourceContext from sqlbuild.executor.auditing.models import AuditExecutionResult diff --git a/tests/unit/src/sqlbuild/executor/pipeline/_helpers/test_scenario.py b/tests/unit/src/sqlbuild/executor/pipeline/_helpers/test_scenario.py index c9362d38f9..75f64d2ecc 100644 --- a/tests/unit/src/sqlbuild/executor/pipeline/_helpers/test_scenario.py +++ b/tests/unit/src/sqlbuild/executor/pipeline/_helpers/test_scenario.py @@ -7,8 +7,12 @@ import pytest -from sqlbuild._native import CompilePipelineResult, ScenarioExecutionPlan, SqlLexicalSyntax -from sqlbuild.compiler.compile.models import CompiledSqlScenario +from sqlbuild._native import ( + CompiledSqlScenario, + CompilePipelineResult, + ScenarioExecutionPlan, + SqlLexicalSyntax, +) from sqlbuild.executor.pipeline._helpers import scenario as scenario_pipeline from sqlbuild.executor.scenario._helpers.local import execution as local_scenario from sqlbuild.executor.scenario.constants import ( diff --git a/tests/unit/src/sqlbuild/executor/pipeline/_helpers/test_testing.py b/tests/unit/src/sqlbuild/executor/pipeline/_helpers/test_testing.py index e8b2ca05ff..f3eb21e27d 100644 --- a/tests/unit/src/sqlbuild/executor/pipeline/_helpers/test_testing.py +++ b/tests/unit/src/sqlbuild/executor/pipeline/_helpers/test_testing.py @@ -7,12 +7,14 @@ import pytest -from sqlbuild._native import ChainStep, PlanOutput, SqlTestPlanEntry -from sqlbuild.adapters.duckdb.classes.duckdb_adapter import DuckDbAdapter -from sqlbuild.compiler.compile.models import ( +from sqlbuild._native import ( + ChainStep, CompiledObjectKey, CompiledRelationLocation, + PlanOutput, + SqlTestPlanEntry, ) +from sqlbuild.adapters.duckdb.classes.duckdb_adapter import DuckDbAdapter from sqlbuild.compiler.compile.types import CompiledResourceType from sqlbuild.executor.pipeline._helpers import testing as testing_pipeline from sqlbuild.executor.pipeline._helpers.testing import run_test_pipeline diff --git a/tests/unit/src/sqlbuild/executor/pipeline/main/helpers.py b/tests/unit/src/sqlbuild/executor/pipeline/main/helpers.py index 05e3e432d6..aa1c4745d4 100644 --- a/tests/unit/src/sqlbuild/executor/pipeline/main/helpers.py +++ b/tests/unit/src/sqlbuild/executor/pipeline/main/helpers.py @@ -4,6 +4,8 @@ from typing import ClassVar from sqlbuild._native import ( + CompiledObjectKey, + CompiledRelationLocation, FunctionPlanEntry, MaterializationType, ModelPlanEntry, @@ -17,7 +19,6 @@ from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter from sqlbuild.adapter.contract.classes.statement_recorder import StatementRecorder from sqlbuild.adapter.contract.models import ColumnInfo -from sqlbuild.compiler.compile.models import CompiledObjectKey, CompiledRelationLocation from sqlbuild.compiler.compile.types import CompiledResourceType, FunctionLanguage from sqlbuild.runtime.contracts.types import ExecutionResourceKind from sqlbuild.spec.contracts.constants import DEFAULT_SEED_CSV_SETTINGS diff --git a/tests/unit/src/sqlbuild/executor/pipeline/main/test_schema_preflight.py b/tests/unit/src/sqlbuild/executor/pipeline/main/test_schema_preflight.py index dc3e4c57b4..1118c704ba 100644 --- a/tests/unit/src/sqlbuild/executor/pipeline/main/test_schema_preflight.py +++ b/tests/unit/src/sqlbuild/executor/pipeline/main/test_schema_preflight.py @@ -4,8 +4,15 @@ import pytest -from sqlbuild._native import MaterializationType, ModelPlanEntry, PlanAction, PlanOutput, PlanReason -from sqlbuild.compiler.compile.models import CompiledObjectKey, CompiledRelationLocation +from sqlbuild._native import ( + CompiledObjectKey, + CompiledRelationLocation, + MaterializationType, + ModelPlanEntry, + PlanAction, + PlanOutput, + PlanReason, +) from sqlbuild.compiler.compile.types import CompiledResourceType from sqlbuild.executor.pipeline._helpers.graph_width import runnable_graph_width from sqlbuild.executor.pipeline._helpers.schema_preparation import prepare_build_schemas diff --git a/tests/unit/src/sqlbuild/executor/run/_helpers/helpers.py b/tests/unit/src/sqlbuild/executor/run/_helpers/helpers.py index cb3008e1c8..37a09440f3 100644 --- a/tests/unit/src/sqlbuild/executor/run/_helpers/helpers.py +++ b/tests/unit/src/sqlbuild/executor/run/_helpers/helpers.py @@ -4,6 +4,8 @@ from sqlbuild._native import ( AuditPlanEntry, + CompiledObjectKey, + CompiledRelationLocation, DiscoveredHookFunction, MaterializationType, ModelPlanEntry, @@ -26,10 +28,6 @@ AuditRunScope, AuditSeverity, ) -from sqlbuild.compiler.compile.models import ( - CompiledObjectKey, - CompiledRelationLocation, -) from sqlbuild.compiler.compile.types import CompiledResourceType from sqlbuild.executor.auditing.models import AuditExecutionResult from sqlbuild.executor.run._helpers.execution.hooks import execute_hooks diff --git a/tests/unit/src/sqlbuild/executor/run/_helpers/test_cursor_bounds.py b/tests/unit/src/sqlbuild/executor/run/_helpers/test_cursor_bounds.py index a97ececb65..34da88d5e4 100644 --- a/tests/unit/src/sqlbuild/executor/run/_helpers/test_cursor_bounds.py +++ b/tests/unit/src/sqlbuild/executor/run/_helpers/test_cursor_bounds.py @@ -9,6 +9,8 @@ import pytest from sqlbuild._native import ( + CompiledObjectKey, + CompiledRelationLocation, CursorBounds, CursorGrain, CursorInputRelation, @@ -29,7 +31,6 @@ from sqlbuild._native import cursor_sentinel_to_token as sentinel_to_token from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter from sqlbuild.adapters.duckdb.classes.duckdb_adapter import DuckDbAdapter -from sqlbuild.compiler.compile.models import CompiledObjectKey, CompiledRelationLocation from sqlbuild.compiler.compile.types import CompiledResourceType from sqlbuild.errors.contracts.exceptions import ExecutorInputError from sqlbuild.executor.run._helpers.validation.cursor_bounds import ( diff --git a/tests/unit/src/sqlbuild/executor/run/_helpers/test_hooks.py b/tests/unit/src/sqlbuild/executor/run/_helpers/test_hooks.py index 9a37f1cbf1..970a227712 100644 --- a/tests/unit/src/sqlbuild/executor/run/_helpers/test_hooks.py +++ b/tests/unit/src/sqlbuild/executor/run/_helpers/test_hooks.py @@ -8,6 +8,7 @@ import pytest from sqlbuild._native import ( + CompiledRelationLocation, CostContext, DiscoveredHookFunction, ModelPlanEntry, @@ -16,7 +17,6 @@ ) from sqlbuild.adapter.contract.classes.statement_recorder import StatementRecorder from sqlbuild.adapters.duckdb.classes.duckdb_adapter import DuckDbAdapter -from sqlbuild.compiler.compile.models import CompiledRelationLocation from sqlbuild.cost.models import CostResourceContext from sqlbuild.errors.contracts.exceptions import ExecutorInputError from sqlbuild.executor.run._helpers.execution.hooks import execute_hooks, render_hooks diff --git a/tests/unit/src/sqlbuild/executor/run/_helpers/test_snapshot_adapter_boundary.py b/tests/unit/src/sqlbuild/executor/run/_helpers/test_snapshot_adapter_boundary.py index b17ded6151..d9c4caf2ae 100644 --- a/tests/unit/src/sqlbuild/executor/run/_helpers/test_snapshot_adapter_boundary.py +++ b/tests/unit/src/sqlbuild/executor/run/_helpers/test_snapshot_adapter_boundary.py @@ -9,6 +9,8 @@ import pytest from sqlbuild._native import ( + CompiledObjectKey, + CompiledRelationLocation, HistoricalInput, MaterializationType, ModelPlanEntry, @@ -16,7 +18,6 @@ PlanReason, ) from sqlbuild.adapters.duckdb.classes.duckdb_adapter import DuckDbAdapter -from sqlbuild.compiler.compile.models import CompiledObjectKey, CompiledRelationLocation from sqlbuild.compiler.compile.types import CompiledResourceType from sqlbuild.executor.run._helpers.materializations.snapshot import execute_snapshot_entry from sqlbuild.executor.run.models import ModelExecutionResult, ModelMaterializationContext diff --git a/tests/unit/src/sqlbuild/executor/scenario/_helpers/helpers.py b/tests/unit/src/sqlbuild/executor/scenario/_helpers/helpers.py index 2c74f5b417..8d88deaef8 100644 --- a/tests/unit/src/sqlbuild/executor/scenario/_helpers/helpers.py +++ b/tests/unit/src/sqlbuild/executor/scenario/_helpers/helpers.py @@ -5,6 +5,8 @@ from typing import cast from sqlbuild._native import ( + CompiledObjectKey, + CompiledRelationLocation, MaterializationType, ModelPlanEntry, PlanAction, @@ -19,10 +21,6 @@ ScenarioRelationPlan, SeedPlanEntry, ) -from sqlbuild.compiler.compile.models import ( - CompiledObjectKey, - CompiledRelationLocation, -) from sqlbuild.compiler.compile.types import CompiledResourceType from sqlbuild.executor.scenario.models import ( ScenarioSnapshotCaptureRunResult, diff --git a/tests/unit/src/sqlbuild/executor/scenario/main/helpers.py b/tests/unit/src/sqlbuild/executor/scenario/main/helpers.py index 07d230a5f5..16e0263698 100644 --- a/tests/unit/src/sqlbuild/executor/scenario/main/helpers.py +++ b/tests/unit/src/sqlbuild/executor/scenario/main/helpers.py @@ -5,6 +5,8 @@ from typing import Any, ClassVar from sqlbuild._native import ( + CompiledObjectKey, + CompiledRelationLocation, MaterializationType, ModelPlanEntry, PlanAction, @@ -21,10 +23,6 @@ from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter from sqlbuild.adapter.contract.classes.statement_recorder import StatementRecorder from sqlbuild.adapter.contract.models import ColumnInfo, QueryResult, RelationInfo -from sqlbuild.compiler.compile.models import ( - CompiledObjectKey, - CompiledRelationLocation, -) from sqlbuild.compiler.compile.types import CompiledResourceType from sqlbuild.spec.contracts.constants import DEFAULT_SEED_CSV_SETTINGS diff --git a/tests/unit/src/sqlbuild/executor/seed/main/helpers.py b/tests/unit/src/sqlbuild/executor/seed/main/helpers.py index 2804a83c52..1f2a173ddf 100644 --- a/tests/unit/src/sqlbuild/executor/seed/main/helpers.py +++ b/tests/unit/src/sqlbuild/executor/seed/main/helpers.py @@ -4,9 +4,13 @@ from pathlib import Path -from sqlbuild._native import SeedCsvSettings, SeedPlanEntry +from sqlbuild._native import ( + CompiledObjectKey, + CompiledRelationLocation, + SeedCsvSettings, + SeedPlanEntry, +) from sqlbuild.adapter.contract.models import ColumnInfo -from sqlbuild.compiler.compile.models import CompiledObjectKey, CompiledRelationLocation from sqlbuild.compiler.compile.types import CompiledResourceType diff --git a/tests/unit/src/sqlbuild/executor/testing/main/helpers.py b/tests/unit/src/sqlbuild/executor/testing/main/helpers.py index 6d6495a4d8..2dc0788dde 100644 --- a/tests/unit/src/sqlbuild/executor/testing/main/helpers.py +++ b/tests/unit/src/sqlbuild/executor/testing/main/helpers.py @@ -1,10 +1,9 @@ -from sqlbuild._native import ChainStep, SqlTestAssertionStep, SqlTestPlanEntry +from sqlbuild._native import ChainStep, CompiledObjectKey, SqlTestAssertionStep, SqlTestPlanEntry from sqlbuild.adapters.bigquery.classes.bigquery_adapter import BigQueryAdapter from sqlbuild.adapters.databricks.classes.databricks_adapter import DatabricksAdapter from sqlbuild.adapters.duckdb.classes.duckdb_adapter import DuckDbAdapter from sqlbuild.adapters.snowflake.classes.snowflake_adapter import SnowflakeAdapter from sqlbuild.adapters.sqlserver.classes.sqlserver_adapter import SqlServerAdapter -from sqlbuild.compiler.compile.models import CompiledObjectKey from sqlbuild.compiler.compile.types import CompiledResourceType diff --git a/tests/unit/src/sqlbuild/integrations/dbt/helpers.py b/tests/unit/src/sqlbuild/integrations/dbt/helpers.py index 42c0c11e9f..9d88c15421 100644 --- a/tests/unit/src/sqlbuild/integrations/dbt/helpers.py +++ b/tests/unit/src/sqlbuild/integrations/dbt/helpers.py @@ -11,7 +11,10 @@ from sqlbuild._native import ( BackfillAction, BackfillResult, + CompiledObjectKey, + CompiledRelationLocation, CompileModelConfig, + CompileModelInput, CompileProjectInputs, DbtCliConfigOverrides, DbtCliOptions, @@ -36,11 +39,6 @@ extract_sql_references, transitive_closure, ) -from sqlbuild.compiler.compile.models import ( - CompiledObjectKey, - CompiledRelationLocation, - CompileModelInput, -) from sqlbuild.compiler.compile.types import CompiledProjectView, CompiledResourceType _GENERIC_SQL_SYNTAX: SqlLexicalSyntax = SqlLexicalSyntax() diff --git a/tests/unit/src/sqlbuild/integrations/dbt/test_execute.py b/tests/unit/src/sqlbuild/integrations/dbt/test_execute.py index 0709a6e627..da5187c81c 100644 --- a/tests/unit/src/sqlbuild/integrations/dbt/test_execute.py +++ b/tests/unit/src/sqlbuild/integrations/dbt/test_execute.py @@ -13,6 +13,7 @@ from sqlbuild import _native as phases_module from sqlbuild import _native as plan_module from sqlbuild._native import ( + CompiledProject, DbtCliOptions, DbtCombinedGraph, DbtCommandExecutionResult, @@ -35,7 +36,6 @@ execute_dbt_commands, ) from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter -from sqlbuild.compiler.compile.models import CompiledProject from tests.unit.src.sqlbuild.integrations.dbt._test_types import ( DbtCompileFullRefreshPipelineTestCase, DbtExecutionSelectionStatusTestCase, diff --git a/tests/unit/src/sqlbuild/rule_engine/_helpers/engine/helpers.py b/tests/unit/src/sqlbuild/rule_engine/_helpers/engine/helpers.py index 031847235e..35f0234893 100644 --- a/tests/unit/src/sqlbuild/rule_engine/_helpers/engine/helpers.py +++ b/tests/unit/src/sqlbuild/rule_engine/_helpers/engine/helpers.py @@ -13,6 +13,8 @@ from sqlbuild._native import ( CompiledDirectLogicSqlTestPayload, CompiledModelSqlTestPayload, + CompiledObjectKey, + CompiledSqlTest, CompiledSqlTestResource, CompileSqlTestCte, CustomRulesOutcome, @@ -30,7 +32,6 @@ ) from sqlbuild._native import evaluate_custom_rules as evaluate_custom_rules_cached from sqlbuild._native import evaluate_project_rules as evaluate -from sqlbuild.compiler.compile.models import CompiledObjectKey, CompiledSqlTest from sqlbuild.compiler.compile.types import CompiledProjectView, CompiledResourceType, SqlTestMode from sqlbuild.rule_engine.constants import MIN_CUSTOM_RULE_TEST_CASES from sqlbuild.rule_engine.models import Finding, Rule, RulesConfig, RulesResult diff --git a/tests/unit/src/sqlbuild/rule_engine/_helpers/engine/test_custom_rules.py b/tests/unit/src/sqlbuild/rule_engine/_helpers/engine/test_custom_rules.py index 4a6aafef9b..10f47a4863 100644 --- a/tests/unit/src/sqlbuild/rule_engine/_helpers/engine/test_custom_rules.py +++ b/tests/unit/src/sqlbuild/rule_engine/_helpers/engine/test_custom_rules.py @@ -6,6 +6,7 @@ import pytest from sqlbuild._native import ( + CompactLineageFacts, RuleExemption, RuleIgnore, RulesError, @@ -13,7 +14,6 @@ project_with, ) from sqlbuild._native import evaluate_project_rules as evaluate -from sqlbuild.compiler.compile.models import CompactLineageFacts from sqlbuild.compiler.compile.types import CompiledProjectView from sqlbuild.rule_engine.models import RulesConfig, RulesResult from tests.unit.src.sqlbuild.rule_engine._helpers.engine._test_types import ( diff --git a/tests/unit/src/sqlbuild/rule_engine/_helpers/engine/test_native_facts.py b/tests/unit/src/sqlbuild/rule_engine/_helpers/engine/test_native_facts.py index fd679a27f9..92cac5b2f2 100644 --- a/tests/unit/src/sqlbuild/rule_engine/_helpers/engine/test_native_facts.py +++ b/tests/unit/src/sqlbuild/rule_engine/_helpers/engine/test_native_facts.py @@ -4,13 +4,11 @@ import pytest from sqlbuild._native import ( - DiscoveredSqlScenarioFile, - project_with, -) -from sqlbuild.compiler.compile.models import ( CompiledObjectKey, CompiledSqlScenario, CompiledSqlTest, + DiscoveredSqlScenarioFile, + project_with, ) from sqlbuild.compiler.compile.types import CompiledProjectView, CompiledResourceType, SqlTestMode from tests.unit.src.sqlbuild.rule_engine._helpers.engine._test_types import ( diff --git a/tests/unit/src/sqlbuild/rule_engine/classes/audit_facts/helpers.py b/tests/unit/src/sqlbuild/rule_engine/classes/audit_facts/helpers.py index 52e4b5abfa..d3ce2e7515 100644 --- a/tests/unit/src/sqlbuild/rule_engine/classes/audit_facts/helpers.py +++ b/tests/unit/src/sqlbuild/rule_engine/classes/audit_facts/helpers.py @@ -1,7 +1,11 @@ from pathlib import Path -from sqlbuild._native import DiscoveredAuditBlock, DiscoveredAuditFile -from sqlbuild.compiler.compile.models import CompiledAudit, CompiledObjectKey +from sqlbuild._native import ( + CompiledAudit, + CompiledObjectKey, + DiscoveredAuditBlock, + DiscoveredAuditFile, +) from sqlbuild.compiler.compile.types import CompiledAuditView, CompiledResourceType diff --git a/tests/unit/src/sqlbuild/rule_engine/classes/model_test_facts/test_model_test_facts.py b/tests/unit/src/sqlbuild/rule_engine/classes/model_test_facts/test_model_test_facts.py index 2db8ccba43..4325436ffd 100644 --- a/tests/unit/src/sqlbuild/rule_engine/classes/model_test_facts/test_model_test_facts.py +++ b/tests/unit/src/sqlbuild/rule_engine/classes/model_test_facts/test_model_test_facts.py @@ -6,9 +6,9 @@ import pytest from sqlbuild._native import ( + CompiledSqlTest, project_with, ) -from sqlbuild.compiler.compile.models import CompiledSqlTest from sqlbuild.compiler.compile.types import CompiledProjectView, SqlTestMode from sqlbuild.rule_engine.classes.model_test_facts import TestFacts as ModelTestFacts from sqlbuild.rule_engine.models import Model diff --git a/tests/unit/src/sqlbuild/rule_engine/classes/shared_fact_encoder/helpers.py b/tests/unit/src/sqlbuild/rule_engine/classes/shared_fact_encoder/helpers.py index f43d38b810..ab577d63c1 100644 --- a/tests/unit/src/sqlbuild/rule_engine/classes/shared_fact_encoder/helpers.py +++ b/tests/unit/src/sqlbuild/rule_engine/classes/shared_fact_encoder/helpers.py @@ -1,12 +1,12 @@ from pathlib import Path from sqlbuild._native import ( + CompiledObjectKey, CompiledSource, DiscoveredSourceFile, SourceEntry, project_with, ) -from sqlbuild.compiler.compile.models import CompiledObjectKey from sqlbuild.compiler.compile.types import ( CompiledProjectView, CompiledResourceType, diff --git a/tests/unit/src/sqlbuild/rule_engine/main/evaluate/_test_types.py b/tests/unit/src/sqlbuild/rule_engine/main/evaluate/_test_types.py index dda4c72d3f..19f198096b 100644 --- a/tests/unit/src/sqlbuild/rule_engine/main/evaluate/_test_types.py +++ b/tests/unit/src/sqlbuild/rule_engine/main/evaluate/_test_types.py @@ -2,7 +2,7 @@ from dataclasses import dataclass, field -from sqlbuild.compiler.compile.models import CompileSqlReference +from sqlbuild._native import CompileSqlReference from sqlbuild.rule_engine.models import RulesConfig diff --git a/tests/unit/src/sqlbuild/rule_engine/main/evaluate/helpers.py b/tests/unit/src/sqlbuild/rule_engine/main/evaluate/helpers.py index 49fc43cf74..4c888d539f 100644 --- a/tests/unit/src/sqlbuild/rule_engine/main/evaluate/helpers.py +++ b/tests/unit/src/sqlbuild/rule_engine/main/evaluate/helpers.py @@ -2,13 +2,15 @@ from pathlib import Path -from sqlbuild._native import CompileModelConfig, EnumDeclaration, EnumMember -from sqlbuild.compiler.compile.models import ( +from sqlbuild._native import ( CompiledModel, CompiledObjectKey, CompiledProject, CompiledRelationLocation, + CompileModelConfig, CompileSqlReference, + EnumDeclaration, + EnumMember, ) from sqlbuild.compiler.compile.types import ( CompiledModelView, diff --git a/tests/unit/src/sqlbuild/rule_engine/main/evaluate/test_evaluate.py b/tests/unit/src/sqlbuild/rule_engine/main/evaluate/test_evaluate.py index 15e0370529..9105fac042 100644 --- a/tests/unit/src/sqlbuild/rule_engine/main/evaluate/test_evaluate.py +++ b/tests/unit/src/sqlbuild/rule_engine/main/evaluate/test_evaluate.py @@ -5,9 +5,8 @@ import pytest -from sqlbuild._native import SelectStarAllow +from sqlbuild._native import CompileSqlReference, SelectStarAllow from sqlbuild._native import evaluate_project_rules as evaluate -from sqlbuild.compiler.compile.models import CompileSqlReference from sqlbuild.rule_engine.models import Finding, RulesConfig, RulesResult from tests.unit.src.sqlbuild.rule_engine.main.evaluate._test_types import ( PolicyEvaluationTestCase, From 2e9fdb1faa0c3bf0847250da1477827ad6e58480 Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Thu, 15 Oct 2026 18:07:23 +0100 Subject: [PATCH 06/48] refactor: host custom-rule records natively and move Rust-only rule constants to Rust --- .../src/hosted_shapes/constants.rs | 83 +++- .../_helpers/compile_driver/pipeline.rs | 2 +- .../_helpers/compiler_enums/constants.rs | 6 + .../bindings/_helpers/plan_command/rules.rs | 2 +- .../_helpers/record_behaviours/constants.rs | 24 +- .../_helpers/record_behaviours/mod.rs | 1 + .../_helpers/record_behaviours/rule_shapes.rs | 190 +++++++++ .../_helpers/rule_records/constants.rs | 253 +++++++++++ .../_helpers/rules/builtin_catalogue.rs | 4 +- .../src/bindings/_helpers/rules/constants.rs | 8 + .../_helpers/rules/custom_evaluation.rs | 3 +- .../bindings/_helpers/rules/custom_hosts.rs | 10 +- .../src/bindings/_helpers/rules/findings.rs | 2 +- .../bindings/_helpers/rules/rule_dispatch.rs | 11 +- .../src/bindings/_helpers/rules/rule_plan.rs | 6 +- .../src/bindings/_helpers/rules/rule_run.rs | 2 +- .../bindings/_helpers/rules/rules_config.rs | 2 +- .../src/bindings/_helpers/rules/sql_rules.rs | 2 +- python-boundary.toml | 2 +- .../_helpers/varied_benchmark.py | 9 +- src/sqlbuild/_native.pyi | 393 ++++++++++++++++++ .../_helpers/engine/_custom_catalogue.py | 3 +- .../_helpers/engine/custom_payloads.py | 2 +- .../rule_engine/_helpers/engine/definition.py | 5 +- .../_helpers/engine/fact_replay.py | 3 +- .../_helpers/engine/hermeticity.py | 3 +- .../_helpers/engine/rule_harness.py | 12 +- .../_helpers/host/custom_evaluation.py | 27 +- .../rule_engine/_helpers/host/custom_host.py | 19 +- .../rule_engine/_helpers/host/module_state.py | 8 +- .../_helpers/host/tracked_facts.py | 5 +- .../rule_engine/classes/audit_facts.py | 2 +- .../rule_engine/classes/column_facts.py | 3 +- .../rule_engine/classes/contract_facts.py | 2 +- .../rule_engine/classes/fact_reads.py | 2 +- .../rule_engine/classes/graph_facts.py | 2 +- .../rule_engine/classes/model_test_facts.py | 3 +- .../classes/module_state_monitor.py | 3 +- .../rule_engine/classes/project_facts.py | 2 +- .../rule_engine/classes/project_tree.py | 3 +- .../rule_engine/classes/rule_context.py | 22 +- .../rule_engine/classes/sql_document.py | 3 +- src/sqlbuild/rule_engine/classes/sql_facts.py | 3 +- src/sqlbuild/rule_engine/constants.py | 4 - src/sqlbuild/rule_engine/main/define.py | 2 +- .../rule_engine/main/evaluate_rule.py | 2 +- src/sqlbuild/rule_engine/models.py | 334 --------------- src/sqlbuild/rule_engine/types.py | 16 +- src/sqlbuild/rules/__init__.py | 6 +- src/sqlbuild/rules/testing.py | 2 +- .../compile/test_dense_compile_performance.py | 9 +- .../rule_engine/host_payload/helpers.py | 2 +- .../test_sql_rule_cache_publication.py | 3 +- .../rule_engine/_helpers/engine/helpers.py | 5 +- .../_helpers/engine/test_catalogue.py | 4 +- .../_helpers/engine/test_config.py | 10 +- .../engine/test_custom_rule_evidence.py | 4 +- .../_helpers/engine/test_custom_rules.py | 3 +- .../_helpers/engine/test_definition.py | 3 +- .../_helpers/engine/test_hermeticity.py | 3 +- .../engine/test_incremental_native_rules.py | 3 +- .../_helpers/engine/test_native_memo.py | 2 +- .../rule_engine/_helpers/host/helpers.py | 3 +- .../_helpers/host/test_custom_host_pool.py | 5 +- .../_helpers/host/test_module_state.py | 12 + .../_helpers/run/test_native_rows.py | 3 +- .../classes/audit_facts/test_audit_facts.py | 2 +- .../model_test_facts/test_model_test_facts.py | 2 +- .../rule_engine/main/evaluate/_test_types.py | 3 +- .../evaluate/test_cache_and_suppressions.py | 3 +- .../main/evaluate/test_evaluate.py | 3 +- .../main/evaluate_rule/test_evaluate_rule.py | 3 +- .../main/render_result/_test_types.py | 2 +- .../main/render_result/test_render_result.py | 2 +- .../rule_engine/main/skills/test_skills.py | 3 +- 75 files changed, 1122 insertions(+), 493 deletions(-) create mode 100644 crates/sqlbuild-python/src/bindings/_helpers/record_behaviours/rule_shapes.rs delete mode 100644 src/sqlbuild/rule_engine/models.py diff --git a/crates/sqlbuild-cache/src/hosted_shapes/constants.rs b/crates/sqlbuild-cache/src/hosted_shapes/constants.rs index e29f5f9244..8f30546304 100644 --- a/crates/sqlbuild-cache/src/hosted_shapes/constants.rs +++ b/crates/sqlbuild-cache/src/hosted_shapes/constants.rs @@ -3,7 +3,7 @@ /// The extension module hosting the shapes. pub const NATIVE_MODULE: &str = "sqlbuild._native"; /// The Python model type a natively hosted compiler shape is captured as. -pub const HOSTED_SHAPE_TYPES: [(&str, &str); 470] = [ +pub const HOSTED_SHAPE_TYPES: [(&str, &str); 491] = [ ( "sqlbuild._native:AlignedInterval", "sqlbuild.cursor_algebra.models:AlignedInterval", @@ -328,6 +328,18 @@ pub const HOSTED_SHAPE_TYPES: [(&str, &str); 470] = [ "sqlbuild._native:CursorsConfig", "sqlbuild.spec.contracts.models:CursorsConfig", ), + ( + "sqlbuild._native:CustomHostPartition", + "sqlbuild.rule_engine.models:CustomHostPartition", + ), + ( + "sqlbuild._native:CustomRuleEvaluation", + "sqlbuild.rule_engine.models:CustomRuleEvaluation", + ), + ( + "sqlbuild._native:CustomRuleRun", + "sqlbuild.rule_engine.models:CustomRuleRun", + ), ( "sqlbuild._native:CustomRulesOutcome", "sqlbuild.rule_engine.models:CustomRulesOutcome", @@ -688,6 +700,10 @@ pub const HOSTED_SHAPE_TYPES: [(&str, &str); 470] = [ "sqlbuild._native:FactDigestError", "sqlbuild.rule_engine.exceptions:FactDigestError", ), + ( + "sqlbuild._native:Finding", + "sqlbuild.rule_engine.models:Finding", + ), ( "sqlbuild._native:Fingerprint", "sqlbuild.compiler.fingerprints.models:Fingerprint", @@ -828,6 +844,10 @@ pub const HOSTED_SHAPE_TYPES: [(&str, &str); 470] = [ "sqlbuild._native:JanitorConfig", "sqlbuild.spec.contracts.models:JanitorConfig", ), + ( + "sqlbuild._native:LayoutConfig", + "sqlbuild.rule_engine.models:LayoutConfig", + ), ( "sqlbuild._native:LifecycleEventSinkFilterConfig", "sqlbuild.spec.contracts.models:LifecycleEventSinkFilterConfig", @@ -1014,6 +1034,10 @@ pub const HOSTED_SHAPE_TYPES: [(&str, &str); 470] = [ "sqlbuild._native:MissingMigrationOriginPolicy", "sqlbuild.spec.contracts.types:MissingMigrationOriginPolicy", ), + ( + "sqlbuild._native:Model", + "sqlbuild.rule_engine.models:Model", + ), ( "sqlbuild._native:ModelColumnLineage", "sqlbuild.compiler.lineage.models:ModelColumnLineage", @@ -1038,6 +1062,10 @@ pub const HOSTED_SHAPE_TYPES: [(&str, &str); 470] = [ "sqlbuild._native:ModelSchemaDeclaration", "sqlbuild.compiler.discovery.models:ModelSchemaDeclaration", ), + ( + "sqlbuild._native:ModelSql", + "sqlbuild.rule_engine.models:ModelSql", + ), ( "sqlbuild._native:ModelSqlAnalysis", "sqlbuild.compiler.compile.models:ModelSqlAnalysis", @@ -1050,6 +1078,14 @@ pub const HOSTED_SHAPE_TYPES: [(&str, &str); 470] = [ "sqlbuild._native:ModelValidationRequest", "sqlbuild.compiler.compile.models:ModelValidationRequest", ), + ( + "sqlbuild._native:ModuleStateFingerprint", + "sqlbuild.rule_engine.models:ModuleStateFingerprint", + ), + ( + "sqlbuild._native:ModuleStateTrace", + "sqlbuild.rule_engine.models:ModuleStateTrace", + ), ( "sqlbuild._native:MovePreview", "sqlbuild.compiler.scopes.models:MovePreview", @@ -1242,6 +1278,10 @@ pub const HOSTED_SHAPE_TYPES: [(&str, &str); 470] = [ "sqlbuild._native:PreparedSqlLintResult", "sqlbuild.rule_engine.models:PreparedSqlLintResult", ), + ( + "sqlbuild._native:Project", + "sqlbuild.rule_engine.models:Project", + ), ( "sqlbuild._native:ProjectCompileError", "sqlbuild.lint.exceptions:ProjectCompileError", @@ -1258,6 +1298,10 @@ pub const HOSTED_SHAPE_TYPES: [(&str, &str); 470] = [ "sqlbuild._native:ProjectGraph", "sqlbuild.compiler.pipeline.models:ProjectGraph", ), + ( + "sqlbuild._native:ProjectPath", + "sqlbuild.rule_engine.models:ProjectPath", + ), ( "sqlbuild._native:ProjectPathError", "sqlbuild.compiler.discovery.exceptions:ProjectPathError", @@ -1394,14 +1438,27 @@ pub const HOSTED_SHAPE_TYPES: [(&str, &str); 470] = [ "sqlbuild._native:RetentionPlanPhase", "sqlbuild.compiler.planner.types:RetentionPlanPhase", ), + ("sqlbuild._native:Rule", "sqlbuild.rule_engine.models:Rule"), ( "sqlbuild._native:RuleAssertionError", "sqlbuild.rule_engine.exceptions:RuleAssertionError", ), + ( + "sqlbuild._native:RuleCase", + "sqlbuild.rule_engine.models:RuleCase", + ), ( "sqlbuild._native:RuleExemption", "sqlbuild.rule_engine.models:RuleExemption", ), + ( + "sqlbuild._native:RuleFactViews", + "sqlbuild.rule_engine.models:RuleFactViews", + ), + ( + "sqlbuild._native:RuleFile", + "sqlbuild.rule_engine.models:RuleFile", + ), ( "sqlbuild._native:RuleFixResult", "sqlbuild.lint.models:RuleFixResult", @@ -1414,10 +1471,22 @@ pub const HOSTED_SHAPE_TYPES: [(&str, &str); 470] = [ "sqlbuild._native:RuleIgnore", "sqlbuild.rule_engine.models:RuleIgnore", ), + ( + "sqlbuild._native:RuleOption", + "sqlbuild.rule_engine.models:RuleOption", + ), + ( + "sqlbuild._native:RuleResult", + "sqlbuild.rule_engine.models:RuleResult", + ), ( "sqlbuild._native:RuleRunPlan", "sqlbuild.rule_engine.models:RuleRunPlan", ), + ( + "sqlbuild._native:RuleSubject", + "sqlbuild.rule_engine.types:RuleSubject", + ), ( "sqlbuild._native:RuleUsageError", "sqlbuild.rule_engine.exceptions:RuleUsageError", @@ -1426,6 +1495,10 @@ pub const HOSTED_SHAPE_TYPES: [(&str, &str); 470] = [ "sqlbuild._native:RulesCacheConfig", "sqlbuild.rule_engine.models:RulesCacheConfig", ), + ( + "sqlbuild._native:RulesConfig", + "sqlbuild.rule_engine.models:RulesConfig", + ), ( "sqlbuild._native:RulesConfigError", "sqlbuild.rule_engine.exceptions:RulesConfigError", @@ -1434,6 +1507,10 @@ pub const HOSTED_SHAPE_TYPES: [(&str, &str); 470] = [ "sqlbuild._native:RulesError", "sqlbuild.rule_engine.exceptions:RulesError", ), + ( + "sqlbuild._native:RulesResult", + "sqlbuild.rule_engine.models:RulesResult", + ), ( "sqlbuild._native:RulesRunResult", "sqlbuild.rule_engine.models:RulesRunResult", @@ -1710,6 +1787,10 @@ pub const HOSTED_SHAPE_TYPES: [(&str, &str); 470] = [ "sqlbuild._native:SqlLogicalType", "sqlbuild.sql_values.models:SqlLogicalType", ), + ( + "sqlbuild._native:SqlNode", + "sqlbuild.rule_engine.models:SqlNode", + ), ( "sqlbuild._native:SqlReferenceKind", "sqlbuild.compiler.references.types:SqlReferenceKind", diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/pipeline.rs b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/pipeline.rs index 0b9ccb6019..4f929f4271 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/pipeline.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/pipeline.rs @@ -28,7 +28,7 @@ const ANALYSIS_SELECTION_MODULE: &str = "sqlbuild._native"; const TIMING_TRACKER_MODULE: &str = "sqlbuild._native"; const RULE_EXCEPTIONS_MODULE: &str = "sqlbuild._native"; const RULE_THREADS_MODULE: &str = "sqlbuild._native"; -const RULE_MODELS_MODULE: &str = "sqlbuild.rule_engine.models"; +const RULE_MODELS_MODULE: &str = "sqlbuild._native"; const RUNTIME_MODELS_MODULE: &str = "sqlbuild.runtime.contracts.models"; const GENERIC_DIALECT: &str = "generic"; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compiler_enums/constants.rs b/crates/sqlbuild-python/src/bindings/_helpers/compiler_enums/constants.rs index cd6583d2f6..b9f26149d2 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compiler_enums/constants.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compiler_enums/constants.rs @@ -698,6 +698,11 @@ const E_LOADER_CONNECTION_MODE: EnumSpec = EnumSpec { doc: "How a source loader owns the destination warehouse connection.", members: &[("SQLBUILD", "sqlbuild"), ("EXTERNAL", "external")], }; +const E_RULE_SUBJECT: EnumSpec = EnumSpec { + name: "RuleSubject", + doc: "Compiler-owned unit used to invoke and cache one custom rule.", + members: &[("MODEL", "model"), ("PROJECT", "project")], +}; /// Compiler, rule, lint and config string enums. pub(crate) const COMPILER_ENUMS: &[EnumSpec] = &[ @@ -771,4 +776,5 @@ pub(crate) const COMPILER_ENUMS: &[EnumSpec] = &[ E_SQL_REFERENCE_KIND, E_HARD_CODED_RELATION_OWNER_KIND, E_LOADER_CONNECTION_MODE, + E_RULE_SUBJECT, ]; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/plan_command/rules.rs b/crates/sqlbuild-python/src/bindings/_helpers/plan_command/rules.rs index bfb27544ff..d26831217b 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/plan_command/rules.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/plan_command/rules.rs @@ -10,7 +10,7 @@ use crate::bindings::_helpers::command_runtime::adapters::resolve_cli_adapter; use crate::bindings::_helpers::compile_driver::python_calls::{call_with, imported}; const NATIVE_MODULE: &str = "sqlbuild._native"; -const RULE_MODELS: &str = "sqlbuild.rule_engine.models"; +const RULE_MODELS: &str = "sqlbuild._native"; const REJECT_UNIT_TESTS_MODULE: &str = "sqlbuild._native"; const GRAPH_SELECTION_MODULE: &str = "sqlbuild._native"; const LIST_ACTION: &str = "list"; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/record_behaviours/constants.rs b/crates/sqlbuild-python/src/bindings/_helpers/record_behaviours/constants.rs index eb6fa71ad2..ecdc5270ee 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/record_behaviours/constants.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/record_behaviours/constants.rs @@ -60,6 +60,11 @@ use crate::bindings::_helpers::record_behaviours::references::{ reference_example_call, reference_fixture_cte_prefix, reference_function_name, reference_placeholder_call, }; +use crate::bindings::_helpers::record_behaviours::rule_shapes::{ + project_path_depth, project_path_name, project_path_parts, rule_option_boolean, + rule_option_class_getitem, rule_option_integer, rule_option_integer_list, rule_option_string, + rule_option_string_list, rule_project_wide, rule_result_finding_count, +}; use crate::bindings::_helpers::record_behaviours::scopes::{ completeness_complete, target_query_unknown, }; @@ -250,6 +255,11 @@ pub(crate) const BEHAVIOUR_PROPERTIES: &[(&str, &str, ClassProperty)] = &[ "output_column_count", model_output_column_count, ), + ("ProjectPath", "parts", project_path_parts), + ("ProjectPath", "name", project_path_name), + ("ProjectPath", "depth", project_path_depth), + ("Rule", "project_wide", rule_project_wide), + ("RuleResult", "finding_count", rule_result_finding_count), ]; /// Computed attributes of compiler shapes that a record computes once and keeps. pub(crate) const BEHAVIOUR_CACHED_PROPERTIES: &[(&str, &str, ClassProperty)] = &[ @@ -319,9 +329,19 @@ pub(crate) const BEHAVIOUR_METHODS: &[(&str, &str, ClassMethod)] = &[ ("CompactLineageFacts", "__iter__", facts_iter), ("CompactLineageFacts", "__eq__", facts_eq), ("CompactLineageFacts", "resource_name", facts_resource_name), + ("RuleOption", "__class_getitem__", rule_option_class_getitem), + ("RuleOption", "boolean", rule_option_boolean), + ("RuleOption", "integer", rule_option_integer), + ("RuleOption", "string", rule_option_string), + ("RuleOption", "string_list", rule_option_string_list), + ("RuleOption", "integer_list", rule_option_integer_list), ]; /// Compiler records registered as `collections.abc.Sequence` virtual subclasses. pub(crate) const BEHAVIOUR_SEQUENCES: &[&str] = &["CompactLineageFacts"]; /// Compiler records ordered by their compared fields. -pub(crate) const BEHAVIOUR_ORDERED: &[&str] = - &["ResourceIdentity", "DeclarationIdentity", "OwnershipRoot"]; +pub(crate) const BEHAVIOUR_ORDERED: &[&str] = &[ + "ResourceIdentity", + "DeclarationIdentity", + "OwnershipRoot", + "ProjectPath", +]; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/record_behaviours/mod.rs b/crates/sqlbuild-python/src/bindings/_helpers/record_behaviours/mod.rs index b8853c66d4..76a8bf272e 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/record_behaviours/mod.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/record_behaviours/mod.rs @@ -15,5 +15,6 @@ pub(crate) mod migrations; pub(crate) mod planner; pub(crate) mod python_nodes; pub(crate) mod references; +pub(crate) mod rule_shapes; pub(crate) mod scopes; pub(crate) mod values; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/record_behaviours/rule_shapes.rs b/crates/sqlbuild-python/src/bindings/_helpers/record_behaviours/rule_shapes.rs new file mode 100644 index 0000000000..a535e8be08 --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/record_behaviours/rule_shapes.rs @@ -0,0 +1,190 @@ +//! Derived views and constructors of the natively hosted custom-rule records. + +use pyo3::IntoPyObjectExt; +use pyo3::exceptions::PyTypeError; +use pyo3::prelude::{Bound, Py, PyAny, PyAnyMethods, PyResult, Python}; +use pyo3::types::{PyDict, PyDictMethods, PyTuple, PyTupleMethods}; + +use crate::bindings::_helpers::compile_driver::python_calls::imported; +use crate::bindings::_helpers::records::constants::NATIVE_MODULE; + +/// Keyword fields every `RuleOption` constructor requires. +const REQUIRED_OPTION_FIELDS: [&str; 3] = ["name", "default", "description"]; + +/// `pathlib.Path(value)` of a `ProjectPath`. +fn project_path<'py>(instance: &Bound<'py, PyAny>) -> PyResult> { + imported(instance.py(), "pathlib", "Path")?.call1((instance.getattr("value")?,)) +} + +/// `ProjectPath.parts`. +pub(crate) fn project_path_parts(instance: &Bound<'_, PyAny>) -> PyResult> { + Ok(project_path(instance)?.getattr("parts")?.unbind()) +} + +/// `ProjectPath.name`. +pub(crate) fn project_path_name(instance: &Bound<'_, PyAny>) -> PyResult> { + Ok(project_path(instance)?.getattr("name")?.unbind()) +} + +/// `ProjectPath.depth`: the number of path parts. +pub(crate) fn project_path_depth(instance: &Bound<'_, PyAny>) -> PyResult> { + project_path(instance)? + .getattr("parts")? + .len()? + .into_py_any(instance.py()) +} + +/// `Rule.project_wide`: whether the rule runs once for the compiled project. +pub(crate) fn rule_project_wide(instance: &Bound<'_, PyAny>) -> PyResult> { + let project: Bound<'_, PyAny> = + imported(instance.py(), NATIVE_MODULE, "RuleSubject")?.getattr("PROJECT")?; + instance + .getattr("subject")? + .is(&project) + .into_py_any(instance.py()) +} + +/// `RuleResult.finding_count`. +pub(crate) fn rule_result_finding_count(instance: &Bound<'_, PyAny>) -> PyResult> { + instance + .getattr("findings")? + .len()? + .into_py_any(instance.py()) +} + +/// `RuleOption[T]`: the generic alias annotations use. +pub(crate) fn rule_option_class_getitem( + receiver: &Bound<'_, PyAny>, + args: &Bound<'_, PyTuple>, + _kwargs: Option<&Bound<'_, PyDict>>, +) -> PyResult> { + let py: Python<'_> = receiver.py(); + let class: Bound<'_, PyAny> = imported(py, NATIVE_MODULE, "RuleOption")?; + Ok(imported(py, "types", "GenericAlias")? + .call1((class, args.get_item(0)?))? + .unbind()) +} + +/// A `RuleOption` of `value_type` from the keyword fields named in `names`, plus `extra`. +fn rule_option( + (args, kwargs): (&Bound<'_, PyTuple>, Option<&Bound<'_, PyDict>>), + (value_type, names): (Bound<'_, PyAny>, &[&str]), + extra: &[(&str, Bound<'_, PyAny>)], +) -> PyResult> { + let py: Python<'_> = value_type.py(); + if !args.is_empty() { + return Err(PyTypeError::new_err( + "RuleOption constructors take keyword arguments only", + )); + } + let fields: Bound<'_, PyDict> = PyDict::new(py); + fields.set_item("value_type", &value_type)?; + if let Some(kwargs) = kwargs { + for (name, value) in kwargs.iter() { + let name: String = name.extract()?; + if !names.contains(&name.as_str()) { + return Err(PyTypeError::new_err(format!( + "RuleOption constructor got an unexpected keyword argument '{name}'" + ))); + } + fields.set_item(name, value)?; + } + } + for name in REQUIRED_OPTION_FIELDS { + if !fields.contains(name)? { + return Err(PyTypeError::new_err(format!( + "RuleOption constructor missing required keyword-only argument: '{name}'" + ))); + } + } + for (name, value) in extra { + fields.set_item(*name, value)?; + } + Ok(imported(py, NATIVE_MODULE, "RuleOption")? + .call((), Some(&fields))? + .unbind()) +} + +/// The builtin type `name`. +fn builtin<'py>(py: Python<'py>, name: &str) -> PyResult> { + imported(py, "builtins", name) +} + +/// `RuleOption.boolean(*, name, default, description)`. +pub(crate) fn rule_option_boolean( + receiver: &Bound<'_, PyAny>, + args: &Bound<'_, PyTuple>, + kwargs: Option<&Bound<'_, PyDict>>, +) -> PyResult> { + let value_type: Bound<'_, PyAny> = builtin(receiver.py(), "bool")?; + rule_option( + (args, kwargs), + (value_type, &["name", "default", "description"]), + &[], + ) +} + +/// `RuleOption.integer(*, name, default, description, minimum=None, maximum=None)`. +pub(crate) fn rule_option_integer( + receiver: &Bound<'_, PyAny>, + args: &Bound<'_, PyTuple>, + kwargs: Option<&Bound<'_, PyDict>>, +) -> PyResult> { + let value_type: Bound<'_, PyAny> = builtin(receiver.py(), "int")?; + rule_option( + (args, kwargs), + ( + value_type, + &["name", "default", "description", "minimum", "maximum"], + ), + &[], + ) +} + +/// `RuleOption.string(*, name, default, description, choices=())`. +pub(crate) fn rule_option_string( + receiver: &Bound<'_, PyAny>, + args: &Bound<'_, PyTuple>, + kwargs: Option<&Bound<'_, PyDict>>, +) -> PyResult> { + let value_type: Bound<'_, PyAny> = builtin(receiver.py(), "str")?; + rule_option( + (args, kwargs), + (value_type, &["name", "default", "description", "choices"]), + &[], + ) +} + +/// `RuleOption.string_list(*, name, default, description, minimum_items=None)`. +pub(crate) fn rule_option_string_list( + receiver: &Bound<'_, PyAny>, + args: &Bound<'_, PyTuple>, + kwargs: Option<&Bound<'_, PyDict>>, +) -> PyResult> { + let py: Python<'_> = receiver.py(); + rule_option( + (args, kwargs), + ( + builtin(py, "tuple")?, + &["name", "default", "description", "minimum_items"], + ), + &[("item_type", builtin(py, "str")?)], + ) +} + +/// `RuleOption.integer_list(*, name, default, description, minimum_items=None)`. +pub(crate) fn rule_option_integer_list( + receiver: &Bound<'_, PyAny>, + args: &Bound<'_, PyTuple>, + kwargs: Option<&Bound<'_, PyDict>>, +) -> PyResult> { + let py: Python<'_> = receiver.py(); + rule_option( + (args, kwargs), + ( + builtin(py, "tuple")?, + &["name", "default", "description", "minimum_items"], + ), + &[("item_type", builtin(py, "int")?)], + ) +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/rule_records/constants.rs b/crates/sqlbuild-python/src/bindings/_helpers/rule_records/constants.rs index 9ba81b59be..7eb07bb0da 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/rule_records/constants.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/rule_records/constants.rs @@ -155,6 +155,239 @@ const R_CUSTOM_RULES_OUTCOME: RecordSpec = RecordSpec { FieldSpec::new("custom_cpu_ms", FieldDefault::Int(0)), ], }; +const R_MODEL: RecordSpec = RecordSpec { + name: "Model", + doc: "Stable identity and common metadata for one compiled SQL model.", + fields: &[ + FieldSpec::required("name"), + FieldSpec::required("path"), + FieldSpec::required("materialization"), + ], +}; +const R_PROJECT: RecordSpec = RecordSpec { + name: "Project", + doc: "Stable summary for one compiled project invocation.", + fields: &[ + FieldSpec::required("model_count"), + FieldSpec::required("target_name"), + ], +}; +const R_PROJECT_PATH: RecordSpec = RecordSpec { + name: "ProjectPath", + doc: "One normalized project-relative file or directory.", + fields: &[ + FieldSpec::required("value"), + FieldSpec::required("is_file"), + FieldSpec::required("is_dir"), + ], +}; +const R_SQL_NODE: RecordSpec = RecordSpec { + name: "SqlNode", + doc: "Stable source location for one common SQL structure fact.", + fields: &[ + FieldSpec::required("kind"), + FieldSpec::required("line"), + FieldSpec::required("column"), + ], +}; +const R_MODEL_SQL: RecordSpec = RecordSpec { + name: "ModelSql", + doc: "Authored and compiler-expanded SQL documents for one model.", + fields: &[ + FieldSpec::required("authored"), + FieldSpec::required("expanded"), + ], +}; +const R_RULE_FACT_VIEWS: RecordSpec = RecordSpec { + name: "RuleFactViews", + doc: "Views shared by every invocation in one custom-rule host evaluation.", + fields: &[ + FieldSpec::required("project"), + FieldSpec::required("sql"), + FieldSpec::required("graph"), + FieldSpec::required("columns"), + FieldSpec::required("contracts"), + FieldSpec::required("tests"), + FieldSpec::required("audits"), + FieldSpec::required("declarations"), + ], +}; +const R_RULE_OPTION: RecordSpec = RecordSpec { + name: "RuleOption", + doc: "One typed configuration option declared by a rule.", + fields: &[ + FieldSpec::required("name"), + FieldSpec::required("value_type"), + FieldSpec::required("default"), + FieldSpec::required("description"), + FieldSpec::new("choices", FieldDefault::EmptyTuple), + FieldSpec::new("minimum", FieldDefault::None), + FieldSpec::new("maximum", FieldDefault::None), + FieldSpec::new("minimum_items", FieldDefault::None), + FieldSpec::new("item_type", FieldDefault::None), + ], +}; +const R_RULE: RecordSpec = RecordSpec { + name: "Rule", + doc: "Immutable metadata and implementation for one rule.", + fields: &[ + FieldSpec::required("code"), + FieldSpec::required("family"), + FieldSpec::required("slug"), + FieldSpec::required("message"), + FieldSpec::required("remediation"), + FieldSpec::required("check"), + FieldSpec::new("options", FieldDefault::EmptyTuple), + FieldSpec::new("enabled_by_default", FieldDefault::False), + FieldSpec::new("custom", FieldDefault::False), + FieldSpec::new("source", FieldDefault::None), + FieldSpec::new("subject", FieldDefault::None), + FieldSpec::new("subject_parameter", FieldDefault::None), + FieldSpec::new("context_parameter", FieldDefault::None), + FieldSpec::new("guidance", FieldDefault::None), + ], +}; +const R_FINDING: RecordSpec = RecordSpec { + name: "Finding", + doc: "One deterministic compiler rules violation.", + fields: &[ + FieldSpec::required("code"), + FieldSpec::required("path"), + FieldSpec::required("line"), + FieldSpec::required("column"), + FieldSpec::required("message"), + FieldSpec::required("remediation"), + FieldSpec::new("unevaluated", FieldDefault::False), + FieldSpec::new("affected_rules", FieldDefault::EmptyTuple), + FieldSpec::new("fixable", FieldDefault::False), + ], +}; +const R_LAYOUT_CONFIG: RecordSpec = RecordSpec { + name: "LayoutConfig", + doc: "", + fields: &[ + FieldSpec::new( + "levels", + FieldDefault::TextTuple(&[ + "staging", + "intermediate/clean", + "intermediate/enriched", + "mart", + ]), + ), + FieldSpec::new("domain_roots", FieldDefault::EmptyTuple), + ], +}; +const R_RULES_CONFIG: RecordSpec = RecordSpec { + name: "RulesConfig", + doc: "", + fields: &[ + FieldSpec::new("select", FieldDefault::EmptyTuple), + FieldSpec::new("ignore", FieldDefault::EmptyTuple), + FieldSpec::new("thresholds", FieldDefault::EmptyDict), + FieldSpec::new("threshold_overrides", FieldDefault::EmptyTuple), + FieldSpec::new("rule_options", FieldDefault::EmptyDict), + FieldSpec::new("rule_exceptions", FieldDefault::EmptyTuple), + FieldSpec::new("graph_edge_exceptions", FieldDefault::EmptyTuple), + FieldSpec::new("rule_ignores", FieldDefault::EmptyTuple), + FieldSpec::new("allow_exceptions", FieldDefault::True), + FieldSpec::new("select_star_allow", FieldDefault::EmptyTuple), + FieldSpec::new("domains", FieldDefault::EmptyTuple), + FieldSpec::new("approved_source_tokens", FieldDefault::EmptyTuple), + FieldSpec::new("retired_source_tokens", FieldDefault::EmptyDict), + FieldSpec::new( + "sql_tests", + FieldDefault::Call("sqlbuild._native", "SqlTestRulesConfig"), + ), + FieldSpec::new( + "layout", + FieldDefault::Call("sqlbuild._native", "LayoutConfig"), + ), + FieldSpec::new( + "cache", + FieldDefault::Call("sqlbuild._native", "RulesCacheConfig"), + ), + ], +}; +const R_RULES_RESULT: RecordSpec = RecordSpec { + name: "RulesResult", + doc: "", + fields: &[ + FieldSpec::required("findings"), + FieldSpec::required("evaluated_models"), + FieldSpec::new("unevaluated_resources", FieldDefault::Int(0)), + FieldSpec::new("cache_hits", FieldDefault::Int(0)), + FieldSpec::new("cache_misses", FieldDefault::Int(0)), + FieldSpec::new("built_in_ms", FieldDefault::Int(0)), + FieldSpec::new("custom_ms", FieldDefault::Int(0)), + FieldSpec::new("custom_cpu_ms", FieldDefault::Int(0)), + ], +}; +const R_RULE_FILE: RecordSpec = RecordSpec { + name: "RuleFile", + doc: "", + fields: &[FieldSpec::required("path"), FieldSpec::required("source")], +}; +const R_RULE_CASE: RecordSpec = RecordSpec { + name: "RuleCase", + doc: "", + fields: &[ + FieldSpec::required("description"), + FieldSpec::required("source"), + FieldSpec::required("expected_finding_count"), + FieldSpec::new( + "path", + FieldDefault::Text("models/mart/example__mart__result.sql"), + ), + FieldSpec::new("files", FieldDefault::EmptyTuple), + FieldSpec::new("config", FieldDefault::EmptyDict), + ], +}; +const R_RULE_RESULT: RecordSpec = RecordSpec { + name: "RuleResult", + doc: "", + fields: &[FieldSpec::required("findings")], +}; +const R_CUSTOM_RULE_EVALUATION: RecordSpec = RecordSpec { + name: "CustomRuleEvaluation", + doc: "Findings and observed fact reads of one rule invocation for one subject.", + fields: &[ + FieldSpec::required("code"), + FieldSpec::required("subject"), + FieldSpec::required("findings"), + FieldSpec::required("reads"), + ], +}; +const R_MODULE_STATE_TRACE: RecordSpec = RecordSpec { + name: "ModuleStateTrace", + doc: "What exact module-state walks touched: object and type identities and their decisions.", + fields: &[ + FieldSpec::required("touched"), + FieldSpec::required("descended_types"), + FieldSpec::required("checked_instances"), + ], +}; +const R_MODULE_STATE_FINGERPRINT: RecordSpec = RecordSpec { + name: "ModuleStateFingerprint", + doc: "One exact custom-rule module-state fingerprint and the trace of the walk behind it.", + fields: &[FieldSpec::required("token"), FieldSpec::required("trace")], +}; +const R_CUSTOM_RULE_RUN: RecordSpec = RecordSpec { + name: "CustomRuleRun", + doc: "Every invocation of one host run plus rules whose reads could not be attributed.", + fields: &[ + FieldSpec::required("evaluations"), + FieldSpec::required("untracked_codes"), + FieldSpec::new("uncacheable_codes", FieldDefault::EmptyFrozenset), + FieldSpec::new("observed", FieldDefault::EmptyTuple), + FieldSpec::new("stateful_codes", FieldDefault::EmptyFrozenset), + ], +}; +const R_CUSTOM_HOST_PARTITION: RecordSpec = RecordSpec { + name: "CustomHostPartition", + doc: "How one host of a split run detects module state and learns the run was cancelled.", + fields: &[FieldSpec::required("cancelled")], +}; /// The rule engine value records. pub(crate) const RULE_RECORDS: &[RecordSpec] = &[ @@ -174,4 +407,24 @@ pub(crate) const RULE_RECORDS: &[RecordSpec] = &[ R_RULES_RUN_RESULT, R_NATIVE_RULES_EVALUATION, R_CUSTOM_RULES_OUTCOME, + R_MODEL, + R_PROJECT, + R_PROJECT_PATH, + R_SQL_NODE, + R_MODEL_SQL, + R_RULE_FACT_VIEWS, + R_RULE_OPTION, + R_RULE, + R_FINDING, + R_LAYOUT_CONFIG, + R_RULES_CONFIG, + R_RULES_RESULT, + R_RULE_FILE, + R_RULE_CASE, + R_RULE_RESULT, + R_CUSTOM_RULE_EVALUATION, + R_MODULE_STATE_TRACE, + R_MODULE_STATE_FINGERPRINT, + R_CUSTOM_RULE_RUN, + R_CUSTOM_HOST_PARTITION, ]; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/rules/builtin_catalogue.rs b/crates/sqlbuild-python/src/bindings/_helpers/rules/builtin_catalogue.rs index aef30876a5..3a0cfaa261 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/rules/builtin_catalogue.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/rules/builtin_catalogue.rs @@ -12,8 +12,8 @@ use sqlbuild_rules::rules::models::{BuiltinOption, BuiltinOptionKind}; use crate::bindings::_helpers::boundary::panics::compiler_guard; -const MODELS_MODULE: &str = "sqlbuild.rule_engine.models"; -const TYPES_MODULE: &str = "sqlbuild.rule_engine.types"; +const MODELS_MODULE: &str = "sqlbuild._native"; +const TYPES_MODULE: &str = "sqlbuild._native"; const FAMILY_SUFFIX_LENGTH: usize = 3; fn keywords<'py>( diff --git a/crates/sqlbuild-python/src/bindings/_helpers/rules/constants.rs b/crates/sqlbuild-python/src/bindings/_helpers/rules/constants.rs index d535039dd1..1a0e8670cf 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/rules/constants.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/rules/constants.rs @@ -16,3 +16,11 @@ pub(crate) const SAFE_RULE_FIX_CODES: [&str; 8] = [ pub(crate) const RULE_FIX_AVAILABLE_NOTE: &str = "Fix available: `sqb format --fix` applies it after compiler verification"; pub(crate) const RULES_NATIVE_API_VERSION: i64 = 1; +/// File name of the custom-rule result cache under the project cache directory. +pub(crate) const CUSTOM_RULES_CACHE_FILE: &str = "custom-rules.json"; +/// Suffix of a custom host scratch directory abandoned by a cancelled run. +pub(crate) const CUSTOM_HOST_ABANDONED_SUFFIX: &str = ".abandoned"; +/// Marker file whose presence tells split custom hosts the run was cancelled. +pub(crate) const CUSTOM_HOST_CANCELLED_MARKER: &str = "cancelled"; +/// Rules that need proven column types and are skipped without SQL analysis. +pub(crate) const TYPE_PROOF_RULE_CODES: [&str; 1] = ["SQBRCONTRACT105"]; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/rules/custom_evaluation.rs b/crates/sqlbuild-python/src/bindings/_helpers/rules/custom_evaluation.rs index 991da1aefa..5b7dd1d2a5 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/rules/custom_evaluation.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/rules/custom_evaluation.rs @@ -19,6 +19,7 @@ use sqlbuild_rules::custom_rule_cache::types::ReadSet; use crate::bindings::_helpers::cache::native_store::current_code_identity; use crate::bindings::_helpers::compile_driver::python_calls::{call_with, imported}; +use crate::bindings::_helpers::rules::constants::CUSTOM_RULES_CACHE_FILE; use crate::bindings::_helpers::rules::custom_hosts::{ HostCancel, HostRequest, host_payload_project, run_custom_hosts, }; @@ -112,7 +113,7 @@ pub(crate) fn evaluate_custom_rules_cached<'py>( ("project_dir", run.project_dir.clone()), ( "file_name", - imported(py, RULES_CONSTANTS_MODULE, "CUSTOM_RULES_CACHE_FILE")?, + pyo3::types::PyString::new(py, CUSTOM_RULES_CACHE_FILE).into_any(), ), ], )? diff --git a/crates/sqlbuild-python/src/bindings/_helpers/rules/custom_hosts.rs b/crates/sqlbuild-python/src/bindings/_helpers/rules/custom_hosts.rs index fa99e84757..59ddcd9a3c 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/rules/custom_hosts.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/rules/custom_hosts.rs @@ -26,6 +26,9 @@ use sqlbuild_rules::custom_host_pool::types::{HostInvocation, HostPlan}; use sqlbuild_rules::engine::main::custom_host::run_custom_host_json; use crate::bindings::_helpers::compile_driver::python_calls::imported; +use crate::bindings::_helpers::rules::constants::{ + CUSTOM_HOST_ABANDONED_SUFFIX, CUSTOM_HOST_CANCELLED_MARKER, +}; use crate::bindings::_helpers::rules::custom_host_observation::{ LaunchLog, ObservedHosts, before_publish, observed_hosts, record_launch, }; @@ -118,10 +121,7 @@ pub(crate) fn run_custom_hosts( let input_dir: PathBuf = input_dir.canonicalize()?; let stem: String = format!("project-{}", uuid_hex(py)?); let input_path: PathBuf = input_dir.join(format!("{stem}.pickle")); - let abandoned: PathBuf = input_dir.join(format!( - "{stem}{}", - constant::(py, "CUSTOM_HOST_ABANDONED_SUFFIX")? - )); + let abandoned: PathBuf = input_dir.join(format!("{stem}{}", CUSTOM_HOST_ABANDONED_SUFFIX)); let mut payload: Map = Map::new(); payload.insert("project_pickle_path".to_owned(), json!(text(&input_path))); payload.insert("abandoned_marker".to_owned(), json!(text(&abandoned))); @@ -221,7 +221,7 @@ fn run_partitions<'py>( .collect(); } let cancel_dir: PathBuf = unique_directory(input_dir, "hosts-")?; - let marker: PathBuf = cancel_dir.join(constant::(py, "CUSTOM_HOST_CANCELLED_MARKER")?); + let marker: PathBuf = cancel_dir.join(CUSTOM_HOST_CANCELLED_MARKER); let specs: Vec = plans .iter() .map(|partition| launch.spec(partition, Some(&marker))) diff --git a/crates/sqlbuild-python/src/bindings/_helpers/rules/findings.rs b/crates/sqlbuild-python/src/bindings/_helpers/rules/findings.rs index 361917fa49..2fd567c7ae 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/rules/findings.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/rules/findings.rs @@ -5,7 +5,7 @@ use pyo3::types::{PyDict, PyDictMethods}; use serde_json::Value; use sqlbuild_rules::models::Fault; -const MODELS_MODULE: &str = "sqlbuild.rule_engine.models"; +const MODELS_MODULE: &str = "sqlbuild._native"; const FINDING_CLASS: &str = "Finding"; const PATHLIB_MODULE: &str = "pathlib"; const PATH_CLASS: &str = "Path"; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/rules/rule_dispatch.rs b/crates/sqlbuild-python/src/bindings/_helpers/rules/rule_dispatch.rs index 0d6f2b81a1..3dbea99f83 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/rules/rule_dispatch.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/rules/rule_dispatch.rs @@ -18,6 +18,7 @@ use crate::bindings::_helpers::cache::native_store::current_code_identity; use crate::bindings::_helpers::compile_driver::python_calls::{ KeywordArguments, call_with, imported, within, }; +use crate::bindings::_helpers::rules::constants::TYPE_PROOF_RULE_CODES; use crate::bindings::_helpers::rules::custom_evaluation::{ CustomRun, evaluate_custom_rules_cached, outcome, }; @@ -26,8 +27,7 @@ use crate::bindings::_helpers::rules::custom_hosts::HostCancel; const NATIVE_RECORDS_MODULE: &str = "sqlbuild._native"; const NATIVE_MODULE: &str = "sqlbuild._native"; -const RULES_MODELS_MODULE: &str = "sqlbuild.rule_engine.models"; -const RULES_CONSTANTS_MODULE: &str = "sqlbuild.rule_engine.constants"; +const RULES_MODELS_MODULE: &str = "sqlbuild._native"; const CUSTOM_PAYLOADS_MODULE: &str = "sqlbuild.rule_engine._helpers.engine.custom_payloads"; const HERMETICITY_MODULE: &str = "sqlbuild.rule_engine._helpers.engine.hermeticity"; const CUSTOM_THREAD: &str = "sqlbuild-custom-rules"; @@ -532,11 +532,14 @@ fn native_request<'py>( )?, }; let mut include_type_proof: bool = false; - for code in imported(py, RULES_CONSTANTS_MODULE, "TYPE_PROOF_RULE_CODES")?.try_iter()? { + for code in TYPE_PROOF_RULE_CODES { include_type_proof = include_type_proof || call_with( &native(py, "rule_selected")?, - &[("config", run.config.clone()), ("code", code?)], + &[ + ("config", run.config.clone()), + ("code", pyo3::types::PyString::new(py, code).into_any()), + ], )? .is_truthy()?; } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/rules/rule_plan.rs b/crates/sqlbuild-python/src/bindings/_helpers/rules/rule_plan.rs index b7f202a1c5..9efa1652f1 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/rules/rule_plan.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/rules/rule_plan.rs @@ -10,13 +10,13 @@ use crate::bindings::_helpers::rules::catalogue::{ references_custom_rules, resolved_ignore_selectors, rule_catalogue, select_rules, verify_custom_rules, }; +use crate::bindings::_helpers::rules::constants::TYPE_PROOF_RULE_CODES; use crate::bindings::_helpers::rules::rule_run::freed_memory_released; const NATIVE_RECORDS_MODULE: &str = "sqlbuild._native"; const NATIVE_MODULE: &str = "sqlbuild._native"; const CUSTOM_PAYLOADS_MODULE: &str = "sqlbuild.rule_engine._helpers.engine.custom_payloads"; -const RULES_CONSTANTS_MODULE: &str = "sqlbuild.rule_engine.constants"; const COMPILE_TYPES_MODULE: &str = "sqlbuild.compiler.compile.types"; const DATACLASSES_MODULE: &str = "dataclasses"; const PATHLIB_MODULE: &str = "pathlib"; @@ -57,12 +57,10 @@ fn plan_rule_run<'py>(options: Option<&Bound<'py, PyDict>>) -> PyResult = - imported(py, RULES_CONSTANTS_MODULE, "TYPE_PROOF_RULE_CODES")?; let mut kept: Vec> = Vec::with_capacity(selected.len()); for rule in selected { let code: Bound<'py, PyAny> = rule.getattr("code")?; - if type_proof.contains(&code)? { + if TYPE_PROOF_RULE_CODES.contains(&code.extract::()?.as_str()) { skipped_type_proof_rules.push(code); } else { kept.push(rule); diff --git a/crates/sqlbuild-python/src/bindings/_helpers/rules/rule_run.rs b/crates/sqlbuild-python/src/bindings/_helpers/rules/rule_run.rs index f5de4a16c1..02667e3faf 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/rules/rule_run.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/rules/rule_run.rs @@ -17,7 +17,7 @@ use crate::bindings::_helpers::rules::rows::finalize_rule_findings_rows; const NATIVE_RECORDS_MODULE: &str = "sqlbuild._native"; -const MODELS_MODULE: &str = "sqlbuild.rule_engine.models"; +const MODELS_MODULE: &str = "sqlbuild._native"; const PATHLIB_MODULE: &str = "pathlib"; const CTYPES_MODULE: &str = "ctypes"; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/rules/rules_config.rs b/crates/sqlbuild-python/src/bindings/_helpers/rules/rules_config.rs index 196cf383c5..b6c0de35f1 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/rules/rules_config.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/rules/rules_config.rs @@ -14,7 +14,7 @@ use sqlbuild_rules::models::RulesConfig; use crate::bindings::_helpers::boundary::panics::compiler_guard; -const MODELS_MODULE: &str = "sqlbuild.rule_engine.models"; +const MODELS_MODULE: &str = "sqlbuild._native"; const ERRORS_MODULE: &str = "sqlbuild._native"; const NATIVE_MODULE: &str = "sqlbuild._native"; /// The rules configuration models that stay Python for the custom-rule API. diff --git a/crates/sqlbuild-python/src/bindings/_helpers/rules/sql_rules.rs b/crates/sqlbuild-python/src/bindings/_helpers/rules/sql_rules.rs index 6c483d569b..d8cf5699fb 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/rules/sql_rules.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/rules/sql_rules.rs @@ -18,7 +18,7 @@ use crate::bindings::_helpers::rules::early_sql_lint::{sql_lint_config, sql_rule const NATIVE_RECORDS_MODULE: &str = "sqlbuild._native"; const NATIVE_MODULE: &str = "sqlbuild._native"; -const RULES_MODELS_MODULE: &str = "sqlbuild.rule_engine.models"; +const RULES_MODELS_MODULE: &str = "sqlbuild._native"; const LINT_EXCEPTIONS_MODULE: &str = "sqlbuild._native"; const DATACLASSES_MODULE: &str = "dataclasses"; const JSON_MODULE: &str = "json"; diff --git a/python-boundary.toml b/python-boundary.toml index 80fd15ec88..a60d534a55 100644 --- a/python-boundary.toml +++ b/python-boundary.toml @@ -2,7 +2,7 @@ # migration work that moves to Rust. `remaining_lines` must equal the current count of that work: # a PR that moves code lowers it, and adding movable Python fails the check. # Native failures must surface as errors; `handled_native_errors` lists the reviewed exceptions. -remaining_lines = 791 +remaining_lines = 439 [[stays]] path = "executor" diff --git a/scripts/cold_compile_performance/_helpers/varied_benchmark.py b/scripts/cold_compile_performance/_helpers/varied_benchmark.py index 651959d413..78c4a05c6e 100644 --- a/scripts/cold_compile_performance/_helpers/varied_benchmark.py +++ b/scripts/cold_compile_performance/_helpers/varied_benchmark.py @@ -7,8 +7,13 @@ from scripts.cold_compile_performance._helpers.dense_project import dense_model_name from scripts.cold_compile_performance._helpers.varied_project import write_varied_compile_project from scripts.cold_compile_performance.exceptions import CompileBenchmarkFixtureError -from sqlbuild._native import build_rule_catalogue, load_rules_config, select_catalogue_rules -from sqlbuild.rule_engine.models import Rule, RulesConfig +from sqlbuild._native import ( + Rule, + RulesConfig, + build_rule_catalogue, + load_rules_config, + select_catalogue_rules, +) from tests.e2e.src.sqlbuild.cli.commands.main.compile._test_types import VariedCompileCacheTestCase from tests.e2e.src.sqlbuild.cli.commands.main.compile.helpers import ( _RETAINED_BENCHMARK_PAYLOAD_KEYS, diff --git a/src/sqlbuild/_native.pyi b/src/sqlbuild/_native.pyi index f6a757f77a..bf902c774e 100644 --- a/src/sqlbuild/_native.pyi +++ b/src/sqlbuild/_native.pyi @@ -18101,3 +18101,396 @@ class CompiledSqlTest(NativeRecord): target_model_names: tuple[str, ...] = ..., tested_resources: tuple[CompiledSqlTestResource, ...] = ..., ) -> None: ... + +class RuleSubject(StrEnum): + """Compiler-owned unit used to invoke and cache one custom rule.""" + + MODEL = "model" + PROJECT = "project" + +class Model(NativeRecord): + """Stable identity and common metadata for one compiled SQL model.""" + + name: str + path: Path + materialization: str | None + def __init__( + self, + name: str, + path: Path, + materialization: str | None, + ) -> None: ... + +class Project(NativeRecord): + """Stable summary for one compiled project invocation.""" + + model_count: int + target_name: str | None + def __init__( + self, + model_count: int, + target_name: str | None, + ) -> None: ... + +class ProjectPath(NativeRecord): + """One normalized project-relative file or directory.""" + + value: str + is_file: bool + is_dir: bool + def __init__( + self, + value: str, + is_file: bool, + is_dir: bool, + ) -> None: ... + @property + def parts(self) -> tuple[str, ...]: ... + @property + def name(self) -> str: ... + @property + def depth(self) -> int: ... + +class SqlNode(NativeRecord): + """Stable source location for one common SQL structure fact.""" + + kind: str + line: int + column: int + def __init__( + self, + kind: str, + line: int, + column: int, + ) -> None: ... + +class ModelSql(NativeRecord): + """Authored and compiler-expanded SQL documents for one model.""" + + authored: Any + expanded: Any + def __init__( + self, + authored: Any, + expanded: Any, + ) -> None: ... + +class RuleFactViews(NativeRecord): + """Views shared by every invocation in one custom-rule host evaluation.""" + + project: Any + sql: Any + graph: Any + columns: Any + contracts: Any + tests: Any + audits: Any + declarations: Any + def __init__( + self, + project: Any, + sql: Any, + graph: Any, + columns: Any, + contracts: Any, + tests: Any, + audits: Any, + declarations: Any, + ) -> None: ... + +class RuleOption[T](NativeRecord): + """One typed configuration option declared by a rule.""" + + @property + def name(self) -> str: ... + @property + def value_type(self) -> type[T]: ... + @property + def default(self) -> T: ... + @property + def description(self) -> str: ... + @property + def choices(self) -> tuple[T, ...]: ... + @property + def minimum(self) -> int | None: ... + @property + def maximum(self) -> int | None: ... + @property + def minimum_items(self) -> int | None: ... + @property + def item_type(self) -> type[object] | None: ... + def __init__( + self, + name: str, + value_type: type[T], + default: T, + description: str, + choices: tuple[T, ...] = ..., + minimum: int | None = ..., + maximum: int | None = ..., + minimum_items: int | None = ..., + item_type: type[object] | None = ..., + ) -> None: ... + @classmethod + def boolean(cls, *, name: str, default: bool, description: str) -> RuleOption[bool]: ... + @classmethod + def integer( + cls, + *, + name: str, + default: int, + description: str, + minimum: int | None = None, + maximum: int | None = None, + ) -> RuleOption[int]: ... + @classmethod + def string( + cls, *, name: str, default: str, description: str, choices: tuple[str, ...] = () + ) -> RuleOption[str]: ... + @classmethod + def string_list( + cls, + *, + name: str, + default: tuple[str, ...], + description: str, + minimum_items: int | None = None, + ) -> RuleOption[tuple[str, ...]]: ... + @classmethod + def integer_list( + cls, + *, + name: str, + default: tuple[int, ...], + description: str, + minimum_items: int | None = None, + ) -> RuleOption[tuple[int, ...]]: ... + +class Rule(NativeRecord): + """Immutable metadata and implementation for one rule.""" + + code: str + family: str + slug: str + message: str + remediation: str + check: RuleCheck + options: tuple[RuleOption[object], ...] + enabled_by_default: bool + custom: bool + source: str | None + subject: RuleSubject | None + subject_parameter: str | None + context_parameter: str | None + guidance: RuleGuidance | None + def __init__( + self, + code: str, + family: str, + slug: str, + message: str, + remediation: str, + check: RuleCheck, + options: tuple[RuleOption[object], ...] = ..., + enabled_by_default: bool = ..., + custom: bool = ..., + source: str | None = ..., + subject: RuleSubject | None = ..., + subject_parameter: str | None = ..., + context_parameter: str | None = ..., + guidance: RuleGuidance | None = ..., + ) -> None: ... + @property + def project_wide(self) -> bool: ... + +class Finding(NativeRecord): + """One deterministic compiler rules violation.""" + + code: str + path: Path + line: int + column: int + message: str + remediation: str + unevaluated: bool + affected_rules: tuple[str, ...] + fixable: bool + def __init__( + self, + code: str, + path: Path, + line: int, + column: int, + message: str, + remediation: str, + unevaluated: bool = ..., + affected_rules: tuple[str, ...] = ..., + fixable: bool = ..., + ) -> None: ... + +class LayoutConfig(NativeRecord): + levels: tuple[str, ...] + domain_roots: tuple[str, ...] + def __init__( + self, + levels: tuple[str, ...] = ..., + domain_roots: tuple[str, ...] = ..., + ) -> None: ... + +class RulesConfig(NativeRecord): + select: tuple[str, ...] + ignore: tuple[str, ...] + thresholds: dict[str, int] + threshold_overrides: tuple[ThresholdOverride, ...] + rule_options: dict[str, dict[str, RuleOptionValue]] + rule_exceptions: tuple[RuleExemption, ...] + graph_edge_exceptions: tuple[GraphEdgeExclusion, ...] + rule_ignores: tuple[RuleIgnore, ...] + allow_exceptions: bool + select_star_allow: tuple[SelectStarAllow, ...] + domains: tuple[str, ...] + approved_source_tokens: tuple[str, ...] + retired_source_tokens: dict[str, str] + sql_tests: SqlTestRulesConfig + layout: LayoutConfig + cache: RulesCacheConfig + def __init__( + self, + select: tuple[str, ...] = ..., + ignore: tuple[str, ...] = ..., + thresholds: dict[str, int] = ..., + threshold_overrides: tuple[ThresholdOverride, ...] = ..., + rule_options: dict[str, dict[str, RuleOptionValue]] = ..., + rule_exceptions: tuple[RuleExemption, ...] = ..., + graph_edge_exceptions: tuple[GraphEdgeExclusion, ...] = ..., + rule_ignores: tuple[RuleIgnore, ...] = ..., + allow_exceptions: bool = ..., + select_star_allow: tuple[SelectStarAllow, ...] = ..., + domains: tuple[str, ...] = ..., + approved_source_tokens: tuple[str, ...] = ..., + retired_source_tokens: dict[str, str] = ..., + sql_tests: SqlTestRulesConfig = ..., + layout: LayoutConfig = ..., + cache: RulesCacheConfig = ..., + ) -> None: ... + +class RulesResult(NativeRecord): + findings: tuple[Finding, ...] + evaluated_models: int + unevaluated_resources: int + cache_hits: int + cache_misses: int + built_in_ms: int + custom_ms: int + custom_cpu_ms: int + def __init__( + self, + findings: tuple[Finding, ...], + evaluated_models: int, + unevaluated_resources: int = ..., + cache_hits: int = ..., + cache_misses: int = ..., + built_in_ms: int = ..., + custom_ms: int = ..., + custom_cpu_ms: int = ..., + ) -> None: ... + +class RuleFile(NativeRecord): + path: str + source: str + def __init__( + self, + path: str, + source: str, + ) -> None: ... + +class RuleCase(NativeRecord): + description: str + source: str + expected_finding_count: int + path: str + files: tuple[RuleFile, ...] + config: dict[str, object] + def __init__( + self, + description: str, + source: str, + expected_finding_count: int, + path: str = ..., + files: tuple[RuleFile, ...] = ..., + config: dict[str, object] = ..., + ) -> None: ... + +class RuleResult(NativeRecord): + findings: tuple[Finding, ...] + def __init__( + self, + findings: tuple[Finding, ...], + ) -> None: ... + @property + def finding_count(self) -> int: ... + +class CustomRuleEvaluation(NativeRecord): + """Findings and observed fact reads of one rule invocation for one subject.""" + + code: str + subject: str | None + findings: tuple[Finding, ...] + reads: frozenset[tuple[object, ...]] | None + def __init__( + self, + code: str, + subject: str | None, + findings: tuple[Finding, ...], + reads: frozenset[tuple[object, ...]] | None, + ) -> None: ... + +class ModuleStateTrace(NativeRecord): + """What exact module-state walks touched: object and type identities and their decisions.""" + + touched: dict[int, int] + descended_types: frozenset[int] + checked_instances: frozenset[int] + def __init__( + self, + touched: dict[int, int], + descended_types: frozenset[int], + checked_instances: frozenset[int], + ) -> None: ... + +class ModuleStateFingerprint(NativeRecord): + """One exact custom-rule module-state fingerprint and the trace of the walk behind it.""" + + token: str + trace: ModuleStateTrace + def __init__( + self, + token: str, + trace: ModuleStateTrace, + ) -> None: ... + +class CustomRuleRun(NativeRecord): + """Every invocation of one host run plus rules whose reads could not be attributed.""" + + evaluations: tuple[CustomRuleEvaluation, ...] + untracked_codes: frozenset[str] + uncacheable_codes: frozenset[str] + observed: tuple[tuple[tuple[str, ...], str | None], ...] + stateful_codes: frozenset[str] + def __init__( + self, + evaluations: tuple[CustomRuleEvaluation, ...], + untracked_codes: frozenset[str], + uncacheable_codes: frozenset[str] = ..., + observed: tuple[tuple[tuple[str, ...], str | None], ...] = ..., + stateful_codes: frozenset[str] = ..., + ) -> None: ... + +class CustomHostPartition(NativeRecord): + """How one host of a split run detects module state and learns the run was cancelled.""" + + cancelled: Callable[[], bool] + def __init__( + self, + cancelled: Callable[[], bool], + ) -> None: ... diff --git a/src/sqlbuild/rule_engine/_helpers/engine/_custom_catalogue.py b/src/sqlbuild/rule_engine/_helpers/engine/_custom_catalogue.py index 3e995d1a69..c60ca2c777 100644 --- a/src/sqlbuild/rule_engine/_helpers/engine/_custom_catalogue.py +++ b/src/sqlbuild/rule_engine/_helpers/engine/_custom_catalogue.py @@ -9,13 +9,12 @@ from pathlib import Path from types import ModuleType -from sqlbuild._native import RulesError +from sqlbuild._native import Rule, RulesError from sqlbuild.rule_engine._helpers.engine.definition import ( resolve_rule_signature, rule_from_value, ) from sqlbuild.rule_engine.constants import INIT_MODULE_NAME, RULE_DECORATOR_TOKEN -from sqlbuild.rule_engine.models import Rule def load_custom_rules(*, project_dir: Path) -> tuple[Rule, ...]: diff --git a/src/sqlbuild/rule_engine/_helpers/engine/custom_payloads.py b/src/sqlbuild/rule_engine/_helpers/engine/custom_payloads.py index d18c5786e7..708754bb50 100644 --- a/src/sqlbuild/rule_engine/_helpers/engine/custom_payloads.py +++ b/src/sqlbuild/rule_engine/_helpers/engine/custom_payloads.py @@ -6,11 +6,11 @@ from pathlib import Path from sqlbuild._native import ( + Rule, custom_rule_implementation_fingerprint, custom_rule_import_closure, custom_rule_test_evidence, ) -from sqlbuild.rule_engine.models import Rule def custom_rule_payloads( diff --git a/src/sqlbuild/rule_engine/_helpers/engine/definition.py b/src/sqlbuild/rule_engine/_helpers/engine/definition.py index 1df2c5fefc..5f7644caba 100644 --- a/src/sqlbuild/rule_engine/_helpers/engine/definition.py +++ b/src/sqlbuild/rule_engine/_helpers/engine/definition.py @@ -6,9 +6,8 @@ from collections.abc import Callable from typing import cast, get_type_hints -from sqlbuild._native import RulesError -from sqlbuild.rule_engine.models import Model, Project, Rule, RuleOption -from sqlbuild.rule_engine.types import RuleCheck, RuleContext, RuleSubject +from sqlbuild._native import Model, Project, Rule, RuleOption, RulesError, RuleSubject +from sqlbuild.rule_engine.types import RuleCheck, RuleContext _RULE_PARAMETER_COUNT: int = 2 _RULE_CODE_DIGIT_COUNT: int = 3 diff --git a/src/sqlbuild/rule_engine/_helpers/engine/fact_replay.py b/src/sqlbuild/rule_engine/_helpers/engine/fact_replay.py index 97600b0354..d3868e3a87 100644 --- a/src/sqlbuild/rule_engine/_helpers/engine/fact_replay.py +++ b/src/sqlbuild/rule_engine/_helpers/engine/fact_replay.py @@ -9,7 +9,7 @@ import orjson -from sqlbuild._native import FactDigestError +from sqlbuild._native import FactDigestError, Model, RuleFactViews from sqlbuild.rule_engine.classes.shared_fact_encoder import SharedFactEncoder from sqlbuild.rule_engine.classes.sql_document import SqlDocument from sqlbuild.rule_engine.constants import ( @@ -43,7 +43,6 @@ FACT_TREE_RELATIVE_PARTS, FACT_TREE_RESOURCES_UNDER, ) -from sqlbuild.rule_engine.models import Model, RuleFactViews from sqlbuild.rule_engine.types import FactKey _TRACKED_TEXT_SUFFIXES: frozenset[str] = frozenset({".py", ".sql", ".toml", ".yaml", ".yml"}) diff --git a/src/sqlbuild/rule_engine/_helpers/engine/hermeticity.py b/src/sqlbuild/rule_engine/_helpers/engine/hermeticity.py index efd3c40536..343c66867d 100644 --- a/src/sqlbuild/rule_engine/_helpers/engine/hermeticity.py +++ b/src/sqlbuild/rule_engine/_helpers/engine/hermeticity.py @@ -5,9 +5,8 @@ import ast from pathlib import Path -from sqlbuild._native import NonHermeticRuleError, RulesError, custom_rule_import_closure +from sqlbuild._native import NonHermeticRuleError, Rule, RulesError, custom_rule_import_closure from sqlbuild.rule_engine.constants import CUSTOM_RULE_IMPORT_ROOTS -from sqlbuild.rule_engine.models import Rule def verify_custom_rules(*, rules: tuple[Rule, ...], project_dir: Path) -> None: diff --git a/src/sqlbuild/rule_engine/_helpers/engine/rule_harness.py b/src/sqlbuild/rule_engine/_helpers/engine/rule_harness.py index c0f3e10635..affcc7f3d4 100644 --- a/src/sqlbuild/rule_engine/_helpers/engine/rule_harness.py +++ b/src/sqlbuild/rule_engine/_helpers/engine/rule_harness.py @@ -8,7 +8,12 @@ from sqlbuild._native import ( DiscoveredProjectInputs, + Rule, RuleAssertionError, + RuleCase, + RuleResult, + RulesConfig, + RulesResult, RuleUsageError, build_compiled_project, discover_project_inputs, @@ -22,13 +27,6 @@ rule_from_value, ) from sqlbuild.rule_engine.constants import MIN_CUSTOM_RULE_TEST_CASES -from sqlbuild.rule_engine.models import ( - Rule, - RuleCase, - RuleResult, - RulesConfig, - RulesResult, -) from sqlbuild.rule_engine.types import RuleCheck diff --git a/src/sqlbuild/rule_engine/_helpers/host/custom_evaluation.py b/src/sqlbuild/rule_engine/_helpers/host/custom_evaluation.py index 6c3f676b7a..389d46c6b9 100644 --- a/src/sqlbuild/rule_engine/_helpers/host/custom_evaluation.py +++ b/src/sqlbuild/rule_engine/_helpers/host/custom_evaluation.py @@ -9,7 +9,20 @@ from itertools import chain from pathlib import Path -from sqlbuild._native import HostCancelledError, NonHermeticRuleError, RulesError +from sqlbuild._native import ( + CustomHostPartition, + CustomRuleEvaluation, + CustomRuleRun, + Finding, + HostCancelledError, + Model, + NonHermeticRuleError, + Project, + Rule, + RulesConfig, + RulesError, + RuleSubject, +) from sqlbuild.compiler.compile.types import CompiledModelView, CompiledProjectView from sqlbuild.rule_engine._helpers.host.module_state import rule_namespaces from sqlbuild.rule_engine._helpers.host.tracked_facts import tracked_fact_views @@ -26,17 +39,7 @@ CUSTOM_HOST_CANCELLED_MESSAGE, CUSTOM_RULE_PROJECT_SUBJECT, ) -from sqlbuild.rule_engine.models import ( - CustomHostPartition, - CustomRuleEvaluation, - CustomRuleRun, - Finding, - Model, - Project, - Rule, - RulesConfig, -) -from sqlbuild.rule_engine.types import CustomRulePlan, RuleSubject +from sqlbuild.rule_engine.types import CustomRulePlan type _FindingKey = tuple[str, str, int, int, str, str] diff --git a/src/sqlbuild/rule_engine/_helpers/host/custom_host.py b/src/sqlbuild/rule_engine/_helpers/host/custom_host.py index 9cbbd16f24..9f6cad703d 100644 --- a/src/sqlbuild/rule_engine/_helpers/host/custom_host.py +++ b/src/sqlbuild/rule_engine/_helpers/host/custom_host.py @@ -14,7 +14,17 @@ from pathlib import Path from typing import Any, BinaryIO -from sqlbuild._native import CompiledProject, HostCancelledError, RulesError, build_rule_catalogue +from sqlbuild._native import ( + CompiledProject, + CustomHostPartition, + CustomRuleRun, + Finding, + HostCancelledError, + Rule, + RulesConfig, + RulesError, + build_rule_catalogue, +) from sqlbuild.compiler.compile.types import CompiledProjectView from sqlbuild.rule_engine._helpers.engine.fact_replay import fact_key_payload from sqlbuild.rule_engine._helpers.host.custom_evaluation import evaluate_custom_rules @@ -32,13 +42,6 @@ CUSTOM_HOST_PROTOCOL_VERSION, CUSTOM_HOST_RUNTIME_VERSION, ) -from sqlbuild.rule_engine.models import ( - CustomHostPartition, - CustomRuleRun, - Finding, - Rule, - RulesConfig, -) from sqlbuild.rule_engine.types import CustomRulePlan diff --git a/src/sqlbuild/rule_engine/_helpers/host/module_state.py b/src/sqlbuild/rule_engine/_helpers/host/module_state.py index c72d0c4160..93004b792b 100644 --- a/src/sqlbuild/rule_engine/_helpers/host/module_state.py +++ b/src/sqlbuild/rule_engine/_helpers/host/module_state.py @@ -2,6 +2,7 @@ from __future__ import annotations +import dataclasses import enum import hashlib import os @@ -16,8 +17,7 @@ from types import BuiltinFunctionType, CellType, FunctionType, MethodType, ModuleType from typing import Any, cast -from sqlbuild._native import OpaqueModuleStateError -from sqlbuild.rule_engine.models import ModuleStateFingerprint, ModuleStateTrace +from sqlbuild._native import ModuleStateFingerprint, ModuleStateTrace, OpaqueModuleStateError _MAX_STATE_NODES: int = 50_000 _ATOMIC_TYPES: tuple[type, ...] = ( @@ -557,6 +557,10 @@ def _instance_state(value: object) -> dict[str, object]: for slot in getattr(owner, "__slots__", ()): if isinstance(slot, str) and hasattr(value, slot): state[slot] = getattr(value, slot) + if not state and dataclasses.is_dataclass(value) and not isinstance(value, type): + state.update( + (field.name, getattr(value, field.name)) for field in dataclasses.fields(value) + ) return state diff --git a/src/sqlbuild/rule_engine/_helpers/host/tracked_facts.py b/src/sqlbuild/rule_engine/_helpers/host/tracked_facts.py index ca54c9ee78..854fc1dc92 100644 --- a/src/sqlbuild/rule_engine/_helpers/host/tracked_facts.py +++ b/src/sqlbuild/rule_engine/_helpers/host/tracked_facts.py @@ -12,6 +12,10 @@ EnumDeclaration, FactDigestError, InferredColumn, + Model, + ModelSql, + ProjectPath, + RuleFactViews, ) from sqlbuild.compiler.compile.types import CompiledAuditView from sqlbuild.rule_engine._helpers.engine.fact_replay import fact_error_digest, fact_value_digest @@ -47,7 +51,6 @@ FACT_TREE_RELATIVE_PARTS, FACT_TREE_RESOURCES_UNDER, ) -from sqlbuild.rule_engine.models import Model, ModelSql, ProjectPath, RuleFactViews class _TrackedView: diff --git a/src/sqlbuild/rule_engine/classes/audit_facts.py b/src/sqlbuild/rule_engine/classes/audit_facts.py index 0f033460cb..bcbb39994a 100644 --- a/src/sqlbuild/rule_engine/classes/audit_facts.py +++ b/src/sqlbuild/rule_engine/classes/audit_facts.py @@ -4,10 +4,10 @@ from sqlbuild._native import ( CompiledAudit, + Model, compiled_audit_record, ) from sqlbuild.compiler.compile.types import CompiledProjectView -from sqlbuild.rule_engine.models import Model class AuditFacts: diff --git a/src/sqlbuild/rule_engine/classes/column_facts.py b/src/sqlbuild/rule_engine/classes/column_facts.py index 72722b74f5..6cad2a9fa7 100644 --- a/src/sqlbuild/rule_engine/classes/column_facts.py +++ b/src/sqlbuild/rule_engine/classes/column_facts.py @@ -2,9 +2,8 @@ from functools import cached_property -from sqlbuild._native import InferredColumn +from sqlbuild._native import InferredColumn, Model from sqlbuild.compiler.compile.types import CompiledModelView, CompiledProjectView -from sqlbuild.rule_engine.models import Model class ColumnFacts: diff --git a/src/sqlbuild/rule_engine/classes/contract_facts.py b/src/sqlbuild/rule_engine/classes/contract_facts.py index f5e757925a..b4b3a35abd 100644 --- a/src/sqlbuild/rule_engine/classes/contract_facts.py +++ b/src/sqlbuild/rule_engine/classes/contract_facts.py @@ -1,8 +1,8 @@ """Contract facts for custom rules.""" +from sqlbuild._native import Model from sqlbuild.compiler.compile.types import CompiledModelView, CompiledProjectView from sqlbuild.rule_engine.constants import CONTRACT_ENFORCED -from sqlbuild.rule_engine.models import Model class ContractFacts: diff --git a/src/sqlbuild/rule_engine/classes/fact_reads.py b/src/sqlbuild/rule_engine/classes/fact_reads.py index 8680946276..7fa8ed04e5 100644 --- a/src/sqlbuild/rule_engine/classes/fact_reads.py +++ b/src/sqlbuild/rule_engine/classes/fact_reads.py @@ -4,7 +4,7 @@ import os -from sqlbuild.rule_engine.models import Model +from sqlbuild._native import Model class FactReads: diff --git a/src/sqlbuild/rule_engine/classes/graph_facts.py b/src/sqlbuild/rule_engine/classes/graph_facts.py index 98df4f64ec..1ccc5069fd 100644 --- a/src/sqlbuild/rule_engine/classes/graph_facts.py +++ b/src/sqlbuild/rule_engine/classes/graph_facts.py @@ -2,9 +2,9 @@ from functools import cached_property +from sqlbuild._native import Model from sqlbuild.compiler.compile.types import CompiledModelView, CompiledProjectView from sqlbuild.rule_engine.classes.project_tree import public_model -from sqlbuild.rule_engine.models import Model class GraphFacts: diff --git a/src/sqlbuild/rule_engine/classes/model_test_facts.py b/src/sqlbuild/rule_engine/classes/model_test_facts.py index 6a031a4b05..46efec3754 100644 --- a/src/sqlbuild/rule_engine/classes/model_test_facts.py +++ b/src/sqlbuild/rule_engine/classes/model_test_facts.py @@ -2,9 +2,8 @@ from functools import cached_property -from sqlbuild._native import CompiledSqlTest +from sqlbuild._native import CompiledSqlTest, Model from sqlbuild.compiler.compile.types import CompiledProjectView, SqlTestMode -from sqlbuild.rule_engine.models import Model class TestFacts: diff --git a/src/sqlbuild/rule_engine/classes/module_state_monitor.py b/src/sqlbuild/rule_engine/classes/module_state_monitor.py index ac4788dd2f..9bfa9d6385 100644 --- a/src/sqlbuild/rule_engine/classes/module_state_monitor.py +++ b/src/sqlbuild/rule_engine/classes/module_state_monitor.py @@ -5,13 +5,12 @@ from itertools import chain from pathlib import Path -from sqlbuild._native import OpaqueModuleStateError +from sqlbuild._native import ModuleStateFingerprint, ModuleStateTrace, OpaqueModuleStateError from sqlbuild.rule_engine._helpers.host.module_state import ( module_state_fingerprint, module_state_snapshot, rules_root_prefix, ) -from sqlbuild.rule_engine.models import ModuleStateFingerprint, ModuleStateTrace _EMPTY_TRACE: ModuleStateTrace = ModuleStateTrace( touched={}, descended_types=frozenset(), checked_instances=frozenset() diff --git a/src/sqlbuild/rule_engine/classes/project_facts.py b/src/sqlbuild/rule_engine/classes/project_facts.py index b07de8a219..8c5b37873c 100644 --- a/src/sqlbuild/rule_engine/classes/project_facts.py +++ b/src/sqlbuild/rule_engine/classes/project_facts.py @@ -3,6 +3,7 @@ from pathlib import Path from sqlbuild._native import ( + Model, compiled_audit_record, compiled_function_record, compiled_seed_record, @@ -10,7 +11,6 @@ ) from sqlbuild.compiler.compile.types import CompiledProjectView from sqlbuild.rule_engine.classes.project_tree import ProjectTree, public_model -from sqlbuild.rule_engine.models import Model class ProjectFacts: diff --git a/src/sqlbuild/rule_engine/classes/project_tree.py b/src/sqlbuild/rule_engine/classes/project_tree.py index 407dadf1cf..99b34707a1 100644 --- a/src/sqlbuild/rule_engine/classes/project_tree.py +++ b/src/sqlbuild/rule_engine/classes/project_tree.py @@ -8,10 +8,9 @@ from pathlib import Path import sqlbuild._native as _native -from sqlbuild._native import RuleUsageError +from sqlbuild._native import Model, ProjectPath, RuleUsageError from sqlbuild.compiler.compile.types import CompiledProjectView from sqlbuild.rule_engine.constants import PARENT_DIRECTORY_TOKEN -from sqlbuild.rule_engine.models import Model, ProjectPath _EXCLUDED_ROOTS: frozenset[str] = frozenset({".git", ".venv", "logs", "target", "venv"}) _TRACKED_TEXT_SUFFIXES: frozenset[str] = frozenset({".py", ".sql", ".toml", ".yaml", ".yml"}) diff --git a/src/sqlbuild/rule_engine/classes/rule_context.py b/src/sqlbuild/rule_engine/classes/rule_context.py index 66fec806ea..0c20fc0a8e 100644 --- a/src/sqlbuild/rule_engine/classes/rule_context.py +++ b/src/sqlbuild/rule_engine/classes/rule_context.py @@ -5,7 +5,17 @@ from pathlib import Path from typing import Any, cast -from sqlbuild._native import RuleUsageError +from sqlbuild._native import ( + Finding, + Model, + ProjectPath, + Rule, + RuleFactViews, + RuleOption, + RulesConfig, + RuleUsageError, + SqlNode, +) from sqlbuild.compiler.compile.types import CompiledModelView, CompiledProjectView from sqlbuild.rule_engine.classes.audit_facts import AuditFacts from sqlbuild.rule_engine.classes.column_facts import ColumnFacts @@ -17,16 +27,6 @@ from sqlbuild.rule_engine.classes.project_facts import ProjectFacts from sqlbuild.rule_engine.classes.sql_facts import SqlFacts from sqlbuild.rule_engine.constants import PARENT_DIRECTORY_TOKEN -from sqlbuild.rule_engine.models import ( - Finding, - Model, - ProjectPath, - Rule, - RuleFactViews, - RuleOption, - RulesConfig, - SqlNode, -) def build_rule_fact_views( diff --git a/src/sqlbuild/rule_engine/classes/sql_document.py b/src/sqlbuild/rule_engine/classes/sql_document.py index e8eed481ec..774b3c9c6e 100644 --- a/src/sqlbuild/rule_engine/classes/sql_document.py +++ b/src/sqlbuild/rule_engine/classes/sql_document.py @@ -2,10 +2,9 @@ from typing import Any -from sqlbuild._native import RuleUsageError +from sqlbuild._native import RuleUsageError, SqlNode from sqlbuild.compiler.sql_analysis.main.import_polyglot_sql import import_polyglot_sql from sqlbuild.rule_engine.constants import AST_CTE_KIND -from sqlbuild.rule_engine.models import SqlNode class SqlDocument: diff --git a/src/sqlbuild/rule_engine/classes/sql_facts.py b/src/sqlbuild/rule_engine/classes/sql_facts.py index 7267d1003e..c600b94107 100644 --- a/src/sqlbuild/rule_engine/classes/sql_facts.py +++ b/src/sqlbuild/rule_engine/classes/sql_facts.py @@ -2,10 +2,9 @@ from functools import cached_property -from sqlbuild._native import RuleUsageError +from sqlbuild._native import Model, ModelSql, RuleUsageError from sqlbuild.compiler.compile.types import CompiledModelView, CompiledProjectView from sqlbuild.rule_engine.classes.sql_document import SqlDocument -from sqlbuild.rule_engine.models import Model, ModelSql class SqlFacts: diff --git a/src/sqlbuild/rule_engine/constants.py b/src/sqlbuild/rule_engine/constants.py index c448dbf8f3..e95a2147b5 100644 --- a/src/sqlbuild/rule_engine/constants.py +++ b/src/sqlbuild/rule_engine/constants.py @@ -4,14 +4,11 @@ CUSTOM_HOST_RUNTIME_VERSION: str = "sqlbuild-rules-custom-v2" CUSTOM_HOST_MAX_TRACKED_READS: int = 100_000 CUSTOM_RULE_PROJECT_SUBJECT: str = "" -CUSTOM_RULES_CACHE_FILE: str = "custom-rules.json" CUSTOM_HOST_INPUT_TUPLE_SIZE: int = 2 SHARED_FACT_DATACLASS_TAG: str = "\0dataclass" -CUSTOM_HOST_ABANDONED_SUFFIX: str = ".abandoned" CUSTOM_HOST_INPUT_POLL_SECONDS: float = 0.002 CUSTOM_HOST_INPUT_WAIT_SECONDS: float = 120.0 CUSTOM_HOST_MODULE: str = "sqlbuild.rule_engine._helpers.host.custom_host" -CUSTOM_HOST_CANCELLED_MARKER: str = "cancelled" CUSTOM_HOST_CANCELLED_EXIT_CODE: int = 75 CUSTOM_HOST_CANCELLED_MESSAGE: str = "custom host cancelled after another host failed" CUSTOM_HOST_LAUNCH_MODULE: str = "sqlbuild.rule_engine._helpers.host.custom_host_launch" @@ -62,7 +59,6 @@ AST_CTE_KIND: str = "cte" CONTRACT_ENFORCED: str = "enforced" -TYPE_PROOF_RULE_CODES: frozenset[str] = frozenset({"SQBRCONTRACT105"}) RULE_DECORATOR_TOKEN: str = "@rule" diff --git a/src/sqlbuild/rule_engine/main/define.py b/src/sqlbuild/rule_engine/main/define.py index 38527a7c2f..9c60a48417 100644 --- a/src/sqlbuild/rule_engine/main/define.py +++ b/src/sqlbuild/rule_engine/main/define.py @@ -4,8 +4,8 @@ from collections.abc import Callable +from sqlbuild._native import RuleOption from sqlbuild.rule_engine._helpers.engine.definition import define_rule -from sqlbuild.rule_engine.models import RuleOption from sqlbuild.rule_engine.types import RuleCheck diff --git a/src/sqlbuild/rule_engine/main/evaluate_rule.py b/src/sqlbuild/rule_engine/main/evaluate_rule.py index d728597f83..de6def3a96 100644 --- a/src/sqlbuild/rule_engine/main/evaluate_rule.py +++ b/src/sqlbuild/rule_engine/main/evaluate_rule.py @@ -1,6 +1,6 @@ """Public custom rule test harness.""" -from sqlbuild.rule_engine.models import Rule, RuleCase, RuleResult +from sqlbuild._native import Rule, RuleCase, RuleResult from sqlbuild.rule_engine.types import RuleCheck diff --git a/src/sqlbuild/rule_engine/models.py b/src/sqlbuild/rule_engine/models.py deleted file mode 100644 index 362c0a90b3..0000000000 --- a/src/sqlbuild/rule_engine/models.py +++ /dev/null @@ -1,334 +0,0 @@ -"""Structured rules configuration, rule, and evaluation models.""" - -from __future__ import annotations - -from collections.abc import Callable -from dataclasses import dataclass, field -from pathlib import Path -from typing import Any - -from sqlbuild._native import ( - GraphEdgeExclusion, - RuleExemption, - RuleGuidance, - RuleIgnore, - RulesCacheConfig, - SelectStarAllow, - SqlTestRulesConfig, - ThresholdOverride, -) -from sqlbuild.rule_engine.types import RuleCheck, RuleOptionValue, RuleSubject - - -@dataclass(frozen=True, slots=True) -class Model: - """Stable identity and common metadata for one compiled SQL model.""" - - name: str - path: Path - materialization: str | None - - -@dataclass(frozen=True, slots=True) -class Project: - """Stable summary for one compiled project invocation.""" - - model_count: int - target_name: str | None - - -@dataclass(frozen=True, slots=True, order=True) -class ProjectPath: - """One normalized project-relative file or directory.""" - - value: str - is_file: bool - is_dir: bool - - @property - def parts(self) -> tuple[str, ...]: - return Path(self.value).parts - - @property - def name(self) -> str: - return Path(self.value).name - - @property - def depth(self) -> int: - return len(self.parts) - - -@dataclass(frozen=True, slots=True) -class SqlNode: - """Stable source location for one common SQL structure fact.""" - - kind: str - line: int - column: int - - -@dataclass(frozen=True, slots=True) -class ModelSql: - """Authored and compiler-expanded SQL documents for one model.""" - - authored: Any - expanded: Any - - -@dataclass(frozen=True, slots=True) -class RuleFactViews: - """Views shared by every invocation in one custom-rule host evaluation.""" - - project: Any - sql: Any - graph: Any - columns: Any - contracts: Any - tests: Any - audits: Any - declarations: Any - - -@dataclass(frozen=True) -class RuleOption[T]: - """One typed configuration option declared by a rule.""" - - name: str - value_type: type[T] - default: T - description: str - choices: tuple[T, ...] = () - minimum: int | None = None - maximum: int | None = None - minimum_items: int | None = None - item_type: type[object] | None = None - - @classmethod - def boolean(cls, *, name: str, default: bool, description: str) -> RuleOption[bool]: - return RuleOption(name=name, value_type=bool, default=default, description=description) - - @classmethod - def integer( - cls, - *, - name: str, - default: int, - description: str, - minimum: int | None = None, - maximum: int | None = None, - ) -> RuleOption[int]: - return RuleOption( - name=name, - value_type=int, - default=default, - description=description, - minimum=minimum, - maximum=maximum, - ) - - @classmethod - def string( - cls, - *, - name: str, - default: str, - description: str, - choices: tuple[str, ...] = (), - ) -> RuleOption[str]: - return RuleOption( - name=name, - value_type=str, - default=default, - description=description, - choices=choices, - ) - - @classmethod - def string_list( - cls, - *, - name: str, - default: tuple[str, ...], - description: str, - minimum_items: int | None = None, - ) -> RuleOption[tuple[str, ...]]: - return RuleOption( - name=name, - value_type=tuple, - default=default, - description=description, - minimum_items=minimum_items, - item_type=str, - ) - - @classmethod - def integer_list( - cls, - *, - name: str, - default: tuple[int, ...], - description: str, - minimum_items: int | None = None, - ) -> RuleOption[tuple[int, ...]]: - return RuleOption( - name=name, - value_type=tuple, - default=default, - description=description, - minimum_items=minimum_items, - item_type=int, - ) - - -@dataclass(frozen=True) -class Rule: - """Immutable metadata and implementation for one rule.""" - - code: str - family: str - slug: str - message: str - remediation: str - check: RuleCheck - options: tuple[RuleOption[object], ...] = () - enabled_by_default: bool = False - custom: bool = False - source: str | None = None - subject: RuleSubject | None = None - subject_parameter: str | None = None - context_parameter: str | None = None - guidance: RuleGuidance | None = None - - @property - def project_wide(self) -> bool: - """Return whether this rule is invoked once for the compiled project.""" - - return self.subject is RuleSubject.PROJECT - - -@dataclass(frozen=True) -class Finding: - """One deterministic compiler rules violation.""" - - code: str - path: Path - line: int - column: int - message: str - remediation: str - unevaluated: bool = False - affected_rules: tuple[str, ...] = () - fixable: bool = False - - -@dataclass(frozen=True) -class LayoutConfig: - levels: tuple[str, ...] = ( - "staging", - "intermediate/clean", - "intermediate/enriched", - "mart", - ) - domain_roots: tuple[str, ...] = () - - -@dataclass(frozen=True) -class RulesConfig: - select: tuple[str, ...] = () - ignore: tuple[str, ...] = () - thresholds: dict[str, int] = field(default_factory=dict) - threshold_overrides: tuple[ThresholdOverride, ...] = () - rule_options: dict[str, dict[str, RuleOptionValue]] = field(default_factory=dict) - rule_exceptions: tuple[RuleExemption, ...] = () - graph_edge_exceptions: tuple[GraphEdgeExclusion, ...] = () - rule_ignores: tuple[RuleIgnore, ...] = () - allow_exceptions: bool = True - select_star_allow: tuple[SelectStarAllow, ...] = () - domains: tuple[str, ...] = () - approved_source_tokens: tuple[str, ...] = () - retired_source_tokens: dict[str, str] = field(default_factory=dict) - sql_tests: SqlTestRulesConfig = field(default_factory=SqlTestRulesConfig) - layout: LayoutConfig = field(default_factory=LayoutConfig) - cache: RulesCacheConfig = field(default_factory=RulesCacheConfig) - - -@dataclass(frozen=True) -class RulesResult: - findings: tuple[Finding, ...] - evaluated_models: int - unevaluated_resources: int = 0 - cache_hits: int = 0 - cache_misses: int = 0 - built_in_ms: int = 0 - custom_ms: int = 0 - custom_cpu_ms: int = 0 - - -@dataclass(frozen=True) -class RuleFile: - path: str - source: str - - -@dataclass(frozen=True) -class RuleCase: - description: str - source: str - expected_finding_count: int - path: str = "models/mart/example__mart__result.sql" - files: tuple[RuleFile, ...] = () - config: dict[str, object] = field(default_factory=dict) - - -@dataclass(frozen=True) -class RuleResult: - findings: tuple[Finding, ...] - - @property - def finding_count(self) -> int: - return len(self.findings) - - -@dataclass(frozen=True) -class CustomRuleEvaluation: - """Findings and observed fact reads of one rule invocation for one subject.""" - - code: str - subject: str | None - findings: tuple[Finding, ...] - reads: frozenset[tuple[object, ...]] | None - - -@dataclass(frozen=True) -class ModuleStateTrace: - """What exact module-state walks touched: object and type identities and their decisions.""" - - touched: dict[int, int] - descended_types: frozenset[int] - checked_instances: frozenset[int] - - -@dataclass(frozen=True) -class ModuleStateFingerprint: - """One exact custom-rule module-state fingerprint and the trace of the walk behind it.""" - - token: str - trace: ModuleStateTrace - - -@dataclass(frozen=True) -class CustomRuleRun: - """Every invocation of one host run plus rules whose reads could not be attributed.""" - - evaluations: tuple[CustomRuleEvaluation, ...] - untracked_codes: frozenset[str] - uncacheable_codes: frozenset[str] = frozenset() - observed: tuple[tuple[tuple[str, ...], str | None], ...] = () - stateful_codes: frozenset[str] = frozenset() - - -@dataclass(frozen=True) -class CustomHostPartition: - """How one host of a split run detects module state and learns the run was cancelled.""" - - cancelled: Callable[[], bool] diff --git a/src/sqlbuild/rule_engine/types.py b/src/sqlbuild/rule_engine/types.py index ccaedcbba2..fec5a01fac 100644 --- a/src/sqlbuild/rule_engine/types.py +++ b/src/sqlbuild/rule_engine/types.py @@ -3,11 +3,11 @@ from __future__ import annotations from collections.abc import Callable -from enum import StrEnum from pathlib import Path from typing import TYPE_CHECKING, Protocol if TYPE_CHECKING: + from sqlbuild._native import Finding, Model, ProjectPath, RuleOption, SqlNode from sqlbuild.rule_engine.classes.audit_facts import AuditFacts from sqlbuild.rule_engine.classes.column_facts import ColumnFacts from sqlbuild.rule_engine.classes.contract_facts import ContractFacts @@ -16,26 +16,12 @@ from sqlbuild.rule_engine.classes.model_test_facts import TestFacts from sqlbuild.rule_engine.classes.project_facts import ProjectFacts from sqlbuild.rule_engine.classes.sql_facts import SqlFacts - from sqlbuild.rule_engine.models import ( - Finding, - Model, - ProjectPath, - RuleOption, - SqlNode, - ) type RuleOptionValue = bool | int | str | tuple[str, ...] | tuple[int, ...] type FactKey = tuple[str, ...] type CustomRulePlan = dict[str, frozenset[str] | None] -class RuleSubject(StrEnum): - """Compiler-owned unit used to invoke and cache one custom rule.""" - - MODEL = "model" - PROJECT = "project" - - class RuleContext(Protocol): """Stable typed compiler-fact views available to custom rules.""" diff --git a/src/sqlbuild/rules/__init__.py b/src/sqlbuild/rules/__init__.py index 583061e6ac..1ae437ada4 100644 --- a/src/sqlbuild/rules/__init__.py +++ b/src/sqlbuild/rules/__init__.py @@ -2,9 +2,7 @@ from collections.abc import Callable -from sqlbuild.rule_engine.main.define import rule as _rule -from sqlbuild.rule_engine.main.evaluate_rule import evaluate_rule -from sqlbuild.rule_engine.models import ( +from sqlbuild._native import ( Finding, Model, Project, @@ -15,6 +13,8 @@ RuleResult, SqlNode, ) +from sqlbuild.rule_engine.main.define import rule as _rule +from sqlbuild.rule_engine.main.evaluate_rule import evaluate_rule from sqlbuild.rule_engine.types import RuleCheck, RuleContext __all__ = ( diff --git a/src/sqlbuild/rules/testing.py b/src/sqlbuild/rules/testing.py index 0efe82f090..e316fe23f7 100644 --- a/src/sqlbuild/rules/testing.py +++ b/src/sqlbuild/rules/testing.py @@ -1,6 +1,6 @@ """Public test harness for repository-owned compiler rules.""" +from sqlbuild._native import RuleCase, RuleFile, RuleResult from sqlbuild.rule_engine.main.evaluate_rule import evaluate_rule -from sqlbuild.rule_engine.models import RuleCase, RuleFile, RuleResult __all__ = ("RuleCase", "RuleFile", "RuleResult", "evaluate_rule") diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/compile/test_dense_compile_performance.py b/tests/e2e/src/sqlbuild/cli/commands/main/compile/test_dense_compile_performance.py index 5e0749c357..e5e44d18b7 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/compile/test_dense_compile_performance.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/compile/test_dense_compile_performance.py @@ -14,8 +14,13 @@ from scripts.cold_compile_performance.main.assert_required_cgroup_memory_limit import ( assert_required_cgroup_memory_limit, ) -from sqlbuild._native import build_rule_catalogue, load_rules_config, select_catalogue_rules -from sqlbuild.rule_engine.models import Rule, RulesConfig +from sqlbuild._native import ( + Rule, + RulesConfig, + build_rule_catalogue, + load_rules_config, + select_catalogue_rules, +) from tests.e2e.src.sqlbuild.cli.commands.main.compile._test_types import ( DenseCompileGuardTestCase, DenseWarmEditCompileGuardTestCase, diff --git a/tests/integration/src/sqlbuild/rule_engine/host_payload/helpers.py b/tests/integration/src/sqlbuild/rule_engine/host_payload/helpers.py index 193acaf6ff..ddaa1811cd 100644 --- a/tests/integration/src/sqlbuild/rule_engine/host_payload/helpers.py +++ b/tests/integration/src/sqlbuild/rule_engine/host_payload/helpers.py @@ -10,6 +10,7 @@ from sqlbuild._native import ( CompiledModel, CompiledProject, + Model, build_compiled_project, discover_project_inputs, ) @@ -22,7 +23,6 @@ ) from sqlbuild.rule_engine.classes.project_tree import public_model from sqlbuild.rule_engine.classes.rule_context import RuleFactViews, build_rule_fact_views -from sqlbuild.rule_engine.models import Model from sqlbuild.rule_engine.types import FactKey _TREE_ARGUMENTS: tuple[str, ...] = ("models/marts/order_totals.sql", "models") diff --git a/tests/integration/src/sqlbuild/rule_engine/native_rows/test_sql_rule_cache_publication.py b/tests/integration/src/sqlbuild/rule_engine/native_rows/test_sql_rule_cache_publication.py index f464671cd6..91ab02ce99 100644 --- a/tests/integration/src/sqlbuild/rule_engine/native_rows/test_sql_rule_cache_publication.py +++ b/tests/integration/src/sqlbuild/rule_engine/native_rows/test_sql_rule_cache_publication.py @@ -10,8 +10,7 @@ import pytest import sqlbuild._native as native -from sqlbuild._native import SqlRulesEvaluation -from sqlbuild.rule_engine.models import Finding +from sqlbuild._native import Finding, SqlRulesEvaluation from tests.integration.src.sqlbuild.rule_engine.native_rows._test_types import ( SqlRulePublicationTestCase, ) diff --git a/tests/unit/src/sqlbuild/rule_engine/_helpers/engine/helpers.py b/tests/unit/src/sqlbuild/rule_engine/_helpers/engine/helpers.py index 35f0234893..d277fd2aba 100644 --- a/tests/unit/src/sqlbuild/rule_engine/_helpers/engine/helpers.py +++ b/tests/unit/src/sqlbuild/rule_engine/_helpers/engine/helpers.py @@ -20,11 +20,15 @@ CustomRulesOutcome, DiscoveredSqlTestBlock, DiscoveredSqlTestFile, + Finding, NativeRulesEvaluation, NativeRulesRequest, + Rule, RuleExemption, RuleIgnore, RulesCacheConfig, + RulesConfig, + RulesResult, ScopeIndex, build_rule_catalogue, install_rules_skills, @@ -34,7 +38,6 @@ from sqlbuild._native import evaluate_project_rules as evaluate from sqlbuild.compiler.compile.types import CompiledProjectView, CompiledResourceType, SqlTestMode from sqlbuild.rule_engine.constants import MIN_CUSTOM_RULE_TEST_CASES -from sqlbuild.rule_engine.models import Finding, Rule, RulesConfig, RulesResult from tests.unit.src.sqlbuild.rule_engine._helpers.engine._test_types import CustomRuleTestCase from tests.unit.src.sqlbuild.rule_engine.main.evaluate.helpers import build_project diff --git a/tests/unit/src/sqlbuild/rule_engine/_helpers/engine/test_catalogue.py b/tests/unit/src/sqlbuild/rule_engine/_helpers/engine/test_catalogue.py index d82902f8a9..81a9a6ec02 100644 --- a/tests/unit/src/sqlbuild/rule_engine/_helpers/engine/test_catalogue.py +++ b/tests/unit/src/sqlbuild/rule_engine/_helpers/engine/test_catalogue.py @@ -5,7 +5,10 @@ import pytest from sqlbuild._native import ( + Rule, RulesCacheConfig, + RulesConfig, + RulesResult, SqlTestRulesConfig, ThresholdOverride, build_rule_catalogue, @@ -14,7 +17,6 @@ select_catalogue_rules, ) from sqlbuild._native import evaluate_project_rules as evaluate -from sqlbuild.rule_engine.models import Rule, RulesConfig, RulesResult from tests.unit.src.sqlbuild.rule_engine._helpers.engine._test_types import ( PolicyGuidanceTestCase, PolicySelectionTestCase, diff --git a/tests/unit/src/sqlbuild/rule_engine/_helpers/engine/test_config.py b/tests/unit/src/sqlbuild/rule_engine/_helpers/engine/test_config.py index 2a1482d858..adc5dd4395 100644 --- a/tests/unit/src/sqlbuild/rule_engine/_helpers/engine/test_config.py +++ b/tests/unit/src/sqlbuild/rule_engine/_helpers/engine/test_config.py @@ -4,8 +4,14 @@ import pytest -from sqlbuild._native import RulesConfigError, RulesError, SqlTestRulesConfig, load_rules_config -from sqlbuild.rule_engine.models import LayoutConfig, RulesConfig +from sqlbuild._native import ( + LayoutConfig, + RulesConfig, + RulesConfigError, + RulesError, + SqlTestRulesConfig, + load_rules_config, +) from tests.unit.src.sqlbuild.rule_engine._helpers.engine._test_types import ( AllowExceptionsConfigTestCase, PolicyLayoutConfigTestCase, diff --git a/tests/unit/src/sqlbuild/rule_engine/_helpers/engine/test_custom_rule_evidence.py b/tests/unit/src/sqlbuild/rule_engine/_helpers/engine/test_custom_rule_evidence.py index 360da45a7f..1a66c58c6f 100644 --- a/tests/unit/src/sqlbuild/rule_engine/_helpers/engine/test_custom_rule_evidence.py +++ b/tests/unit/src/sqlbuild/rule_engine/_helpers/engine/test_custom_rule_evidence.py @@ -5,14 +5,16 @@ import pytest from sqlbuild._native import ( + Rule, + RulesConfig, RulesError, + RulesResult, build_rule_catalogue, custom_rule_implementation_fingerprint, custom_rule_test_evidence, ) from sqlbuild._native import evaluate_project_rules as evaluate from sqlbuild.compiler.compile.types import CompiledProjectView -from sqlbuild.rule_engine.models import Rule, RulesConfig, RulesResult from tests.unit.src.sqlbuild.rule_engine._helpers.engine._test_types import ( CustomRuleEvidenceTestCase, CustomRuleTestCase, diff --git a/tests/unit/src/sqlbuild/rule_engine/_helpers/engine/test_custom_rules.py b/tests/unit/src/sqlbuild/rule_engine/_helpers/engine/test_custom_rules.py index 10f47a4863..a32df2bf26 100644 --- a/tests/unit/src/sqlbuild/rule_engine/_helpers/engine/test_custom_rules.py +++ b/tests/unit/src/sqlbuild/rule_engine/_helpers/engine/test_custom_rules.py @@ -9,13 +9,14 @@ CompactLineageFacts, RuleExemption, RuleIgnore, + RulesConfig, RulesError, + RulesResult, compiled_model_record, project_with, ) from sqlbuild._native import evaluate_project_rules as evaluate from sqlbuild.compiler.compile.types import CompiledProjectView -from sqlbuild.rule_engine.models import RulesConfig, RulesResult from tests.unit.src.sqlbuild.rule_engine._helpers.engine._test_types import ( CustomRuleCacheDependencyTestCase, CustomRuleSuppressionTestCase, diff --git a/tests/unit/src/sqlbuild/rule_engine/_helpers/engine/test_definition.py b/tests/unit/src/sqlbuild/rule_engine/_helpers/engine/test_definition.py index 6cebaf7c89..8bf3473c5a 100644 --- a/tests/unit/src/sqlbuild/rule_engine/_helpers/engine/test_definition.py +++ b/tests/unit/src/sqlbuild/rule_engine/_helpers/engine/test_definition.py @@ -4,12 +4,11 @@ import pytest -from sqlbuild._native import RulesError +from sqlbuild._native import Rule, RulesError from sqlbuild.rule_engine._helpers.engine.definition import ( resolve_rule_signature, rule_from_value, ) -from sqlbuild.rule_engine.models import Rule from sqlbuild.rules import Finding, Model, Project, RuleContext, rule from tests.unit.src.sqlbuild.rule_engine._helpers.engine._test_types import ( RuleCodeTestCase, diff --git a/tests/unit/src/sqlbuild/rule_engine/_helpers/engine/test_hermeticity.py b/tests/unit/src/sqlbuild/rule_engine/_helpers/engine/test_hermeticity.py index bce9564f69..62a2dc8658 100644 --- a/tests/unit/src/sqlbuild/rule_engine/_helpers/engine/test_hermeticity.py +++ b/tests/unit/src/sqlbuild/rule_engine/_helpers/engine/test_hermeticity.py @@ -4,9 +4,8 @@ import pytest -from sqlbuild._native import NonHermeticRuleError +from sqlbuild._native import NonHermeticRuleError, Rule from sqlbuild.rule_engine._helpers.engine.hermeticity import verify_custom_rules -from sqlbuild.rule_engine.models import Rule from tests.unit.src.sqlbuild.rule_engine._helpers.engine._test_types import ( CustomRuleImportTestCase, ) diff --git a/tests/unit/src/sqlbuild/rule_engine/_helpers/engine/test_incremental_native_rules.py b/tests/unit/src/sqlbuild/rule_engine/_helpers/engine/test_incremental_native_rules.py index 40c3583455..5e03cc34d8 100644 --- a/tests/unit/src/sqlbuild/rule_engine/_helpers/engine/test_incremental_native_rules.py +++ b/tests/unit/src/sqlbuild/rule_engine/_helpers/engine/test_incremental_native_rules.py @@ -3,8 +3,7 @@ import pytest -from sqlbuild._native import RuleExemption, RuleIgnore -from sqlbuild.rule_engine.models import RulesResult +from sqlbuild._native import RuleExemption, RuleIgnore, RulesResult from tests.unit.src.sqlbuild.rule_engine._helpers.engine._test_types import ( IncrementalCacheCountsTestCase, IncrementalRulesModeTestCase, diff --git a/tests/unit/src/sqlbuild/rule_engine/_helpers/engine/test_native_memo.py b/tests/unit/src/sqlbuild/rule_engine/_helpers/engine/test_native_memo.py index e66b18c738..8662e1d417 100644 --- a/tests/unit/src/sqlbuild/rule_engine/_helpers/engine/test_native_memo.py +++ b/tests/unit/src/sqlbuild/rule_engine/_helpers/engine/test_native_memo.py @@ -2,7 +2,7 @@ import pytest -from sqlbuild.rule_engine.models import RulesResult +from sqlbuild._native import RulesResult from tests.unit.src.sqlbuild.rule_engine._helpers.engine._test_types import NativeMemoTestCase from tests.unit.src.sqlbuild.rule_engine._helpers.engine.helpers import evaluate_contract_rule diff --git a/tests/unit/src/sqlbuild/rule_engine/_helpers/host/helpers.py b/tests/unit/src/sqlbuild/rule_engine/_helpers/host/helpers.py index 7d785aac72..6b9654ef5c 100644 --- a/tests/unit/src/sqlbuild/rule_engine/_helpers/host/helpers.py +++ b/tests/unit/src/sqlbuild/rule_engine/_helpers/host/helpers.py @@ -19,7 +19,9 @@ CustomHostObserver, CustomRulesOutcome, OpaqueModuleStateError, + Rule, RulesCacheConfig, + RulesConfig, build_rule_catalogue, project_with, ) @@ -27,7 +29,6 @@ from sqlbuild.compiler.compile.types import CompiledProjectView from sqlbuild.rule_engine._helpers.host.module_state import module_state_token, rule_namespaces from sqlbuild.rule_engine.classes.module_state_monitor import ModuleStateMonitor -from sqlbuild.rule_engine.models import Rule, RulesConfig from tests.unit.src.sqlbuild.rule_engine.main.evaluate.helpers import build_project _MODEL_RULE: str = "XSQBRT101" diff --git a/tests/unit/src/sqlbuild/rule_engine/_helpers/host/test_custom_host_pool.py b/tests/unit/src/sqlbuild/rule_engine/_helpers/host/test_custom_host_pool.py index ec91915212..7f2dd7196c 100644 --- a/tests/unit/src/sqlbuild/rule_engine/_helpers/host/test_custom_host_pool.py +++ b/tests/unit/src/sqlbuild/rule_engine/_helpers/host/test_custom_host_pool.py @@ -14,7 +14,6 @@ ) from sqlbuild.compiler.compile.types import CompiledProjectView from sqlbuild.rule_engine._helpers.host import custom_host -from sqlbuild.rule_engine.constants import CUSTOM_RULES_CACHE_FILE from tests.unit.src.sqlbuild.rule_engine._helpers.host._test_types import ( AwaitInputsTestCase, AwaitPublishedInputsTestCase, @@ -83,9 +82,7 @@ def test_given_partitioned_hosts_when_evaluating_then_findings_and_cache_match_o detects_duplicates=test_case.detects_duplicates, ) project: CompiledProjectView = orders_project(model_count=test_case.model_count) - cache_path: Path = rules_bulk_cache_path( - project_dir=tmp_path, file_name=CUSTOM_RULES_CACHE_FILE - ) + cache_path: Path = rules_bulk_cache_path(project_dir=tmp_path, file_name="custom-rules.json") cache_glob: str = cache_path.relative_to(tmp_path).as_posix() cache_enabled: bool = test_case.cache_enabled diff --git a/tests/unit/src/sqlbuild/rule_engine/_helpers/host/test_module_state.py b/tests/unit/src/sqlbuild/rule_engine/_helpers/host/test_module_state.py index 6036adc7b3..2718919e37 100644 --- a/tests/unit/src/sqlbuild/rule_engine/_helpers/host/test_module_state.py +++ b/tests/unit/src/sqlbuild/rule_engine/_helpers/host/test_module_state.py @@ -48,6 +48,18 @@ model_count=4, expected_stateful=_STATELESS, ), + ModuleStateCase( + description="module-level rule option record", + files=state_case_files( + header=( + "from sqlbuild.rules import RuleOption\n" + "LIMIT = RuleOption.integer(name='limit', default=3, description='Limit.')\n" + ), + body="_ = LIMIT.default", + ), + model_count=4, + expected_stateful=_STATELESS, + ), ModuleStateCase( description="dict mutation", files=state_case_files( diff --git a/tests/unit/src/sqlbuild/rule_engine/_helpers/run/test_native_rows.py b/tests/unit/src/sqlbuild/rule_engine/_helpers/run/test_native_rows.py index 79b54483ec..3e1d2306a4 100644 --- a/tests/unit/src/sqlbuild/rule_engine/_helpers/run/test_native_rows.py +++ b/tests/unit/src/sqlbuild/rule_engine/_helpers/run/test_native_rows.py @@ -5,9 +5,8 @@ import pytest import sqlbuild._native as native -from sqlbuild._native import RulesCacheConfig, RulesError +from sqlbuild._native import RulesCacheConfig, RulesConfig, RulesError, RulesResult from sqlbuild.compiler.compile.types import CompiledProjectView -from sqlbuild.rule_engine.models import RulesConfig, RulesResult from tests.unit.src.sqlbuild.rule_engine._helpers.engine.helpers import evaluate_contract_rule from tests.unit.src.sqlbuild.rule_engine._helpers.run._test_types import ( NativeRowsEncodeErrorTestCase, diff --git a/tests/unit/src/sqlbuild/rule_engine/classes/audit_facts/test_audit_facts.py b/tests/unit/src/sqlbuild/rule_engine/classes/audit_facts/test_audit_facts.py index f48e681c8e..633f04db9d 100644 --- a/tests/unit/src/sqlbuild/rule_engine/classes/audit_facts/test_audit_facts.py +++ b/tests/unit/src/sqlbuild/rule_engine/classes/audit_facts/test_audit_facts.py @@ -5,11 +5,11 @@ import pytest from sqlbuild._native import ( + Model, project_with, ) from sqlbuild.compiler.compile.types import CompiledAuditView, CompiledProjectView from sqlbuild.rule_engine.classes.audit_facts import AuditFacts -from sqlbuild.rule_engine.models import Model from tests.unit.src.sqlbuild.rule_engine.classes.audit_facts._test_types import ( AuditFactsTestCase, ) diff --git a/tests/unit/src/sqlbuild/rule_engine/classes/model_test_facts/test_model_test_facts.py b/tests/unit/src/sqlbuild/rule_engine/classes/model_test_facts/test_model_test_facts.py index 4325436ffd..ade7e7839a 100644 --- a/tests/unit/src/sqlbuild/rule_engine/classes/model_test_facts/test_model_test_facts.py +++ b/tests/unit/src/sqlbuild/rule_engine/classes/model_test_facts/test_model_test_facts.py @@ -7,11 +7,11 @@ from sqlbuild._native import ( CompiledSqlTest, + Model, project_with, ) from sqlbuild.compiler.compile.types import CompiledProjectView, SqlTestMode from sqlbuild.rule_engine.classes.model_test_facts import TestFacts as ModelTestFacts -from sqlbuild.rule_engine.models import Model from tests.unit.src.sqlbuild.rule_engine._helpers.engine.helpers import ( direct_sql_test, model_sql_test, diff --git a/tests/unit/src/sqlbuild/rule_engine/main/evaluate/_test_types.py b/tests/unit/src/sqlbuild/rule_engine/main/evaluate/_test_types.py index 19f198096b..7383671c22 100644 --- a/tests/unit/src/sqlbuild/rule_engine/main/evaluate/_test_types.py +++ b/tests/unit/src/sqlbuild/rule_engine/main/evaluate/_test_types.py @@ -2,8 +2,7 @@ from dataclasses import dataclass, field -from sqlbuild._native import CompileSqlReference -from sqlbuild.rule_engine.models import RulesConfig +from sqlbuild._native import CompileSqlReference, RulesConfig @dataclass(frozen=True) diff --git a/tests/unit/src/sqlbuild/rule_engine/main/evaluate/test_cache_and_suppressions.py b/tests/unit/src/sqlbuild/rule_engine/main/evaluate/test_cache_and_suppressions.py index 0d1df9fc33..0142313abe 100644 --- a/tests/unit/src/sqlbuild/rule_engine/main/evaluate/test_cache_and_suppressions.py +++ b/tests/unit/src/sqlbuild/rule_engine/main/evaluate/test_cache_and_suppressions.py @@ -8,13 +8,14 @@ from sqlbuild._native import ( RuleExemption, RuleIgnore, + RulesConfig, RulesError, + RulesResult, compiled_model_record, project_with, ) from sqlbuild._native import evaluate_project_rules as evaluate from sqlbuild.compiler.compile.types import CompiledProjectView -from sqlbuild.rule_engine.models import RulesConfig, RulesResult from tests.unit.src.sqlbuild.rule_engine.main.evaluate._test_types import PolicyBehaviorTestCase from tests.unit.src.sqlbuild.rule_engine.main.evaluate.helpers import build_project diff --git a/tests/unit/src/sqlbuild/rule_engine/main/evaluate/test_evaluate.py b/tests/unit/src/sqlbuild/rule_engine/main/evaluate/test_evaluate.py index 9105fac042..eae4595e5b 100644 --- a/tests/unit/src/sqlbuild/rule_engine/main/evaluate/test_evaluate.py +++ b/tests/unit/src/sqlbuild/rule_engine/main/evaluate/test_evaluate.py @@ -5,9 +5,8 @@ import pytest -from sqlbuild._native import CompileSqlReference, SelectStarAllow +from sqlbuild._native import CompileSqlReference, Finding, RulesConfig, RulesResult, SelectStarAllow from sqlbuild._native import evaluate_project_rules as evaluate -from sqlbuild.rule_engine.models import Finding, RulesConfig, RulesResult from tests.unit.src.sqlbuild.rule_engine.main.evaluate._test_types import ( PolicyEvaluationTestCase, ) diff --git a/tests/unit/src/sqlbuild/rule_engine/main/evaluate_rule/test_evaluate_rule.py b/tests/unit/src/sqlbuild/rule_engine/main/evaluate_rule/test_evaluate_rule.py index a6ad12eae4..01e1b5d527 100644 --- a/tests/unit/src/sqlbuild/rule_engine/main/evaluate_rule/test_evaluate_rule.py +++ b/tests/unit/src/sqlbuild/rule_engine/main/evaluate_rule/test_evaluate_rule.py @@ -6,11 +6,10 @@ import pytest -from sqlbuild._native import RulesCacheConfig, RulesError +from sqlbuild._native import RulesCacheConfig, RulesConfig, RulesError, RulesResult from sqlbuild._native import evaluate_project_rules as evaluate from sqlbuild.compiler.compile.types import CompiledProjectView from sqlbuild.rule_engine.constants import MIN_CUSTOM_RULE_TEST_CASES -from sqlbuild.rule_engine.models import RulesConfig, RulesResult from sqlbuild.rule_engine.types import RuleOptionValue from sqlbuild.rules.testing import RuleCase, RuleResult, evaluate_rule from tests.unit.src.sqlbuild.rule_engine.main.evaluate.helpers import build_project diff --git a/tests/unit/src/sqlbuild/rule_engine/main/render_result/_test_types.py b/tests/unit/src/sqlbuild/rule_engine/main/render_result/_test_types.py index 45254a34f8..339e9ae891 100644 --- a/tests/unit/src/sqlbuild/rule_engine/main/render_result/_test_types.py +++ b/tests/unit/src/sqlbuild/rule_engine/main/render_result/_test_types.py @@ -2,7 +2,7 @@ from dataclasses import dataclass -from sqlbuild.rule_engine.models import RulesResult +from sqlbuild._native import RulesResult @dataclass(frozen=True) diff --git a/tests/unit/src/sqlbuild/rule_engine/main/render_result/test_render_result.py b/tests/unit/src/sqlbuild/rule_engine/main/render_result/test_render_result.py index e2ac06ade3..c526a87aa8 100644 --- a/tests/unit/src/sqlbuild/rule_engine/main/render_result/test_render_result.py +++ b/tests/unit/src/sqlbuild/rule_engine/main/render_result/test_render_result.py @@ -6,8 +6,8 @@ import pytest +from sqlbuild._native import Finding, RulesResult from sqlbuild._native import format_rules_result as format_result -from sqlbuild.rule_engine.models import Finding, RulesResult from tests.unit.src.sqlbuild.rule_engine.main.render_result._test_types import ( RenderResultParityTestCase, ) diff --git a/tests/unit/src/sqlbuild/rule_engine/main/skills/test_skills.py b/tests/unit/src/sqlbuild/rule_engine/main/skills/test_skills.py index 4274b49b5a..bdccd3cc09 100644 --- a/tests/unit/src/sqlbuild/rule_engine/main/skills/test_skills.py +++ b/tests/unit/src/sqlbuild/rule_engine/main/skills/test_skills.py @@ -5,9 +5,8 @@ import pytest -from sqlbuild._native import RuleExemption, RulesError +from sqlbuild._native import RuleExemption, RulesConfig, RulesError from sqlbuild._native import install_rules_skills as install_skills -from sqlbuild.rule_engine.models import RulesConfig from tests.unit.src.sqlbuild.rule_engine.main.skills._test_types import ( SkillDivergenceTestCase, SkillFreshnessTestCase, From 7485cf6eef3da515dd0d92501697068481396273 Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Thu, 15 Oct 2026 18:34:32 +0100 Subject: [PATCH 07/48] refactor: publish eligible direct-build source freshness natively --- .../_helpers/build_command/execution.rs | 20 +- .../src/bindings/_helpers/freshness/mod.rs | 1 + .../_helpers/freshness/publication.rs | 319 ++++++++++++++++++ .../_helpers/freshness/registration.rs | 4 +- python-boundary.toml | 2 +- src/sqlbuild/_native.pyi | 8 + .../_helpers/freshness/source_freshness.py | 152 --------- .../_helpers/test_source_freshness.py | 4 +- 8 files changed, 339 insertions(+), 171 deletions(-) create mode 100644 crates/sqlbuild-python/src/bindings/_helpers/freshness/publication.rs delete mode 100644 src/sqlbuild/cli/commands/_helpers/freshness/source_freshness.py diff --git a/crates/sqlbuild-python/src/bindings/_helpers/build_command/execution.rs b/crates/sqlbuild-python/src/bindings/_helpers/build_command/execution.rs index 2cdade0728..b5e49386a6 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/build_command/execution.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/build_command/execution.rs @@ -20,6 +20,7 @@ use crate::bindings::_helpers::check::check_core::{ }; use crate::bindings::_helpers::compile_driver::python_calls::{call_with, imported, within}; use crate::bindings::_helpers::cursor::overrides::{cursor_integer, cursor_timestamp}; +use crate::bindings::_helpers::freshness::publication::append_freshness_records; use crate::bindings::_helpers::graph::python_node_graph::python_node_graph; const NATIVE_MODULE: &str = "sqlbuild._native"; @@ -27,7 +28,6 @@ const CURSOR_OVERRIDES_MODULE: &str = "sqlbuild._native"; const PLANNER_MODELS: &str = "sqlbuild._native"; const BUILD_MODELS: &str = "sqlbuild.executor.build.models"; const PIPELINE_RUN_MODULE: &str = "sqlbuild.executor.pipeline.main.run"; -const FRESHNESS_MODULE: &str = "sqlbuild.cli.commands._helpers.freshness.source_freshness"; const COST_CONTEXT_MODULE: &str = "sqlbuild._native"; const CHECKS_MODULE: &str = "sqlbuild.executor.python_nodes.main._checks"; const EXECUTOR_PYTHON_MODELS: &str = "sqlbuild.executor.python_nodes.models"; @@ -403,19 +403,11 @@ pub(crate) fn execute_build_plan<'py>( ("initial_state", initial_state), ], )?; - call_with( - &imported( - py, - FRESHNESS_MODULE, - "append_eligible_direct_source_freshness_records", - )?, - &[ - ("plan", plan), - ("result", result.clone()), - ("adapter", adapter), - ("connection_config", connection_config), - ("run_id", project.getattr("run_id")?), - ], + append_freshness_records( + (&plan, &result), + &adapter, + &connection_config, + &project.getattr("run_id")?, )?; lifecycle.finalize(py)?; Ok(Outcome { diff --git a/crates/sqlbuild-python/src/bindings/_helpers/freshness/mod.rs b/crates/sqlbuild-python/src/bindings/_helpers/freshness/mod.rs index 8d6c6cb4b1..98a0933a17 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/freshness/mod.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/freshness/mod.rs @@ -3,6 +3,7 @@ pub(crate) mod freshness_command; pub(crate) mod freshness_observe; pub(crate) mod observation; pub(crate) mod planning; +pub(crate) mod publication; pub(crate) mod registration; pub(crate) mod state_io; pub(crate) mod support; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/freshness/publication.rs b/crates/sqlbuild-python/src/bindings/_helpers/freshness/publication.rs new file mode 100644 index 0000000000..8ee5b46af2 --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/freshness/publication.rs @@ -0,0 +1,319 @@ +//! Append observed source freshness to state once the models it affected have built. + +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyModule, PyModuleMethods, PyResult, Python}; +use pyo3::types::{PyDict, PyDictMethods, PyFrozenSet, PyString, PyTuple}; +use pyo3::{pyfunction, wrap_pyfunction}; + +use crate::bindings::_helpers::compile_driver::python_calls::{ + KeywordArguments, call_with, imported, within, +}; +use crate::bindings::_helpers::records::constants::NATIVE_MODULE; + +/// Module of the execution status enum. +const SCHEDULING_TYPES_MODULE: &str = "sqlbuild.executor.scheduling.types"; +/// Module of the run-scope context manager. +const RUN_SCOPE_MODULE: &str = "sqlbuild.runtime.observability.main.run_scope"; +/// Module of the operation lifecycle context manager. +const LIFECYCLE_MODULE: &str = "sqlbuild.runtime.observability.classes.operation_lifecycle"; +/// Module of the operation attributes record. +const OBSERVABILITY_MODELS_MODULE: &str = "sqlbuild.runtime.observability.models"; + +/// Observed records grouped by `(database, schema)` in first-seen order. +type RecordsByLocation<'py> = Vec<( + (Bound<'py, PyAny>, Bound<'py, PyAny>), + Vec>, +)>; + +/// The names of the selected models that built successfully. +fn successful_model_names<'py>(result: &Bound<'py, PyAny>) -> PyResult>> { + let py: Python<'py> = result.py(); + let success: Bound<'py, PyAny> = + imported(py, SCHEDULING_TYPES_MODULE, "ExecutionStatus")?.getattr("SUCCESS")?; + let mut names: Vec> = Vec::new(); + for model_result in result.getattr("model_results")?.try_iter()? { + let model_result: Bound<'py, PyAny> = model_result?; + let status: Bound<'py, PyAny> = model_result.getattr("status")?; + let value: Bound<'py, PyAny> = status.getattr_opt("value")?.unwrap_or(status); + if value.eq(&success)? { + names.push(model_result.getattr("model_name")?); + } + } + Ok(names) +} + +/// `record` added under each distinct schema location of `affected`, in sorted model order. +fn collect_record<'py>( + mut grouped: RecordsByLocation<'py>, + record: &Bound<'py, PyAny>, + (affected, entries_by_name): (&Bound<'py, PyAny>, &Bound<'py, PyDict>), +) -> PyResult> { + let py: Python<'py> = record.py(); + let sorted: Bound<'py, PyAny> = imported(py, "builtins", "sorted")?.call1((affected,))?; + let mut recorded: Vec<(Bound<'py, PyAny>, Bound<'py, PyAny>)> = Vec::new(); + for model_name in sorted.try_iter()? { + let Some(entry) = entries_by_name.get_item(model_name?)? else { + continue; + }; + let destination: Bound<'py, PyAny> = entry.getattr("destination")?; + let schema: Bound<'py, PyAny> = destination.getattr("schema")?; + if schema.is_none() { + continue; + } + let location: (Bound<'py, PyAny>, Bound<'py, PyAny>) = + (destination.getattr("database")?, schema); + if contains_location(&recorded, &location)? { + continue; + } + recorded.push(location.clone()); + let mut position: Option = None; + for (index, (known, _)) in grouped.iter().enumerate() { + if same_location(known, &location)? { + position = Some(index); + break; + } + } + match position { + Some(index) => grouped[index].1.push(record.clone()), + None => grouped.push((location, vec![record.clone()])), + } + } + Ok(grouped) +} + +/// Whether two `(database, schema)` locations are equal. +fn same_location( + left: &(Bound<'_, PyAny>, Bound<'_, PyAny>), + right: &(Bound<'_, PyAny>, Bound<'_, PyAny>), +) -> PyResult { + Ok(left.0.eq(&right.0)? && left.1.eq(&right.1)?) +} + +/// Whether `locations` holds `location`. +fn contains_location( + locations: &[(Bound<'_, PyAny>, Bound<'_, PyAny>)], + location: &(Bound<'_, PyAny>, Bound<'_, PyAny>), +) -> PyResult { + for known in locations { + if same_location(known, location)? { + return Ok(true); + } + } + Ok(false) +} + +/// The observed records to publish, keyed by the schema of each affected model. +fn eligible_records<'py>( + plan: &Bound<'py, PyAny>, + result: &Bound<'py, PyAny>, + run_id: &Bound<'py, PyAny>, +) -> PyResult> { + let py: Python<'py> = plan.py(); + let freshness: Bound<'py, PyAny> = plan.getattr("source_freshness")?; + if freshness.is_none() || freshness.getattr("propagation")?.is_none() { + return Ok(Vec::new()); + } + let entries_by_name: Bound<'py, PyDict> = PyDict::new(py); + for entry in plan.getattr("model_entries")?.try_iter()? { + let entry: Bound<'py, PyAny> = entry?; + entries_by_name.set_item(entry.getattr("name")?, entry)?; + } + if entries_by_name.is_empty() { + return Ok(Vec::new()); + } + let selected: Bound<'py, PyFrozenSet> = PyFrozenSet::new(py, entries_by_name.keys())?; + let successful: Bound<'py, PyFrozenSet> = + PyFrozenSet::new(py, successful_model_names(result)?)?; + let observed: Bound<'py, PyDict> = PyDict::new(py); + for record in freshness.getattr("observed_records")?.try_iter()? { + let record: Bound<'py, PyAny> = record?; + observed.set_item(record.getattr("identity")?, record)?; + } + let replace: Bound<'py, PyAny> = imported(py, "dataclasses", "replace")?; + let changes: Bound<'py, PyDict> = PyDict::new(py); + changes.set_item("run_id", run_id)?; + let mut grouped: RecordsByLocation<'py> = Vec::new(); + let changed: Bound<'py, PyAny> = freshness + .getattr("propagation")? + .getattr("changed_source_model_names")?; + for item in changed.call_method0("items")?.try_iter()? { + let (identity, affected): (Bound<'py, PyAny>, Bound<'py, PyAny>) = item?.extract()?; + let affected: Bound<'py, PyAny> = affected.call_method1("__and__", (&selected,))?; + if !affected.is_truthy()? + || !affected + .call_method1("__le__", (&successful,))? + .is_truthy()? + { + continue; + } + let Some(record) = observed.get_item(identity)? else { + continue; + }; + let replaced: Bound<'py, PyAny> = replace.call((record,), Some(&changes))?; + grouped = collect_record(grouped, &replaced, (&affected, &entries_by_name))?; + } + Ok(grouped) +} + +/// `SourceFreshnessRenderers` bound to `adapter`. +fn renderers<'py>(adapter: &Bound<'py, PyAny>) -> PyResult> { + call_with( + &imported(adapter.py(), NATIVE_MODULE, "SourceFreshnessRenderers")?, + &[ + ( + "render_qualified_name", + adapter.getattr("render_qualified_name")?, + ), + ( + "render_framework_type", + adapter.getattr("render_framework_type")?, + ), + ( + "render_insert_records_sql", + adapter.getattr("render_insert_source_freshness_records_sql")?, + ), + ( + "render_create_table_sql", + adapter.getattr("render_create_source_freshness_table_sql")?, + ), + ( + "render_create_index_sqls", + adapter.getattr("render_create_source_freshness_index_sqls")?, + ), + ], + ) +} + +/// Write each location's records, one cost scope per source. +fn write_records<'py>( + adapter: &Bound<'py, PyAny>, + connection: &Bound<'py, PyAny>, + grouped: &RecordsByLocation<'py>, +) -> PyResult<()> { + let py: Python<'py> = adapter.py(); + let cost_context: Bound<'py, PyAny> = imported(py, NATIVE_MODULE, "CostContext")?; + let write: Bound<'py, PyAny> = imported(py, NATIVE_MODULE, "write_source_freshness_records")?; + for ((database, schema), records) in grouped { + let mut by_source: Vec<(Bound<'py, PyAny>, Vec>)> = Vec::new(); + for record in records { + let source_name: Bound<'py, PyAny> = record.getattr("source_name")?; + let mut position: Option = None; + for (index, (known, _)) in by_source.iter().enumerate() { + if known.eq(&source_name)? { + position = Some(index); + break; + } + } + match position { + Some(index) => by_source[index].1.push(record.clone()), + None => by_source.push((source_name, vec![record.clone()])), + } + } + for (source_name, source_records) in by_source { + let scope: Bound<'py, PyAny> = call_with( + &cost_context.getattr("resource_scope")?, + &[ + ("resource_type", PyString::new(py, "source").into_any()), + ("resource_name", source_name), + ( + "phase", + PyString::new(py, "freshness_finalization").into_any(), + ), + ], + )?; + within(&scope, |_| { + call_with( + &write, + &[ + ("connection", connection.clone()), + ("execute", adapter.getattr("execute")?), + ("database", database.clone()), + ("schema", schema.clone()), + ("records", PyTuple::new(py, &source_records)?.into_any()), + ("renderers", renderers(adapter)?), + ], + ) + })?; + } + } + Ok(()) +} + +/// Append observed source freshness only after the affected selected models succeed. +pub(crate) fn append_freshness_records<'py>( + (plan, result): (&Bound<'py, PyAny>, &Bound<'py, PyAny>), + adapter: &Bound<'py, PyAny>, + connection_config: &Bound<'py, PyAny>, + run_id: &Bound<'py, PyAny>, +) -> PyResult<()> { + let py: Python<'py> = plan.py(); + let grouped: RecordsByLocation<'py> = eligible_records(plan, result, run_id)?; + let record_count: usize = grouped.iter().map(|(_, records)| records.len()).sum(); + if record_count == 0 { + return Ok(()); + } + let run_scope: Bound<'py, PyAny> = + imported(py, RUN_SCOPE_MODULE, "run_scope")?.call1((run_id,))?; + within(&run_scope, |_| { + let metadata: Bound<'py, PyDict> = PyDict::new(py); + metadata.set_item("item_count", record_count)?; + let attributes: Bound<'py, PyAny> = call_with( + &imported(py, OBSERVABILITY_MODELS_MODULE, "OperationAttributes")?, + &[ + ("phase", PyString::new(py, "publish").into_any()), + ("target_kind", PyString::new(py, "state_batch").into_any()), + ], + )?; + let lifecycle: Bound<'py, PyAny> = call_with( + &imported(py, LIFECYCLE_MODULE, "OperationLifecycle")?, + &[ + ("operation_kind", PyString::new(py, "freshness").into_any()), + ( + "operation_name", + PyString::new(py, "source_freshness_publication").into_any(), + ), + ("metadata", metadata.into_any()), + ("attributes", attributes), + ], + )?; + within(&lifecycle, |_| { + let connection: Bound<'py, PyAny> = + adapter.call_method1("connect", (connection_config,))?; + let written: PyResult<()> = write_records(adapter, &connection, &grouped); + let closed: PyResult> = adapter.call_method1("close", (&connection,)); + written?; + closed?; + Ok(()) + }) + }) +} + +/// Append observed source freshness only after the affected selected models succeed. +#[pyfunction] +#[pyo3(signature = (**options))] +fn append_eligible_direct_source_freshness_records<'py>( + py: Python<'py>, + options: Option>, +) -> PyResult<()> { + let arguments: KeywordArguments<'py> = KeywordArguments::read( + py, + "append_eligible_direct_source_freshness_records", + options, + ( + &["plan", "result", "adapter", "connection_config", "run_id"], + &[], + ), + )?; + append_freshness_records( + (&arguments.value("plan")?, &arguments.value("result")?), + &arguments.value("adapter")?, + &arguments.value("connection_config")?, + &arguments.value("run_id")?, + ) +} + +pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { + module.add_function(wrap_pyfunction!( + append_eligible_direct_source_freshness_records, + module + )?) +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/freshness/registration.rs b/crates/sqlbuild-python/src/bindings/_helpers/freshness/registration.rs index 0d0d44f770..4f10818a2a 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/freshness/registration.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/freshness/registration.rs @@ -3,7 +3,8 @@ use pyo3::prelude::{Bound, PyModule, PyResult}; use crate::bindings::_helpers::freshness::{ - exception_types, freshness_command, freshness_observe, observation, planning, state_io, + exception_types, freshness_command, freshness_observe, observation, planning, publication, + state_io, }; pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { @@ -12,5 +13,6 @@ pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { freshness_observe::register(module)?; observation::register(module)?; planning::register(module)?; + publication::register(module)?; state_io::register(module) } diff --git a/python-boundary.toml b/python-boundary.toml index a60d534a55..c8fe9fbbee 100644 --- a/python-boundary.toml +++ b/python-boundary.toml @@ -2,7 +2,7 @@ # migration work that moves to Rust. `remaining_lines` must equal the current count of that work: # a PR that moves code lowers it, and adding movable Python fails the check. # Native failures must surface as errors; `handled_native_errors` lists the reviewed exceptions. -remaining_lines = 439 +remaining_lines = 287 [[stays]] path = "executor" diff --git a/src/sqlbuild/_native.pyi b/src/sqlbuild/_native.pyi index bf902c774e..8d4c8834b6 100644 --- a/src/sqlbuild/_native.pyi +++ b/src/sqlbuild/_native.pyi @@ -6595,6 +6595,14 @@ def write_source_freshness_records( records: tuple[SourceFreshnessRecord, ...], renderers: SourceFreshnessRenderers, ) -> None: ... +def append_eligible_direct_source_freshness_records( + *, + plan: PlanOutput, + result: object, + adapter: object, + connection_config: dict[str, object], + run_id: str, +) -> None: ... def observe_configured_source_freshness( *, adapter: StrictAdapter, connection: object, source: SourceEntry, observed_at: datetime ) -> SourceFreshnessObservation: ... diff --git a/src/sqlbuild/cli/commands/_helpers/freshness/source_freshness.py b/src/sqlbuild/cli/commands/_helpers/freshness/source_freshness.py deleted file mode 100644 index 2cba6c4c93..0000000000 --- a/src/sqlbuild/cli/commands/_helpers/freshness/source_freshness.py +++ /dev/null @@ -1,152 +0,0 @@ -"""Direct build source freshness state helpers.""" - -from __future__ import annotations - -from dataclasses import replace -from typing import Any - -from sqlbuild._native import ( - CostContext, - DirectSourceFreshnessPlanningResult, - ModelPlanEntry, - PlanOutput, - SourceFreshnessIdentity, - SourceFreshnessRecord, - SourceFreshnessRenderers, - write_source_freshness_records, -) -from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter -from sqlbuild.executor.build.models import BuildExecutionResult -from sqlbuild.executor.scheduling.types import ExecutionStatus -from sqlbuild.runtime.observability.classes.operation_lifecycle import OperationLifecycle -from sqlbuild.runtime.observability.main.run_scope import run_scope -from sqlbuild.runtime.observability.models import OperationAttributes - - -def append_eligible_direct_source_freshness_records( - *, - plan: PlanOutput, - result: BuildExecutionResult, - adapter: BaseAdapter, - connection_config: dict[str, object], - run_id: str, -) -> None: - """Append observed source freshness only after affected selected models succeed.""" - - source_freshness: DirectSourceFreshnessPlanningResult | None = plan.source_freshness - if source_freshness is None or source_freshness.propagation is None: - return - - selected_model_names: frozenset[str] = frozenset(entry.name for entry in plan.model_entries) - if not selected_model_names: - return - - successful_model_names: frozenset[str] = frozenset( - model_result.model_name - for model_result in result.model_results - if getattr(model_result.status, "value", model_result.status) == ExecutionStatus.SUCCESS - ) - observed_records_by_identity: dict[SourceFreshnessIdentity, SourceFreshnessRecord] = { - record.identity: record for record in source_freshness.observed_records - } - entries_by_name: dict[str, ModelPlanEntry] = {entry.name: entry for entry in plan.model_entries} - - records_by_location: dict[tuple[str | None, str], list[SourceFreshnessRecord]] = {} - identity: SourceFreshnessIdentity - affected_model_names: frozenset[str] - for ( - identity, - affected_model_names, - ) in source_freshness.propagation.changed_source_model_names.items(): - affected_selected_model_names: frozenset[str] = affected_model_names & selected_model_names - if not affected_selected_model_names: - continue - if not affected_selected_model_names <= successful_model_names: - continue - record: SourceFreshnessRecord | None = observed_records_by_identity.get(identity) - if record is None: - continue - records_by_location = _collect_record_for_affected_model_schemas( - records_by_location=records_by_location, - record=replace(record, run_id=run_id), - affected_model_names=affected_selected_model_names, - entries_by_name=entries_by_name, - ) - record_count: int = sum(len(records) for records in records_by_location.values()) - if record_count == 0: - return - with run_scope(run_id): - with OperationLifecycle( - operation_kind="freshness", - operation_name="source_freshness_publication", - metadata={"item_count": record_count}, - attributes=OperationAttributes(phase="publish", target_kind="state_batch"), - ): - connection: Any = adapter.connect(connection_config) - try: - _append_records_by_location( - adapter=adapter, - connection=connection, - records_by_location=records_by_location, - ) - finally: - adapter.close(connection) - - -def _collect_record_for_affected_model_schemas( - *, - records_by_location: dict[tuple[str | None, str], list[SourceFreshnessRecord]], - record: SourceFreshnessRecord, - affected_model_names: frozenset[str], - entries_by_name: dict[str, ModelPlanEntry], -) -> dict[tuple[str | None, str], list[SourceFreshnessRecord]]: - recorded_locations: set[tuple[str | None, str]] = set() - model_name: str - for model_name in sorted(affected_model_names): - entry: ModelPlanEntry | None = entries_by_name.get(model_name) - if entry is None or entry.destination.schema is None: - continue - location: tuple[str | None, str] = (entry.destination.database, entry.destination.schema) - if location in recorded_locations: - continue - recorded_locations.add(location) - records_by_location.setdefault(location, []).append(record) - return records_by_location - - -def _append_records_by_location( - *, - adapter: BaseAdapter, - connection: Any, - records_by_location: dict[tuple[str | None, str], list[SourceFreshnessRecord]], -) -> None: - location: tuple[str | None, str] - records: list[SourceFreshnessRecord] - for location, records in records_by_location.items(): - database, schema = location - records_by_source_name: dict[str, list[SourceFreshnessRecord]] = {} - record: SourceFreshnessRecord - for record in records: - records_by_source_name.setdefault(record.source_name, []).append(record) - source_name: str - source_records: list[SourceFreshnessRecord] - for source_name, source_records in records_by_source_name.items(): - with CostContext.resource_scope( - resource_type="source", - resource_name=source_name, - phase="freshness_finalization", - ): - write_source_freshness_records( - connection=connection, - execute=adapter.execute, - database=database, - schema=schema, - records=tuple(source_records), - renderers=SourceFreshnessRenderers( - render_qualified_name=adapter.render_qualified_name, - render_framework_type=adapter.render_framework_type, - render_insert_records_sql=adapter.render_insert_source_freshness_records_sql, - render_create_table_sql=adapter.render_create_source_freshness_table_sql, - render_create_index_sqls=adapter.render_create_source_freshness_index_sqls, - ), - ) diff --git a/tests/unit/src/sqlbuild/cli/commands/_helpers/test_source_freshness.py b/tests/unit/src/sqlbuild/cli/commands/_helpers/test_source_freshness.py index 58400b0b0f..2781355b6b 100644 --- a/tests/unit/src/sqlbuild/cli/commands/_helpers/test_source_freshness.py +++ b/tests/unit/src/sqlbuild/cli/commands/_helpers/test_source_freshness.py @@ -11,11 +11,9 @@ DirectSourceFreshnessPlanningResult, DirectSourceFreshnessPropagationResult, PlanOutput, -) -from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter -from sqlbuild.cli.commands._helpers.freshness.source_freshness import ( append_eligible_direct_source_freshness_records, ) +from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter from sqlbuild.cost.models import CostResourceContext from sqlbuild.executor.build.models import BuildExecutionResult from sqlbuild.executor.build.types import BuildStatus From b766766ac68718c72789eef8072749ca87bbbf2f Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Thu, 15 Oct 2026 18:37:59 +0100 Subject: [PATCH 08/48] chore: record staying constant and rule API shapes and drop empty package markers --- python-boundary.toml | 52 ++++++++++++++++++- src/sqlbuild/__init__.py | 1 - src/sqlbuild/adapter/__init__.py | 1 - src/sqlbuild/cli/__init__.py | 1 - .../cli/commands/_helpers/entry/__init__.py | 1 - src/sqlbuild/cli/commands/classes/__init__.py | 0 src/sqlbuild/cli/commands/main/__init__.py | 1 - .../cli/commands/main/dbt/__init__.py | 0 .../cli/commands/main/inspection/__init__.py | 0 .../cli/commands/main/project/__init__.py | 0 .../cli/commands/main/state/__init__.py | 0 .../cli/commands/main/workspace/__init__.py | 0 src/sqlbuild/cli/compile_reuse/__init__.py | 1 - src/sqlbuild/cli/output/__init__.py | 1 - src/sqlbuild/compiler/__init__.py | 1 - .../compiler/analysis_session/__init__.py | 1 - src/sqlbuild/compiler/auditing/__init__.py | 1 - src/sqlbuild/compiler/compile/__init__.py | 1 - .../compiler/compile/_helpers/__init__.py | 1 - .../compiler/contract_adoption/__init__.py | 1 - src/sqlbuild/compiler/contracts/__init__.py | 1 - src/sqlbuild/compiler/discovery/__init__.py | 1 - .../compiler/discovery/_helpers/__init__.py | 1 - .../compiler/fingerprints/__init__.py | 1 - src/sqlbuild/compiler/frontier/__init__.py | 1 - src/sqlbuild/compiler/graph/__init__.py | 1 - src/sqlbuild/compiler/lineage/__init__.py | 1 - .../compiler/macro_bridge/__init__.py | 1 - .../macro_bridge/_helpers/__init__.py | 1 - .../compiler/macro_bridge/classes/__init__.py | 1 - src/sqlbuild/compiler/migrations/__init__.py | 1 - src/sqlbuild/compiler/pipeline/__init__.py | 1 - .../compiler/pipeline/_helpers/__init__.py | 1 - src/sqlbuild/compiler/planner/__init__.py | 1 - .../compiler/python_nodes/__init__.py | 1 - .../python_nodes/_helpers/__init__.py | 1 - .../compiler/python_nodes/main/__init__.py | 1 - src/sqlbuild/compiler/refactoring/__init__.py | 1 - src/sqlbuild/compiler/references/__init__.py | 1 - .../compiler/resource_names/__init__.py | 1 - .../compiler/source_freshness/__init__.py | 1 - .../compiler/sql_analysis/__init__.py | 1 - .../sql_analysis/_helpers/__init__.py | 1 - .../compiler/sql_analysis/main/__init__.py | 1 - .../compiler/sql_test_glue/__init__.py | 1 - src/sqlbuild/cursor_algebra/__init__.py | 1 - src/sqlbuild/diagnostics/__init__.py | 1 - src/sqlbuild/errors/__init__.py | 1 - src/sqlbuild/errors/contracts/__init__.py | 1 - src/sqlbuild/integrations/__init__.py | 1 - src/sqlbuild/integrations/dbt/__init__.py | 1 - .../integrations/dbt/_helpers/__init__.py | 1 - .../dbt/_helpers/pipeline/__init__.py | 0 .../integrations/dbt/main/__init__.py | 1 - .../dbt/main/pipeline/__init__.py | 0 src/sqlbuild/lint/__init__.py | 1 - src/sqlbuild/presentation/__init__.py | 1 - src/sqlbuild/rule_engine/__init__.py | 1 - src/sqlbuild/rule_engine/_helpers/__init__.py | 1 - src/sqlbuild/rule_engine/classes/__init__.py | 1 - src/sqlbuild/rule_engine/main/__init__.py | 1 - src/sqlbuild/runtime/__init__.py | 1 - src/sqlbuild/runtime/contracts/__init__.py | 1 - src/sqlbuild/spec/__init__.py | 1 - src/sqlbuild/spec/contracts/__init__.py | 1 - .../main/test_startup_import_footprint.py | 1 - 66 files changed, 51 insertions(+), 58 deletions(-) delete mode 100644 src/sqlbuild/cli/commands/classes/__init__.py delete mode 100644 src/sqlbuild/cli/commands/main/dbt/__init__.py delete mode 100644 src/sqlbuild/cli/commands/main/inspection/__init__.py delete mode 100644 src/sqlbuild/cli/commands/main/project/__init__.py delete mode 100644 src/sqlbuild/cli/commands/main/state/__init__.py delete mode 100644 src/sqlbuild/cli/commands/main/workspace/__init__.py delete mode 100644 src/sqlbuild/compiler/analysis_session/__init__.py delete mode 100644 src/sqlbuild/compiler/contract_adoption/__init__.py delete mode 100644 src/sqlbuild/compiler/contracts/__init__.py delete mode 100644 src/sqlbuild/compiler/graph/__init__.py delete mode 100644 src/sqlbuild/compiler/refactoring/__init__.py delete mode 100644 src/sqlbuild/compiler/resource_names/__init__.py delete mode 100644 src/sqlbuild/compiler/sql_test_glue/__init__.py delete mode 100644 src/sqlbuild/integrations/dbt/_helpers/pipeline/__init__.py delete mode 100644 src/sqlbuild/integrations/dbt/main/__init__.py delete mode 100644 src/sqlbuild/integrations/dbt/main/pipeline/__init__.py delete mode 100644 src/sqlbuild/lint/__init__.py delete mode 100644 src/sqlbuild/presentation/__init__.py diff --git a/python-boundary.toml b/python-boundary.toml index c8fe9fbbee..ea9c5d9e4f 100644 --- a/python-boundary.toml +++ b/python-boundary.toml @@ -2,7 +2,7 @@ # migration work that moves to Rust. `remaining_lines` must equal the current count of that work: # a PR that moves code lowers it, and adding movable Python fails the check. # Native failures must surface as errors; `handled_native_errors` lists the reviewed exceptions. -remaining_lines = 287 +remaining_lines = 0 [[stays]] path = "executor" @@ -803,3 +803,53 @@ note = "Shape rule: data constants only (INTEGRATION_RESULT_PATH_ENV), no logic; path = "compiler/compile/types.py" reason = "executor" note = "Shape rule: typing protocols, enums and aliases only (CompiledProjectView, CompiledModelView, CompiledResourceType, AuditInput, ...), no logic; consumed by executor/janitor/_helpers/plan.py, executor/pipeline/_helpers/testing.py, executor/scenario/_helpers/local/execution.py and project audit factories that annotate AuditInput." + +[[stays]] +path = "rule_engine/types.py" +reason = "public-api" +note = "Shape rule: data protocol and aliases only (RuleContext, RuleCheck, RuleOptionValue, FactKey, CustomRulePlan), no logic; re-exported by rules/__init__.py and consumed by rule_engine/main/define.py, rule_engine/_helpers/host/custom_host.py, rule_engine/_helpers/engine/fact_replay.py and 4 more." + +[[stays]] +path = "rule_engine/constants.py" +reason = "user-code" +note = "Shape rule: data constants only (CUSTOM_HOST_PROTOCOL_VERSION, CUSTOM_HOST_RUNTIME_VERSION, CUSTOM_RULE_IMPORT_ROOTS, FACT_PROJECT_MODELS, ...), no logic; consumed by rule_engine/_helpers/host/custom_host.py, rule_engine/_helpers/host/tracked_facts.py, rule_engine/_helpers/engine/fact_replay.py and 10 more." + +[[stays]] +path = "compiler/discovery/constants.py" +reason = "executor" +note = "Shape rule: data constants only (PROJECT_CONFIG_FILENAME, SETTINGS_SECTION, TABLE_PROMOTION_MODE_SETTING_KEY, PYTHON_NODE_ROOT, ...), no logic; consumed by executor/run/main/_execute.py, compiler/discovery/_helpers/filesystem/core.py." + +[[stays]] +path = "compiler/compile/constants.py" +reason = "executor" +note = "Shape rule: data constants only (MIGRATE_FROM_CONFIG_KEY), no logic; consumed by executor/janitor/_helpers/plan.py." + +[[stays]] +path = "compiler/compile/exceptions.py" +reason = "user-code" +note = "Shape rule: data exceptions only (MacroDeclarationLookupError, a CompileInputError that is also a KeyError, which a native error class with one parent cannot express), no logic; consumed by compiler/compile/_helpers/render/user_macros.py." + +[[stays]] +path = "cursor_algebra/constants.py" +reason = "executor" +note = "Shape rule: data constants only (GRAIN_ORDER), no logic; consumed by executor/run/_helpers/validation/cursor_bounds.py." + +[[stays]] +path = "compiler/sql_analysis/constants.py" +reason = "public-api" +note = "Shape rule: data constants only (POLYGLOT_ANALYZE_QUERY_ENTRY_POINT, POLYGLOT_COMPLEXITY_GUARD_OPTION, POLYGLOT_MAX_FUNCTION_CALL_DEPTH, ...), no logic; consumed by compiler/sql_analysis/_helpers/polyglot.py." + +[[stays]] +path = "compiler/planner/constants.py" +reason = "executor" +note = "Shape rule: data constants only (MICROBATCH_START_SENTINEL, MICROBATCH_END_SENTINEL, MIGRATION_FINGERPRINT_METADATA_KEY, SNAPSHOT_DEFAULT_VALID_FROM_COLUMN, ...), no logic; consumed by executor/build/_helpers/scheduler.py, executor/run/_helpers/validation/cursor_bounds.py, executor/run/_helpers/reuse/fingerprint_metadata.py and 1 more." + +[[stays]] +path = "compiler/frontier/constants.py" +reason = "user-code" +note = "Shape rule: data constants only (STAGE_CAPTURE_DIR_ENV_VAR, COMPILE_REUSE_DISABLE_ENV_VAR), no logic; consumed by compiler/macro_bridge/constants.py." + +[[stays]] +path = "cost/constants.py" +reason = "adapter" +note = "Shape rule: data constants only (SQLBUILD_QUERY_TAG_APP), no logic; consumed by adapters/snowflake/classes/snowflake_cursor.py." diff --git a/src/sqlbuild/__init__.py b/src/sqlbuild/__init__.py index d0c8b749a6..e69de29bb2 100644 --- a/src/sqlbuild/__init__.py +++ b/src/sqlbuild/__init__.py @@ -1 +0,0 @@ -"""SQLBuild package.""" diff --git a/src/sqlbuild/adapter/__init__.py b/src/sqlbuild/adapter/__init__.py index cf225709ad..e69de29bb2 100644 --- a/src/sqlbuild/adapter/__init__.py +++ b/src/sqlbuild/adapter/__init__.py @@ -1 +0,0 @@ -"""Adapter interface and base implementation.""" diff --git a/src/sqlbuild/cli/__init__.py b/src/sqlbuild/cli/__init__.py index a3ba1f69ba..e69de29bb2 100644 --- a/src/sqlbuild/cli/__init__.py +++ b/src/sqlbuild/cli/__init__.py @@ -1 +0,0 @@ -"""CLI package.""" diff --git a/src/sqlbuild/cli/commands/_helpers/entry/__init__.py b/src/sqlbuild/cli/commands/_helpers/entry/__init__.py index 053344bc45..e69de29bb2 100644 --- a/src/sqlbuild/cli/commands/_helpers/entry/__init__.py +++ b/src/sqlbuild/cli/commands/_helpers/entry/__init__.py @@ -1 +0,0 @@ -"""CLI entry package.""" diff --git a/src/sqlbuild/cli/commands/classes/__init__.py b/src/sqlbuild/cli/commands/classes/__init__.py deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/src/sqlbuild/cli/commands/main/__init__.py b/src/sqlbuild/cli/commands/main/__init__.py index 37dfe1d1cb..e69de29bb2 100644 --- a/src/sqlbuild/cli/commands/main/__init__.py +++ b/src/sqlbuild/cli/commands/main/__init__.py @@ -1 +0,0 @@ -"""Main CLI command package.""" diff --git a/src/sqlbuild/cli/commands/main/dbt/__init__.py b/src/sqlbuild/cli/commands/main/dbt/__init__.py deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/src/sqlbuild/cli/commands/main/inspection/__init__.py b/src/sqlbuild/cli/commands/main/inspection/__init__.py deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/src/sqlbuild/cli/commands/main/project/__init__.py b/src/sqlbuild/cli/commands/main/project/__init__.py deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/src/sqlbuild/cli/commands/main/state/__init__.py b/src/sqlbuild/cli/commands/main/state/__init__.py deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/src/sqlbuild/cli/commands/main/workspace/__init__.py b/src/sqlbuild/cli/commands/main/workspace/__init__.py deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/src/sqlbuild/cli/compile_reuse/__init__.py b/src/sqlbuild/cli/compile_reuse/__init__.py index e7ebc10ef2..e69de29bb2 100644 --- a/src/sqlbuild/cli/compile_reuse/__init__.py +++ b/src/sqlbuild/cli/compile_reuse/__init__.py @@ -1 +0,0 @@ -"""Whole-project compile reuse when no compile input changed.""" diff --git a/src/sqlbuild/cli/output/__init__.py b/src/sqlbuild/cli/output/__init__.py index c17e7f4325..e69de29bb2 100644 --- a/src/sqlbuild/cli/output/__init__.py +++ b/src/sqlbuild/cli/output/__init__.py @@ -1 +0,0 @@ -"""CLI output capabilities.""" diff --git a/src/sqlbuild/compiler/__init__.py b/src/sqlbuild/compiler/__init__.py index d687f690a0..e69de29bb2 100644 --- a/src/sqlbuild/compiler/__init__.py +++ b/src/sqlbuild/compiler/__init__.py @@ -1 +0,0 @@ -"""Compiler package.""" diff --git a/src/sqlbuild/compiler/analysis_session/__init__.py b/src/sqlbuild/compiler/analysis_session/__init__.py deleted file mode 100644 index e973b2dc08..0000000000 --- a/src/sqlbuild/compiler/analysis_session/__init__.py +++ /dev/null @@ -1 +0,0 @@ -"""The native model analysis session.""" diff --git a/src/sqlbuild/compiler/auditing/__init__.py b/src/sqlbuild/compiler/auditing/__init__.py index 3e85debfe7..e69de29bb2 100644 --- a/src/sqlbuild/compiler/auditing/__init__.py +++ b/src/sqlbuild/compiler/auditing/__init__.py @@ -1 +0,0 @@ -"""Compiler auditing package.""" diff --git a/src/sqlbuild/compiler/compile/__init__.py b/src/sqlbuild/compiler/compile/__init__.py index 033199b706..e69de29bb2 100644 --- a/src/sqlbuild/compiler/compile/__init__.py +++ b/src/sqlbuild/compiler/compile/__init__.py @@ -1 +0,0 @@ -"""Compile package.""" diff --git a/src/sqlbuild/compiler/compile/_helpers/__init__.py b/src/sqlbuild/compiler/compile/_helpers/__init__.py index bac4869b96..e69de29bb2 100644 --- a/src/sqlbuild/compiler/compile/_helpers/__init__.py +++ b/src/sqlbuild/compiler/compile/_helpers/__init__.py @@ -1 +0,0 @@ -"""Helper utilities for compile input attachment.""" diff --git a/src/sqlbuild/compiler/contract_adoption/__init__.py b/src/sqlbuild/compiler/contract_adoption/__init__.py deleted file mode 100644 index 4bbe706e8a..0000000000 --- a/src/sqlbuild/compiler/contract_adoption/__init__.py +++ /dev/null @@ -1 +0,0 @@ -"""Physical contract comparison and repository adoption.""" diff --git a/src/sqlbuild/compiler/contracts/__init__.py b/src/sqlbuild/compiler/contracts/__init__.py deleted file mode 100644 index 9a4ad64d87..0000000000 --- a/src/sqlbuild/compiler/contracts/__init__.py +++ /dev/null @@ -1 +0,0 @@ -"""Model contract validation.""" diff --git a/src/sqlbuild/compiler/discovery/__init__.py b/src/sqlbuild/compiler/discovery/__init__.py index 3634fb2678..e69de29bb2 100644 --- a/src/sqlbuild/compiler/discovery/__init__.py +++ b/src/sqlbuild/compiler/discovery/__init__.py @@ -1 +0,0 @@ -"""Discovery package.""" diff --git a/src/sqlbuild/compiler/discovery/_helpers/__init__.py b/src/sqlbuild/compiler/discovery/_helpers/__init__.py index 84c54a2f0c..e69de29bb2 100644 --- a/src/sqlbuild/compiler/discovery/_helpers/__init__.py +++ b/src/sqlbuild/compiler/discovery/_helpers/__init__.py @@ -1 +0,0 @@ -"""Discovery helpers package.""" diff --git a/src/sqlbuild/compiler/fingerprints/__init__.py b/src/sqlbuild/compiler/fingerprints/__init__.py index a3e4551e45..e69de29bb2 100644 --- a/src/sqlbuild/compiler/fingerprints/__init__.py +++ b/src/sqlbuild/compiler/fingerprints/__init__.py @@ -1 +0,0 @@ -"""Compiler fingerprints package.""" diff --git a/src/sqlbuild/compiler/frontier/__init__.py b/src/sqlbuild/compiler/frontier/__init__.py index 6db46f9b53..e69de29bb2 100644 --- a/src/sqlbuild/compiler/frontier/__init__.py +++ b/src/sqlbuild/compiler/frontier/__init__.py @@ -1 +0,0 @@ -"""The native compiler frontier seam.""" diff --git a/src/sqlbuild/compiler/graph/__init__.py b/src/sqlbuild/compiler/graph/__init__.py deleted file mode 100644 index c4dd642de4..0000000000 --- a/src/sqlbuild/compiler/graph/__init__.py +++ /dev/null @@ -1 +0,0 @@ -"""Compiler graph algorithms.""" diff --git a/src/sqlbuild/compiler/lineage/__init__.py b/src/sqlbuild/compiler/lineage/__init__.py index ba0f8bbfd4..e69de29bb2 100644 --- a/src/sqlbuild/compiler/lineage/__init__.py +++ b/src/sqlbuild/compiler/lineage/__init__.py @@ -1 +0,0 @@ -"""Column lineage semantic analysis.""" diff --git a/src/sqlbuild/compiler/macro_bridge/__init__.py b/src/sqlbuild/compiler/macro_bridge/__init__.py index 73c3a31a12..e69de29bb2 100644 --- a/src/sqlbuild/compiler/macro_bridge/__init__.py +++ b/src/sqlbuild/compiler/macro_bridge/__init__.py @@ -1 +0,0 @@ -"""Native macro bridge: call-site scans, the in-compile call memo and splicing.""" diff --git a/src/sqlbuild/compiler/macro_bridge/_helpers/__init__.py b/src/sqlbuild/compiler/macro_bridge/_helpers/__init__.py index a88eb85027..e69de29bb2 100644 --- a/src/sqlbuild/compiler/macro_bridge/_helpers/__init__.py +++ b/src/sqlbuild/compiler/macro_bridge/_helpers/__init__.py @@ -1 +0,0 @@ -"""Macro call store keys and the environment they are valid in.""" diff --git a/src/sqlbuild/compiler/macro_bridge/classes/__init__.py b/src/sqlbuild/compiler/macro_bridge/classes/__init__.py index 2e8ae1fd31..e69de29bb2 100644 --- a/src/sqlbuild/compiler/macro_bridge/classes/__init__.py +++ b/src/sqlbuild/compiler/macro_bridge/classes/__init__.py @@ -1 +0,0 @@ -"""Macro bridge runtime classes.""" diff --git a/src/sqlbuild/compiler/migrations/__init__.py b/src/sqlbuild/compiler/migrations/__init__.py index eaf52e580e..e69de29bb2 100644 --- a/src/sqlbuild/compiler/migrations/__init__.py +++ b/src/sqlbuild/compiler/migrations/__init__.py @@ -1 +0,0 @@ -"""Direct-mode model migration state package.""" diff --git a/src/sqlbuild/compiler/pipeline/__init__.py b/src/sqlbuild/compiler/pipeline/__init__.py index 235d3cea5e..e69de29bb2 100644 --- a/src/sqlbuild/compiler/pipeline/__init__.py +++ b/src/sqlbuild/compiler/pipeline/__init__.py @@ -1 +0,0 @@ -"""Compiler orchestration pipeline package.""" diff --git a/src/sqlbuild/compiler/pipeline/_helpers/__init__.py b/src/sqlbuild/compiler/pipeline/_helpers/__init__.py index e04a1d8f1a..e69de29bb2 100644 --- a/src/sqlbuild/compiler/pipeline/_helpers/__init__.py +++ b/src/sqlbuild/compiler/pipeline/_helpers/__init__.py @@ -1 +0,0 @@ -"""Pipeline helpers.""" diff --git a/src/sqlbuild/compiler/planner/__init__.py b/src/sqlbuild/compiler/planner/__init__.py index 8d15113566..e69de29bb2 100644 --- a/src/sqlbuild/compiler/planner/__init__.py +++ b/src/sqlbuild/compiler/planner/__init__.py @@ -1 +0,0 @@ -"""Planner package.""" diff --git a/src/sqlbuild/compiler/python_nodes/__init__.py b/src/sqlbuild/compiler/python_nodes/__init__.py index 8835368d5d..e69de29bb2 100644 --- a/src/sqlbuild/compiler/python_nodes/__init__.py +++ b/src/sqlbuild/compiler/python_nodes/__init__.py @@ -1 +0,0 @@ -"""Internal Python-node compiler models and helpers.""" diff --git a/src/sqlbuild/compiler/python_nodes/_helpers/__init__.py b/src/sqlbuild/compiler/python_nodes/_helpers/__init__.py index 69c51bd3d1..e69de29bb2 100644 --- a/src/sqlbuild/compiler/python_nodes/_helpers/__init__.py +++ b/src/sqlbuild/compiler/python_nodes/_helpers/__init__.py @@ -1 +0,0 @@ -"""Internal Python-node helper functions.""" diff --git a/src/sqlbuild/compiler/python_nodes/main/__init__.py b/src/sqlbuild/compiler/python_nodes/main/__init__.py index 65271aefda..e69de29bb2 100644 --- a/src/sqlbuild/compiler/python_nodes/main/__init__.py +++ b/src/sqlbuild/compiler/python_nodes/main/__init__.py @@ -1 +0,0 @@ -"""Public entrypoints for compiler Python-node helpers.""" diff --git a/src/sqlbuild/compiler/refactoring/__init__.py b/src/sqlbuild/compiler/refactoring/__init__.py deleted file mode 100644 index c2d3b86586..0000000000 --- a/src/sqlbuild/compiler/refactoring/__init__.py +++ /dev/null @@ -1 +0,0 @@ -"""Compiler-driven project refactorings: model renames, moves, and column renames.""" diff --git a/src/sqlbuild/compiler/references/__init__.py b/src/sqlbuild/compiler/references/__init__.py index 86fdc55261..e69de29bb2 100644 --- a/src/sqlbuild/compiler/references/__init__.py +++ b/src/sqlbuild/compiler/references/__init__.py @@ -1 +0,0 @@ -"""Compiler-owned SQL reference behavior.""" diff --git a/src/sqlbuild/compiler/resource_names/__init__.py b/src/sqlbuild/compiler/resource_names/__init__.py deleted file mode 100644 index 2dc62df346..0000000000 --- a/src/sqlbuild/compiler/resource_names/__init__.py +++ /dev/null @@ -1 +0,0 @@ -"""Compiler-owned resource name contracts.""" diff --git a/src/sqlbuild/compiler/source_freshness/__init__.py b/src/sqlbuild/compiler/source_freshness/__init__.py index f8e4d9e230..e69de29bb2 100644 --- a/src/sqlbuild/compiler/source_freshness/__init__.py +++ b/src/sqlbuild/compiler/source_freshness/__init__.py @@ -1 +0,0 @@ -"""Direct source freshness state package.""" diff --git a/src/sqlbuild/compiler/sql_analysis/__init__.py b/src/sqlbuild/compiler/sql_analysis/__init__.py index 95940b4c9d..e69de29bb2 100644 --- a/src/sqlbuild/compiler/sql_analysis/__init__.py +++ b/src/sqlbuild/compiler/sql_analysis/__init__.py @@ -1 +0,0 @@ -"""Compiler-owned SQL analysis behavior.""" diff --git a/src/sqlbuild/compiler/sql_analysis/_helpers/__init__.py b/src/sqlbuild/compiler/sql_analysis/_helpers/__init__.py index edf7bcd1ac..e69de29bb2 100644 --- a/src/sqlbuild/compiler/sql_analysis/_helpers/__init__.py +++ b/src/sqlbuild/compiler/sql_analysis/_helpers/__init__.py @@ -1 +0,0 @@ -"""Internal SQL analysis helpers.""" diff --git a/src/sqlbuild/compiler/sql_analysis/main/__init__.py b/src/sqlbuild/compiler/sql_analysis/main/__init__.py index d33bc998ad..e69de29bb2 100644 --- a/src/sqlbuild/compiler/sql_analysis/main/__init__.py +++ b/src/sqlbuild/compiler/sql_analysis/main/__init__.py @@ -1 +0,0 @@ -"""Public SQL analysis entries.""" diff --git a/src/sqlbuild/compiler/sql_test_glue/__init__.py b/src/sqlbuild/compiler/sql_test_glue/__init__.py deleted file mode 100644 index e265190973..0000000000 --- a/src/sqlbuild/compiler/sql_test_glue/__init__.py +++ /dev/null @@ -1 +0,0 @@ -"""Native SQL test planning glue.""" diff --git a/src/sqlbuild/cursor_algebra/__init__.py b/src/sqlbuild/cursor_algebra/__init__.py index 0228f3f65d..e69de29bb2 100644 --- a/src/sqlbuild/cursor_algebra/__init__.py +++ b/src/sqlbuild/cursor_algebra/__init__.py @@ -1 +0,0 @@ -"""Naive timestamps are presumed UTC; aware timestamps are normalized to UTC at ingestion; dates have no timezone semantics.""" # noqa: E501 diff --git a/src/sqlbuild/diagnostics/__init__.py b/src/sqlbuild/diagnostics/__init__.py index 6f268ee63e..e69de29bb2 100644 --- a/src/sqlbuild/diagnostics/__init__.py +++ b/src/sqlbuild/diagnostics/__init__.py @@ -1 +0,0 @@ -"""Diagnostics logging support.""" diff --git a/src/sqlbuild/errors/__init__.py b/src/sqlbuild/errors/__init__.py index b7979eec1d..e69de29bb2 100644 --- a/src/sqlbuild/errors/__init__.py +++ b/src/sqlbuild/errors/__init__.py @@ -1 +0,0 @@ -"""Cross-domain error contracts.""" diff --git a/src/sqlbuild/errors/contracts/__init__.py b/src/sqlbuild/errors/contracts/__init__.py index 406e900146..e69de29bb2 100644 --- a/src/sqlbuild/errors/contracts/__init__.py +++ b/src/sqlbuild/errors/contracts/__init__.py @@ -1 +0,0 @@ -"""Cross-domain expected-error contracts.""" diff --git a/src/sqlbuild/integrations/__init__.py b/src/sqlbuild/integrations/__init__.py index a0dbf8a6fb..e69de29bb2 100644 --- a/src/sqlbuild/integrations/__init__.py +++ b/src/sqlbuild/integrations/__init__.py @@ -1 +0,0 @@ -"""Integration package.""" diff --git a/src/sqlbuild/integrations/dbt/__init__.py b/src/sqlbuild/integrations/dbt/__init__.py index 3767f64770..e69de29bb2 100644 --- a/src/sqlbuild/integrations/dbt/__init__.py +++ b/src/sqlbuild/integrations/dbt/__init__.py @@ -1 +0,0 @@ -"""dbt interop helpers.""" diff --git a/src/sqlbuild/integrations/dbt/_helpers/__init__.py b/src/sqlbuild/integrations/dbt/_helpers/__init__.py index 7195ee3e4d..e69de29bb2 100644 --- a/src/sqlbuild/integrations/dbt/_helpers/__init__.py +++ b/src/sqlbuild/integrations/dbt/_helpers/__init__.py @@ -1 +0,0 @@ -"""dbt integration helper modules.""" diff --git a/src/sqlbuild/integrations/dbt/_helpers/pipeline/__init__.py b/src/sqlbuild/integrations/dbt/_helpers/pipeline/__init__.py deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/src/sqlbuild/integrations/dbt/main/__init__.py b/src/sqlbuild/integrations/dbt/main/__init__.py deleted file mode 100644 index ea321f6ba0..0000000000 --- a/src/sqlbuild/integrations/dbt/main/__init__.py +++ /dev/null @@ -1 +0,0 @@ -"""Public dbt interop entrypoints.""" diff --git a/src/sqlbuild/integrations/dbt/main/pipeline/__init__.py b/src/sqlbuild/integrations/dbt/main/pipeline/__init__.py deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/src/sqlbuild/lint/__init__.py b/src/sqlbuild/lint/__init__.py deleted file mode 100644 index e06fdd147e..0000000000 --- a/src/sqlbuild/lint/__init__.py +++ /dev/null @@ -1 +0,0 @@ -"""Private SQL analysis and source-formatting implementation.""" diff --git a/src/sqlbuild/presentation/__init__.py b/src/sqlbuild/presentation/__init__.py deleted file mode 100644 index eed713edc9..0000000000 --- a/src/sqlbuild/presentation/__init__.py +++ /dev/null @@ -1 +0,0 @@ -"""Human and terminal presentation domain.""" diff --git a/src/sqlbuild/rule_engine/__init__.py b/src/sqlbuild/rule_engine/__init__.py index 5309b45486..e69de29bb2 100644 --- a/src/sqlbuild/rule_engine/__init__.py +++ b/src/sqlbuild/rule_engine/__init__.py @@ -1 +0,0 @@ -"""SQLBuild rules engine.""" diff --git a/src/sqlbuild/rule_engine/_helpers/__init__.py b/src/sqlbuild/rule_engine/_helpers/__init__.py index 8c45c20f48..e69de29bb2 100644 --- a/src/sqlbuild/rule_engine/_helpers/__init__.py +++ b/src/sqlbuild/rule_engine/_helpers/__init__.py @@ -1 +0,0 @@ -"""Internal rules helpers.""" diff --git a/src/sqlbuild/rule_engine/classes/__init__.py b/src/sqlbuild/rule_engine/classes/__init__.py index 8f088725ac..e69de29bb2 100644 --- a/src/sqlbuild/rule_engine/classes/__init__.py +++ b/src/sqlbuild/rule_engine/classes/__init__.py @@ -1 +0,0 @@ -"""Rules runtime classes.""" diff --git a/src/sqlbuild/rule_engine/main/__init__.py b/src/sqlbuild/rule_engine/main/__init__.py index c2922b48c1..e69de29bb2 100644 --- a/src/sqlbuild/rule_engine/main/__init__.py +++ b/src/sqlbuild/rule_engine/main/__init__.py @@ -1 +0,0 @@ -"""Rules public entrypoints.""" diff --git a/src/sqlbuild/runtime/__init__.py b/src/sqlbuild/runtime/__init__.py index 68003a5128..e69de29bb2 100644 --- a/src/sqlbuild/runtime/__init__.py +++ b/src/sqlbuild/runtime/__init__.py @@ -1 +0,0 @@ -"""Runtime integration contracts.""" diff --git a/src/sqlbuild/runtime/contracts/__init__.py b/src/sqlbuild/runtime/contracts/__init__.py index 66ed421c75..e69de29bb2 100644 --- a/src/sqlbuild/runtime/contracts/__init__.py +++ b/src/sqlbuild/runtime/contracts/__init__.py @@ -1 +0,0 @@ -"""Cross-domain runtime contracts.""" diff --git a/src/sqlbuild/spec/__init__.py b/src/sqlbuild/spec/__init__.py index 0c4a535a34..e69de29bb2 100644 --- a/src/sqlbuild/spec/__init__.py +++ b/src/sqlbuild/spec/__init__.py @@ -1 +0,0 @@ -"""Spec package.""" diff --git a/src/sqlbuild/spec/contracts/__init__.py b/src/sqlbuild/spec/contracts/__init__.py index 584e9830ad..e69de29bb2 100644 --- a/src/sqlbuild/spec/contracts/__init__.py +++ b/src/sqlbuild/spec/contracts/__init__.py @@ -1 +0,0 @@ -"""Specification contracts.""" diff --git a/tests/integration/src/sqlbuild/cli/commands/main/test_startup_import_footprint.py b/tests/integration/src/sqlbuild/cli/commands/main/test_startup_import_footprint.py index a748006d60..0b4b96a71d 100644 --- a/tests/integration/src/sqlbuild/cli/commands/main/test_startup_import_footprint.py +++ b/tests/integration/src/sqlbuild/cli/commands/main/test_startup_import_footprint.py @@ -41,7 +41,6 @@ "pydantic", "yaml", "sqlbuild.adapter.contract.classes.base_adapter", - "sqlbuild.lint", "sqlbuild.rule_engine", "sqlbuild.runtime.compute_logs", ) From 91a8c936d295bef54a7dfea6c7032869dabd760f Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Thu, 15 Oct 2026 18:52:29 +0100 Subject: [PATCH 09/48] chore: keep lint and presentation package markers and drop stale package exceptions --- fensu.toml | 35 ------------------- scripts/compiler_differential/constants.py | 1 - src/sqlbuild/lint/__init__.py | 0 src/sqlbuild/presentation/__init__.py | 0 .../main/test_startup_import_footprint.py | 1 + 5 files changed, 1 insertion(+), 36 deletions(-) create mode 100644 src/sqlbuild/lint/__init__.py create mode 100644 src/sqlbuild/presentation/__init__.py diff --git a/fensu.toml b/fensu.toml index d6e7a493a0..8f961394f3 100644 --- a/fensu.toml +++ b/fensu.toml @@ -73,11 +73,6 @@ rule = "FFR309" path = "src/sqlbuild/cost/constants.py" reason = "Native bindings own run-cost allocation, Snowflake cost collection, the statement ledger and the cost context; this package only declares the cost models, types and constants they use." -[[targets.python.rule_exceptions]] -rule = "FFR309" -path = "src/sqlbuild/compiler/contract_adoption/__init__.py" -reason = "Native bindings own contract comparison, edits and repository writes; this package only declares the adoption models, types and errors the native command builds and raises." - [[targets.python.rule_exceptions]] rule = "FFR309" path = "src/sqlbuild/compiler/pipeline/__init__.py" @@ -143,11 +138,6 @@ rule = "FFR309" path = "src/sqlbuild/adapters/sqlserver/__init__.py" reason = "The SQL Server backend intentionally owns behavior through its adapter class rather than function entrypoints." -[[targets.python.rule_exceptions]] -rule = "FFR309" -path = "src/sqlbuild/compiler/contracts/__init__.py" -reason = "Model contracts are evaluated natively; the package keeps only the validation result model its callers read." - [[targets.python.rule_exceptions]] rule = "FFR309" path = "src/sqlbuild/compiler/fingerprints/__init__.py" @@ -163,26 +153,6 @@ rule = "FFR309" path = "src/sqlbuild/compiler/references/__init__.py" reason = "SQL references are scanned natively; the package keeps the reference models, kinds and the reference-call patterns the Python planner still matches." -[[targets.python.rule_exceptions]] -rule = "FFR309" -path = "src/sqlbuild/compiler/resource_names/__init__.py" -reason = "Resource identities are validated natively; the package keeps the error class the native check raises." - -[[targets.python.rule_exceptions]] -rule = "FFR309" -path = "src/sqlbuild/compiler/refactoring/__init__.py" -reason = "Refactorings run natively; the package keeps the compiled-project input, manual-location model and error classes the native command reads and raises." - -[[targets.python.rule_exceptions]] -rule = "FFR309" -path = "src/sqlbuild/compiler/sql_test_glue/__init__.py" -reason = "SQL test assembly and planning run natively; the package keeps the request, result and row shapes the planner and compile stage read." - -[[targets.python.rule_exceptions]] -rule = "FFR309" -path = "src/sqlbuild/compiler/graph/__init__.py" -reason = "Graph algorithms and views run natively; the package keeps only the edge and view models their callers read." - [[targets.python.rule_exceptions]] rule = "FFR309" path = "src/sqlbuild/sql_values/exceptions.py" @@ -198,11 +168,6 @@ rule = "FFR309" path = "src/sqlbuild/compiler/lineage/__init__.py" reason = "Column lineage is built natively; the package keeps only the lineage models, enums and errors the native graph and its callers share." -[[targets.python.rule_exceptions]] -rule = "FFR309" -path = "src/sqlbuild/compiler/analysis_session/__init__.py" -reason = "Model analysis runs natively; the package keeps only the request and result models, row types and constants the native session and its callers share." - [[targets.python.rule_exceptions]] rule = "FFR601" path = "src/sqlbuild/adapter/contract/classes/base_adapter.py" diff --git a/scripts/compiler_differential/constants.py b/scripts/compiler_differential/constants.py index 3d97b3a040..bdfad436fa 100644 --- a/scripts/compiler_differential/constants.py +++ b/scripts/compiler_differential/constants.py @@ -661,7 +661,6 @@ def short_model_names(*, model: Model, ctx: RuleContext) -> list[Finding]: } ANALYSIS_CODE_SCAN_ROOTS: tuple[str, ...] = ( "compile/_helpers/diagnostics", - "contracts", "sql_analysis", ) ANALYSIS_NATIVE_CODE_SCAN_ROOTS: tuple[str, ...] = ( diff --git a/src/sqlbuild/lint/__init__.py b/src/sqlbuild/lint/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/src/sqlbuild/presentation/__init__.py b/src/sqlbuild/presentation/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/tests/integration/src/sqlbuild/cli/commands/main/test_startup_import_footprint.py b/tests/integration/src/sqlbuild/cli/commands/main/test_startup_import_footprint.py index 0b4b96a71d..a748006d60 100644 --- a/tests/integration/src/sqlbuild/cli/commands/main/test_startup_import_footprint.py +++ b/tests/integration/src/sqlbuild/cli/commands/main/test_startup_import_footprint.py @@ -41,6 +41,7 @@ "pydantic", "yaml", "sqlbuild.adapter.contract.classes.base_adapter", + "sqlbuild.lint", "sqlbuild.rule_engine", "sqlbuild.runtime.compute_logs", ) From 5251ffd3ee59832a9a92b0c8e3878de17c9d9c9f Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Thu, 15 Oct 2026 19:01:22 +0100 Subject: [PATCH 10/48] fix: keep the MacroContext import path project macros use --- python-boundary.toml | 5 +++++ src/sqlbuild/compiler/compile/models.py | 5 +++++ tests/e2e/fixtures/waffle_shop/models/marts/_macros/time.py | 2 +- .../integration/src/sqlbuild/compiler/pipeline/test_main.py | 2 +- 4 files changed, 12 insertions(+), 2 deletions(-) create mode 100644 src/sqlbuild/compiler/compile/models.py diff --git a/python-boundary.toml b/python-boundary.toml index ea9c5d9e4f..77e7505ee8 100644 --- a/python-boundary.toml +++ b/python-boundary.toml @@ -853,3 +853,8 @@ note = "Shape rule: data constants only (STAGE_CAPTURE_DIR_ENV_VAR, COMPILE_REUS path = "cost/constants.py" reason = "adapter" note = "Shape rule: data constants only (SQLBUILD_QUERY_TAG_APP), no logic; consumed by adapters/snowflake/classes/snowflake_cursor.py." + +[[stays]] +path = "compiler/compile/models.py" +reason = "public-api" +note = "Shape rule: re-export only (MacroContext, now hosted natively), no logic; project macro modules import it from this path to annotate their ctx parameter, as tests/e2e/fixtures/waffle_shop/models/marts/_macros/time.py does." diff --git a/src/sqlbuild/compiler/compile/models.py b/src/sqlbuild/compiler/compile/models.py new file mode 100644 index 0000000000..6bfcbc16d5 --- /dev/null +++ b/src/sqlbuild/compiler/compile/models.py @@ -0,0 +1,5 @@ +"""Macro context type that project macros import for their `ctx` annotation.""" + +from sqlbuild._native import MacroContext + +__all__ = ("MacroContext",) diff --git a/tests/e2e/fixtures/waffle_shop/models/marts/_macros/time.py b/tests/e2e/fixtures/waffle_shop/models/marts/_macros/time.py index 06ec0c3445..041c18185b 100644 --- a/tests/e2e/fixtures/waffle_shop/models/marts/_macros/time.py +++ b/tests/e2e/fixtures/waffle_shop/models/marts/_macros/time.py @@ -1,6 +1,6 @@ """Time expression macros.""" -from sqlbuild._native import MacroContext +from sqlbuild.compiler.compile.models import MacroContext def timestamp_trunc(ctx: MacroContext, grain: str, expression: str) -> str: diff --git a/tests/integration/src/sqlbuild/compiler/pipeline/test_main.py b/tests/integration/src/sqlbuild/compiler/pipeline/test_main.py index 61d8458840..b8e8be854b 100644 --- a/tests/integration/src/sqlbuild/compiler/pipeline/test_main.py +++ b/tests/integration/src/sqlbuild/compiler/pipeline/test_main.py @@ -313,7 +313,7 @@ def test_given_rule_finding_when_running_shared_compile_pipeline_then_planning_i 'ENUM (name order_status, members (ACTIVE "active", PAUSED "paused"));\n' ), "models/_macros/policy.py": ( - "from sqlbuild._native import MacroContext\n\n\n" + "from sqlbuild.compiler.compile.models import MacroContext\n\n\n" "def policy_columns(ctx: MacroContext) -> str:\n" " minimum = ctx.constants['minimum_quantity']\n" " region_count = len(ctx.constants['regions'])\n" From 54261744d97151c2f07f4be511bb744f3da608ba Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Thu, 15 Oct 2026 19:10:29 +0100 Subject: [PATCH 11/48] chore: record the MacroContext import path exception --- fensu.toml | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/fensu.toml b/fensu.toml index 8f961394f3..db5fceb35c 100644 --- a/fensu.toml +++ b/fensu.toml @@ -311,6 +311,16 @@ rule = "FFR403" path = "src/sqlbuild/rules/testing.py" reason = "This deliberate public shim keeps test-harness internals outside the stable authoring API." +[[targets.python.rule_exceptions]] +rule = "FFR403" +path = "src/sqlbuild/compiler/compile/models.py" +reason = "Project macro modules import MacroContext from this path for their ctx annotation; the class is hosted natively and this module keeps that import path working." + +[[targets.python.rule_exceptions]] +rule = "FFR001" +path = "src/sqlbuild/compiler/compile/models.py" +reason = "The only model here is the natively hosted MacroContext, re-exported for project macro imports." + [[targets.python.rule_exceptions]] rule = "FFS120" path = "src/sqlbuild/integrations/dagster/classes/sqlbuild_cli_resource.py" From 39d6320cea6fe769be6e09d77b7e5e1eff245880 Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Thu, 15 Oct 2026 20:19:25 +0100 Subject: [PATCH 12/48] fix: reconcile round-two native shapes with main's janitor, discovery and event-export changes --- .../_helpers/compile_driver/discovery.rs | 2 +- .../compile_driver/integration_loaders.rs | 4 +- .../_helpers/discovery/imported_modules.rs | 2 +- .../_helpers/event_exporting/constants.rs | 2 +- .../_helpers/janitor/janitor_old_names.rs | 2 +- python-boundary.toml | 11 +-- .../_helpers/corpus/emitted_codes.py | 17 +--- src/sqlbuild/_native.pyi | 87 +------------------ .../compile/_helpers/render/user_macros.py | 8 +- src/sqlbuild/compiler/compile/constants.py | 5 -- .../filesystem/command_output_sinks.py | 14 +-- .../_helpers/filesystem/project_modules.py | 3 +- .../discovery/main/runtime_extensions.py | 22 ++--- .../python_nodes/_helpers/user_inventory.py | 2 +- .../executor/migrations/_helpers/naming.py | 2 +- .../migrations/_helpers/old_name_views.py | 9 ++ .../old_name_views/_helpers/janitor.py | 7 -- .../executor/testing/main/_execute.py | 3 +- .../integrations/dlt/main/_loaders.py | 2 +- .../integrations/ingestr/main/_loaders.py | 1 + .../runtime/event_exporting/constants.py | 1 - .../runtime/event_exporting/models.py | 2 +- .../test_provider_annotation_tolerance.py | 7 +- .../compiler/analysis_session/helpers.py | 2 +- .../test_native_query_columns.py | 8 +- .../sqlbuild/compiler/attachments/helpers.py | 2 +- .../src/sqlbuild/compiler/compile/helpers.py | 4 +- .../compiled_project/test_native_audits.py | 2 +- .../test_native_compiled_project.py | 2 +- .../compiled_project/test_native_functions.py | 2 +- .../test_native_project_assembly.py | 3 +- .../src/sqlbuild/compiler/scopes/helpers.py | 2 +- .../diff/test_query_artifact_cleanup.py | 1 + .../rule_engine/host_payload/helpers.py | 2 +- .../_helpers/runtime/test_connection.py | 2 +- .../cli/commands/main/entry/_test_types.py | 2 +- .../cli/commands/main/entry/test_main.py | 2 +- .../_helpers/config/test_table_type.py | 2 +- .../compiler/compile/_helpers/helpers.py | 2 +- .../_helpers/render/test_cursor_intrinsics.py | 2 +- .../compile/_helpers/test_audit_factories.py | 2 +- .../_helpers/test_declaration_scopes.py | 2 +- .../compile/_helpers/test_declarations.py | 2 +- .../compile/_helpers/test_expansion_spans.py | 2 +- .../_helpers/test_microbatch_redesign.py | 2 +- .../_helpers/test_model_config_validation.py | 2 +- .../compile/_helpers/test_model_schemas.py | 2 +- .../_helpers/test_parameterized_sql_tests.py | 2 +- .../test_filesystem_audit_factories.py | 5 +- .../test_filesystem_command_output_sinks.py | 6 +- .../test_filesystem_event_exporters.py | 6 +- .../_helpers/test_filesystem_factories.py | 5 +- .../_helpers/test_filesystem_hooks.py | 7 -- .../_helpers/test_filesystem_loaders.py | 5 +- .../test_filesystem_materializations.py | 6 -- .../_helpers/test_filesystem_providers.py | 4 +- .../_helpers/test_filesystem_python_nodes.py | 6 -- .../test_filesystem_scoped_declarations.py | 4 - .../compiler/macro_bridge/main/helpers.py | 3 +- .../_helpers/sql_test_assembly/helpers.py | 2 +- .../sql_test_assembly/test_plan_test.py | 2 +- .../integrations/dlt/test_dlt_loaders.py | 4 +- .../integrations/ingestr/test_loaders.py | 4 +- .../event_exporting/classes/helpers.py | 6 +- .../classes/test_command_scope.py | 8 +- .../classes/test_dispatcher.py | 1 + .../runtime/event_exporting/test_policy.py | 2 +- 67 files changed, 111 insertions(+), 248 deletions(-) delete mode 100644 src/sqlbuild/compiler/compile/constants.py diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/discovery.rs b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/discovery.rs index 273d43ecf8..36d097f030 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/discovery.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/discovery.rs @@ -38,7 +38,7 @@ const NATIVE_MODULE: &str = "sqlbuild._native"; const SQL_ANALYSIS_SETTING_KEY: &str = "sql_analysis"; const FILESYSTEM_MODULE: &str = "sqlbuild._native"; const SINKS_MODULE: &str = "sqlbuild.compiler.discovery._helpers.filesystem.command_output_sinks"; -const DISCOVERY_MODELS_MODULE: &str = "sqlbuild.compiler.discovery.models"; +const DISCOVERY_MODELS_MODULE: &str = "sqlbuild._native"; /// Declaration file collections counted into the declaration-parse lifecycle, in field order. const DECLARATION_COLLECTIONS: [&str; 14] = [ diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/integration_loaders.rs b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/integration_loaders.rs index 22dcb11e42..1ac19d99e2 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/integration_loaders.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/integration_loaders.rs @@ -7,8 +7,8 @@ use sqlbuild_discovery::yaml_entries::main::integration_loader_name::integration use crate::bindings::_helpers::compile_driver::python_calls::{call_with, imported}; -const DISCOVERY_MODELS_MODULE: &str = "sqlbuild.compiler.discovery.models"; -const DISCOVERY_TYPES_MODULE: &str = "sqlbuild.compiler.discovery.types"; +const DISCOVERY_MODELS_MODULE: &str = "sqlbuild._native"; +const DISCOVERY_TYPES_MODULE: &str = "sqlbuild._native"; /// One integration: its kind constant and the runtime module building its loader functions. struct Integration { diff --git a/crates/sqlbuild-python/src/bindings/_helpers/discovery/imported_modules.rs b/crates/sqlbuild-python/src/bindings/_helpers/discovery/imported_modules.rs index 6f8dde6114..939a4613b6 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/discovery/imported_modules.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/discovery/imported_modules.rs @@ -10,7 +10,7 @@ use crate::bindings::_helpers::event_exporting::scope::current_scope; pub(crate) const NATIVE_MODULE: &str = "sqlbuild._native"; pub(crate) const SHIM_MODULE: &str = "sqlbuild.compiler.discovery._helpers.filesystem.project_modules"; -pub(crate) const MODELS_MODULE: &str = "sqlbuild.compiler.discovery.models"; +pub(crate) const MODELS_MODULE: &str = "sqlbuild._native"; pub(crate) const ERRORS_MODULE: &str = "sqlbuild._native"; /// One project directory under discovery. diff --git a/crates/sqlbuild-python/src/bindings/_helpers/event_exporting/constants.rs b/crates/sqlbuild-python/src/bindings/_helpers/event_exporting/constants.rs index b03687364e..05c527f2fc 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/event_exporting/constants.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/event_exporting/constants.rs @@ -1,7 +1,7 @@ //! Python modules, thread names and wait limits of native lifecycle-event export. pub(crate) const EXCEPTIONS_MODULE: &str = "sqlbuild.runtime.event_exporting.exceptions"; -pub(crate) const SEVERITY_MODULE: &str = "sqlbuild.spec.contracts.types"; +pub(crate) const SEVERITY_MODULE: &str = "sqlbuild._native"; pub(crate) const CATALOG_MODULE: &str = "sqlbuild.runtime.observability.constants"; pub(crate) const INVOKE_LIFECYCLE_MODULE: &str = "sqlbuild.runtime.event_exporting.main._invoke_lifecycle_event_sink"; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/janitor/janitor_old_names.rs b/crates/sqlbuild-python/src/bindings/_helpers/janitor/janitor_old_names.rs index b891784a0e..3017017b5d 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/janitor/janitor_old_names.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/janitor/janitor_old_names.rs @@ -15,7 +15,7 @@ use crate::bindings::_helpers::janitor::janitor_keys::{ Folded, Key, SchemaKey, optional, record, sorted_schemas, text, tuple, }; -const MIGRATION_TYPES: &str = "sqlbuild.compiler.migrations.types"; +const MIGRATION_TYPES: &str = "sqlbuild._native"; const IS_COMPATIBILITY_VIEW: (&str, &str) = ( "sqlbuild.executor.old_name_views._helpers.janitor", "is_compatibility_view", diff --git a/python-boundary.toml b/python-boundary.toml index 77e7505ee8..e272b29b1a 100644 --- a/python-boundary.toml +++ b/python-boundary.toml @@ -762,7 +762,7 @@ note = "Shape rule: data exceptions only (DbtProfileError), no logic; consumed b [[stays]] path = "compiler/migrations/constants.py" reason = "executor" -note = "Shape rule: data constants only (MIGRATION_TABLE_NAME, MIGRATION_WRITE_ATTEMPTS, COLUMN_MIGRATION_TABLE_NAME, OLD_NAME_VIEW_TABLE_NAME), no logic; consumed by adapters/sqlserver/classes/sqlserver_adapter.py, executor/janitor/constants.py, executor/migrations/_helpers/columns.py and 4 more." +note = "Shape rule: data constants only (MIGRATION_TABLE_NAME, MIGRATION_WRITE_ATTEMPTS, COLUMN_MIGRATION_TABLE_NAME, OLD_NAME_VIEW_TABLE_NAME), no logic; consumed by adapters/sqlserver/classes/sqlserver_adapter.py, executor/migrations/_helpers/columns.py, executor/migrations/_helpers/old_name_facts.py and 2 more." [[stays]] path = "diagnostics/constants.py" @@ -802,7 +802,7 @@ note = "Shape rule: data constants only (INTEGRATION_RESULT_PATH_ENV), no logic; [[stays]] path = "compiler/compile/types.py" reason = "executor" -note = "Shape rule: typing protocols, enums and aliases only (CompiledProjectView, CompiledModelView, CompiledResourceType, AuditInput, ...), no logic; consumed by executor/janitor/_helpers/plan.py, executor/pipeline/_helpers/testing.py, executor/scenario/_helpers/local/execution.py and project audit factories that annotate AuditInput." +note = "Shape rule: typing protocols, enums and aliases only (CompiledProjectView, CompiledModelView, CompiledResourceType, AuditInput, ...), no logic; consumed by executor/pipeline/_helpers/testing.py, executor/scenario/_helpers/local/execution.py and project audit factories that annotate AuditInput." [[stays]] path = "rule_engine/types.py" @@ -817,12 +817,7 @@ note = "Shape rule: data constants only (CUSTOM_HOST_PROTOCOL_VERSION, CUSTOM_HO [[stays]] path = "compiler/discovery/constants.py" reason = "executor" -note = "Shape rule: data constants only (PROJECT_CONFIG_FILENAME, SETTINGS_SECTION, TABLE_PROMOTION_MODE_SETTING_KEY, PYTHON_NODE_ROOT, ...), no logic; consumed by executor/run/main/_execute.py, compiler/discovery/_helpers/filesystem/core.py." - -[[stays]] -path = "compiler/compile/constants.py" -reason = "executor" -note = "Shape rule: data constants only (MIGRATE_FROM_CONFIG_KEY), no logic; consumed by executor/janitor/_helpers/plan.py." +note = "Shape rule: data constants only (PROJECT_CONFIG_FILENAME, SETTINGS_SECTION, TABLE_PROMOTION_MODE_SETTING_KEY, PYTHON_NODE_ROOT, ...), no logic; consumed by executor/run/main/_execute.py, compiler/discovery/_helpers/filesystem/project_modules.py." [[stays]] path = "compiler/compile/exceptions.py" diff --git a/scripts/compiler_differential/_helpers/corpus/emitted_codes.py b/scripts/compiler_differential/_helpers/corpus/emitted_codes.py index c762ffda85..015c086250 100644 --- a/scripts/compiler_differential/_helpers/corpus/emitted_codes.py +++ b/scripts/compiler_differential/_helpers/corpus/emitted_codes.py @@ -39,13 +39,8 @@ FailureCase, ProjectRun, ) -from sqlbuild._native import ( - CompileInputError, - DiscoveryError, - ResourceIdentityError, - ScopeDiagnosticCode, -) -from sqlbuild.compiler.compile import constants as compile_constants +from sqlbuild._native import DiscoveryError, ResourceIdentityError, ScopeDiagnosticCode +from sqlbuild.compiler.compile.exceptions import CompileInputError _SOURCE_DIRECTORY: str = "source" _CODE_KEYWORD: str = "code" @@ -99,18 +94,10 @@ def discovery_error_codes() -> frozenset[str]: def render_error_codes() -> frozenset[str]: """Return every code render, scope, and attachment can report, from compiler constants.""" - compile_codes: set[str] = { - value - for name, value in vars(compile_constants).items() - if name.endswith("_CODE") - and isinstance(value, str) - and DIAGNOSTIC_CODE_PATTERN.fullmatch(value) - } return frozenset( { CompileInputError.code, BUILT_IN_AUDIT_SHADOW_CODE, - *compile_codes, *(code.value for code in ScopeDiagnosticCode), *RENDER_RAISED_CODES, } diff --git a/src/sqlbuild/_native.pyi b/src/sqlbuild/_native.pyi index 8d4c8834b6..ae9a723b5a 100644 --- a/src/sqlbuild/_native.pyi +++ b/src/sqlbuild/_native.pyi @@ -13,7 +13,7 @@ from enum import StrEnum from pathlib import Path, PurePath from threading import Event from types import TracebackType -from typing import Any, ClassVar, Literal, Protocol, Self, TextIO, TypedDict +from typing import Any, ClassVar, Literal, Protocol, Self, TextIO, TypedDict, overload from sqlbuild.adapter.contract.classes.base_adapter import ( BaseAdapter, @@ -64,27 +64,6 @@ from sqlbuild.compiler.compile.types import ( SqlTestMode, TypedSqlValueRenderer, ) -from sqlbuild.compiler.discovery.models import ( - ConstantDeclaration, - DiscoveredCheckFunction, - DiscoveredCommandOutputSink, - DiscoveredEventExporter, - DiscoveredEventExporterDeclaration, - DiscoveredHookFunction, - DiscoveredLoaderFunction, - DiscoveredMacroFile, - DiscoveredMaterializationFile, - DiscoveredProjectInputs, - DiscoveredProvider, - DiscoveredProviderUsage, - DiscoveredPythonNodeFunctions, - DiscoveredSourceFile, - DiscoveredSqlScenarioFile, - DiscoveredSqlTestFile, - EnumDeclaration, - SqlTestParameterDeclaration, -) -from sqlbuild.compiler.discovery.types import LoaderConnectionMode from sqlbuild.compiler.lineage.types import ( InferredNullability, ) @@ -94,20 +73,8 @@ from sqlbuild.compiler.references.models import ( ProjectRelationIndex, RelationName, ) -from sqlbuild.compiler.references.types import ( - ExternalSqlReferenceResolver, - HardCodedRelationOwnerKind, - SqlReferenceKind, -) -from sqlbuild.compiler.source_freshness.models import ( - SourceFreshnessRecord, -) -from sqlbuild.compiler.sql_analysis.models import ( - SqlLexicalSyntax, -) from sqlbuild.cost.models import CostResourceContext, RunCostSummary, StatementExecutionTelemetry from sqlbuild.cost.types import CostStatus -from sqlbuild.cursor_algebra.models import AlignedInterval, DateValue, Duration, TimestampValue from sqlbuild.cursor_algebra.types import Bound, BoundSentinel, CursorScalar from sqlbuild.diagnostics.models import ProcessResourceUsage from sqlbuild.executor.auditing.models import ( @@ -145,12 +112,9 @@ from sqlbuild.executor.testing.models import ( SqlTestExecutionResult, StepResult, ) -from sqlbuild.integrations.dbt.models import ( - ResolvedDbtProfileOutput, -) -from sqlbuild.integrations.dbt.types import DbtInvoker from sqlbuild.microbatches.models import MicrobatchInterval from sqlbuild.provider.classes.container import ProviderContainer +from sqlbuild.provider.classes.provider import Provider from sqlbuild.provider.classes.session import ProviderSession from sqlbuild.python_nodes.models import ( AuditCase, @@ -159,12 +123,7 @@ from sqlbuild.python_nodes.models import ( SqlResourceRef, ) from sqlbuild.python_nodes.types import PythonCheckSeverity -from sqlbuild.rule_engine.models import ( - Finding, - Rule, - RulesConfig, - RulesResult, -) +from sqlbuild.rule_engine.types import RuleCheck, RuleOptionValue from sqlbuild.runtime.compute_logs.models import ( CaptureByteCounts, CaptureInventory, @@ -198,46 +157,6 @@ from sqlbuild.runtime.output_capture.models import ( OutputCaptureContext, ) from sqlbuild.runtime.output_capture.types import CommandOutputStream -from sqlbuild.spec.contracts.models import ( - CostConfig, - DbtConfig, - DefaultsConfig, - ExecutionLimitsConfig, - FutureCursorsConfig, - LoaderDestinationParts, - LocalConfig, - LocalDbtConfig, - ProjectConfig, - ResolvedTableType, - ResolvedTimeTravelRetention, - ScenarioConfig, - SchemaAuditInstance, - SchemaColumn, - SchemaDynamicColumnFamily, - SchemaModelEntry, - SchemaSeedEntry, - SettingsConfig, - SnapshotsConfig, - SourceColumnEntry, - SourceEntry, - SourceFreshnessAgePolicy, - SourceLocation, - StartCursorsConfig, - TargetConfig, -) -from sqlbuild.spec.contracts.types import ( - EventExportSeverity, - MicrobatchLimitAction, - MissingMigrationOriginPolicy, - SourceFreshnessStrategy, - SourceFreshnessValueKind, - SourceWriteStrategy, - WarehouseGroup, -) -from sqlbuild.sql_values.models import SqlValue -from sqlbuild.sql_values.types import ( - StateSqlValueType, -) BUILD_IDENTITY: str diff --git a/src/sqlbuild/compiler/compile/_helpers/render/user_macros.py b/src/sqlbuild/compiler/compile/_helpers/render/user_macros.py index 6c84590d94..12893298c9 100644 --- a/src/sqlbuild/compiler/compile/_helpers/render/user_macros.py +++ b/src/sqlbuild/compiler/compile/_helpers/render/user_macros.py @@ -15,7 +15,6 @@ import sqlbuild._native as _native from sqlbuild._native import ( - CompileInputError, ConstantDeclaration, DeclarationIdentity, DeclarationKind, @@ -24,7 +23,6 @@ DiscoveredMacroFile, EnumDeclaration, LoadedMacro, - MacroContext, SqlValue, SqlValueKind, StaticMacroExport, @@ -32,7 +30,11 @@ StaticMacroInventory, validate_resource_identity, ) -from sqlbuild.compiler.compile.exceptions import MacroDeclarationLookupError +from sqlbuild.compiler.compile.exceptions import ( + CompileInputError, + MacroDeclarationLookupError, +) +from sqlbuild.compiler.compile.models import MacroContext _OBJECT_ENTRY_LENGTH: int = 2 _MACRO_MODULE_LOAD_LOCK: threading.RLock = threading.RLock() diff --git a/src/sqlbuild/compiler/compile/constants.py b/src/sqlbuild/compiler/compile/constants.py deleted file mode 100644 index cd8a7ce9ad..0000000000 --- a/src/sqlbuild/compiler/compile/constants.py +++ /dev/null @@ -1,5 +0,0 @@ -"""Stable constants for compile-time helpers.""" - -from __future__ import annotations - -MIGRATE_FROM_CONFIG_KEY: str = "migrate_from" diff --git a/src/sqlbuild/compiler/discovery/_helpers/filesystem/command_output_sinks.py b/src/sqlbuild/compiler/discovery/_helpers/filesystem/command_output_sinks.py index 73fb50bb7a..45f154939a 100644 --- a/src/sqlbuild/compiler/discovery/_helpers/filesystem/command_output_sinks.py +++ b/src/sqlbuild/compiler/discovery/_helpers/filesystem/command_output_sinks.py @@ -9,12 +9,7 @@ from typing import get_type_hints import sqlbuild._native as _native -from sqlbuild._native import EventExporterDiscoveryError -from sqlbuild.compiler.discovery._helpers.filesystem.project_modules import ( - load_sink_module, - provider_usages, -) -from sqlbuild.compiler.discovery.models import ( +from sqlbuild._native import ( DiscoveredCommandOutputSink, DiscoveredCommandOutputSinkDeclaration, DiscoveredEventExporter, @@ -22,10 +17,9 @@ DiscoveredProviderUsage, EventExporterDiscoveryError, ) -from sqlbuild.compiler.discovery._helpers.filesystem.core import ( - _load_sink_module, - _provider_by_name, - _provider_usages, +from sqlbuild.compiler.discovery._helpers.filesystem.project_modules import ( + load_sink_module, + provider_usages, ) from sqlbuild.runtime.event_exporting.main.get_command_output_sink_definition import ( get_command_output_sink_definition, diff --git a/src/sqlbuild/compiler/discovery/_helpers/filesystem/project_modules.py b/src/sqlbuild/compiler/discovery/_helpers/filesystem/project_modules.py index 4134e3df5b..76b7eca129 100644 --- a/src/sqlbuild/compiler/discovery/_helpers/filesystem/project_modules.py +++ b/src/sqlbuild/compiler/discovery/_helpers/filesystem/project_modules.py @@ -13,6 +13,8 @@ from typing import get_type_hints from sqlbuild._native import ( + DiscoveredProvider, + DiscoveredProviderUsage, EventExporterDiscoveryError, ProviderDiscoveryError, PythonNodeDiscoveryError, @@ -22,7 +24,6 @@ PYTHON_INIT_MODULE_STEM, PYTHON_NODE_ROOT, ) -from sqlbuild.compiler.discovery.models import DiscoveredProvider, DiscoveredProviderUsage from sqlbuild.runtime.event_exporting.constants import EVENT_EXPORTER_EVENT_PARAMETER_NAME from sqlbuild.runtime.observability.models import LifecycleEvent diff --git a/src/sqlbuild/compiler/discovery/main/runtime_extensions.py b/src/sqlbuild/compiler/discovery/main/runtime_extensions.py index 32b378c44a..8a41d0579b 100644 --- a/src/sqlbuild/compiler/discovery/main/runtime_extensions.py +++ b/src/sqlbuild/compiler/discovery/main/runtime_extensions.py @@ -5,28 +5,24 @@ import sqlbuild._native as _native from sqlbuild._native import ( - ProjectConfigError, - bind_event_exporter_declarations, - discover_event_exporter_declarations, - discover_provider_classes, -) -from sqlbuild.compiler.discovery._helpers.filesystem.command_output_sinks import ( - bind_command_output_sink_declarations, - discover_command_output_sink_declarations, -) -from sqlbuild.compiler.discovery.models import ( DiscoveredCommandOutputSink, DiscoveredCommandOutputSinkDeclaration, DiscoveredEventExporter, DiscoveredEventExporterDeclaration, DiscoveredProvider, DiscoveredRuntimeExtensions, -) -from sqlbuild.spec.contracts.models import ( + EventExportSeverity, LifecycleEventSinkFilterConfig, LifecycleEventSinksConfig, + ProjectConfigError, + bind_event_exporter_declarations, + discover_event_exporter_declarations, + discover_provider_classes, +) +from sqlbuild.compiler.discovery._helpers.filesystem.command_output_sinks import ( + bind_command_output_sink_declarations, + discover_command_output_sink_declarations, ) -from sqlbuild.spec.contracts.types import EventExportSeverity def discover_runtime_extensions(*, project_dir: Path) -> DiscoveredRuntimeExtensions: diff --git a/src/sqlbuild/compiler/python_nodes/_helpers/user_inventory.py b/src/sqlbuild/compiler/python_nodes/_helpers/user_inventory.py index 03f3fd1e3c..3e03021cf0 100644 --- a/src/sqlbuild/compiler/python_nodes/_helpers/user_inventory.py +++ b/src/sqlbuild/compiler/python_nodes/_helpers/user_inventory.py @@ -5,8 +5,8 @@ from collections.abc import Callable from pathlib import Path +from sqlbuild._native import PythonNodeIdentity from sqlbuild.compiler.python_nodes._helpers.identity import build_python_identity -from sqlbuild.compiler.python_nodes.models import PythonNodeIdentity from sqlbuild.python_nodes.main.read_asset_definition import read_asset_definition from sqlbuild.python_nodes.main.read_check_definition import read_check_definition from sqlbuild.python_nodes.main.read_loader_definition import read_loader_definition diff --git a/src/sqlbuild/executor/migrations/_helpers/naming.py b/src/sqlbuild/executor/migrations/_helpers/naming.py index 86056211fa..67b53bc9df 100644 --- a/src/sqlbuild/executor/migrations/_helpers/naming.py +++ b/src/sqlbuild/executor/migrations/_helpers/naming.py @@ -6,7 +6,7 @@ from datetime import datetime, timedelta from typing import Any -from sqlbuild._native import build_janitor_archive_name +from sqlbuild._native import ModelMigrationPlanEntry, build_janitor_archive_name from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter from sqlbuild.adapter.contract.models import RelationInfo from sqlbuild.adapter.relations.main.resolve_qualified_name_parts import ( diff --git a/src/sqlbuild/executor/migrations/_helpers/old_name_views.py b/src/sqlbuild/executor/migrations/_helpers/old_name_views.py index 38b7aa6ff0..996d1072af 100644 --- a/src/sqlbuild/executor/migrations/_helpers/old_name_views.py +++ b/src/sqlbuild/executor/migrations/_helpers/old_name_views.py @@ -9,6 +9,15 @@ from typing import Any from sqlbuild._native import ( + ColumnMigrationEvent, + CompiledRelationLocation, + Duration, + MigrationRelation, + OldNameView, + OldNameViewEvent, + OldNameViewEventType, + OldNameViewPlanEntry, + PlanOutput, build_janitor_archive_name, deterministic_migration_event_id, read_recorded_column_renames, diff --git a/src/sqlbuild/executor/old_name_views/_helpers/janitor.py b/src/sqlbuild/executor/old_name_views/_helpers/janitor.py index e9251027d0..e2b7e4e095 100644 --- a/src/sqlbuild/executor/old_name_views/_helpers/janitor.py +++ b/src/sqlbuild/executor/old_name_views/_helpers/janitor.py @@ -9,8 +9,6 @@ OldNameViewDropReason, OldNameViewEvent, OldNameViewEventType, - OldNameViewHistory, - OldNameViewStatus, deterministic_old_name_view_event_id, write_migration_state_event, ) @@ -22,11 +20,6 @@ resolve_qualified_name_parts, ) from sqlbuild.adapter.type_system.main.normalize_relation_type import normalize_relation_type -from sqlbuild.compiler.migrations.models import OldNameViewEvent -from sqlbuild.compiler.migrations.types import ( - OldNameViewDropReason, - OldNameViewEventType, -) from sqlbuild.executor.janitor.models import ( JanitorOldNameView, JanitorOldNameViewPlanning, diff --git a/src/sqlbuild/executor/testing/main/_execute.py b/src/sqlbuild/executor/testing/main/_execute.py index e751c8351b..69951e0ae0 100644 --- a/src/sqlbuild/executor/testing/main/_execute.py +++ b/src/sqlbuild/executor/testing/main/_execute.py @@ -4,8 +4,9 @@ from typing import Any -from sqlbuild._native import CompileInputError, SqlTestPlanEntry, render_sql_test_comparison_sql +from sqlbuild._native import SqlTestPlanEntry, render_sql_test_comparison_sql from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter +from sqlbuild.compiler.compile.exceptions import CompileInputError from sqlbuild.executor.testing._helpers.difference_samples import add_difference_samples from sqlbuild.executor.testing._helpers.expected_column_probe import ( missing_expected_columns_message, diff --git a/src/sqlbuild/integrations/dlt/main/_loaders.py b/src/sqlbuild/integrations/dlt/main/_loaders.py index df463458d3..8c8bc80459 100644 --- a/src/sqlbuild/integrations/dlt/main/_loaders.py +++ b/src/sqlbuild/integrations/dlt/main/_loaders.py @@ -4,10 +4,10 @@ from collections.abc import Callable +from sqlbuild._native import SourceEntry from sqlbuild.executor.load.models import LoaderContext from sqlbuild.integrations.dlt._helpers.runner import run_dlt_source from sqlbuild.integrations.dlt.models import DltSourceConfig -from sqlbuild.spec.contracts.models import SourceEntry def dlt_loader_function(*, source_entry: SourceEntry, name: str) -> Callable[[LoaderContext], None]: diff --git a/src/sqlbuild/integrations/ingestr/main/_loaders.py b/src/sqlbuild/integrations/ingestr/main/_loaders.py index a8c5688131..8516b83633 100644 --- a/src/sqlbuild/integrations/ingestr/main/_loaders.py +++ b/src/sqlbuild/integrations/ingestr/main/_loaders.py @@ -4,6 +4,7 @@ from collections.abc import Callable +from sqlbuild._native import SourceEntry from sqlbuild.executor.load.models import LoaderContext from sqlbuild.integrations.ingestr._helpers.command import build_ingestr_command from sqlbuild.integrations.ingestr._helpers.output import record_ingestr_output diff --git a/src/sqlbuild/runtime/event_exporting/constants.py b/src/sqlbuild/runtime/event_exporting/constants.py index 8135309f9b..1467e799be 100644 --- a/src/sqlbuild/runtime/event_exporting/constants.py +++ b/src/sqlbuild/runtime/event_exporting/constants.py @@ -2,7 +2,6 @@ from sqlbuild._native import EventExportSeverity from sqlbuild.runtime.event_exporting.types import LifecycleEventKind -from sqlbuild.spec.contracts.types import EventExportSeverity EVENT_EXPORTER_EVENT_PARAMETER_NAME: str = "event" EVENT_EXPORT_SEVERITIES: tuple[str, ...] = tuple(item.value for item in EventExportSeverity) diff --git a/src/sqlbuild/runtime/event_exporting/models.py b/src/sqlbuild/runtime/event_exporting/models.py index a75ce68508..6f2f6e6725 100644 --- a/src/sqlbuild/runtime/event_exporting/models.py +++ b/src/sqlbuild/runtime/event_exporting/models.py @@ -4,7 +4,7 @@ from dataclasses import dataclass -from sqlbuild.spec.contracts.types import EventExportSeverity +from sqlbuild._native import EventExportSeverity @dataclass(frozen=True) diff --git a/tests/integration/src/sqlbuild/cli/commands/main/test_provider_annotation_tolerance.py b/tests/integration/src/sqlbuild/cli/commands/main/test_provider_annotation_tolerance.py index 6a4d7ddab6..07ed61665f 100644 --- a/tests/integration/src/sqlbuild/cli/commands/main/test_provider_annotation_tolerance.py +++ b/tests/integration/src/sqlbuild/cli/commands/main/test_provider_annotation_tolerance.py @@ -6,9 +6,12 @@ import pytest -from sqlbuild._native import discover_provider_classes, discover_python_node_functions +from sqlbuild._native import ( + DiscoveredPythonNodeFunctions, + discover_provider_classes, + discover_python_node_functions, +) from sqlbuild._native import run_cli as main -from sqlbuild.compiler.discovery.models import DiscoveredPythonNodeFunctions from tests.integration.src.sqlbuild.cli.commands.main._test_types import ( ProviderAnnotationToleranceTestCase, ) diff --git a/tests/integration/src/sqlbuild/compiler/analysis_session/helpers.py b/tests/integration/src/sqlbuild/compiler/analysis_session/helpers.py index 2986b02dec..a9d2585e20 100644 --- a/tests/integration/src/sqlbuild/compiler/analysis_session/helpers.py +++ b/tests/integration/src/sqlbuild/compiler/analysis_session/helpers.py @@ -20,7 +20,6 @@ ColumnLineageMode, CompactLineageFacts, CompileAdapterContext, - CompileInputError, DynamicColumnContractProof, InferredColumn, ModelSqlAnalysis, @@ -43,6 +42,7 @@ from sqlbuild.adapter.contract.models import ExpressionInferenceProfile from sqlbuild.adapters.duckdb.classes.duckdb_adapter import DuckDbAdapter from sqlbuild.cli.entry.main.entry import main +from sqlbuild.compiler.compile.exceptions import CompileInputError from sqlbuild.compiler.compile.types import CompiledProjectView from sqlbuild.compiler.lineage.types import InferredNullability from tests.integration.src.sqlbuild.compiler.analysis_session._test_types import ( diff --git a/tests/integration/src/sqlbuild/compiler/analysis_session/test_native_query_columns.py b/tests/integration/src/sqlbuild/compiler/analysis_session/test_native_query_columns.py index 5e04593722..78bfa1f58a 100644 --- a/tests/integration/src/sqlbuild/compiler/analysis_session/test_native_query_columns.py +++ b/tests/integration/src/sqlbuild/compiler/analysis_session/test_native_query_columns.py @@ -12,13 +12,9 @@ import pytest import sqlbuild._native as native_module -from sqlbuild._native import ( - CompileInputError, - NativeColumnQuery, - NativeModelAnalyses, - infer_native_query_columns, -) +from sqlbuild._native import NativeColumnQuery, NativeModelAnalyses, infer_native_query_columns from sqlbuild.adapter.contract.models import ExpressionInferenceProfile +from sqlbuild.compiler.compile.exceptions import CompileInputError from sqlbuild.compiler.compile.types import CompiledModelView, CompiledProjectView from tests.integration.src.sqlbuild.compiler.analysis_session._test_types import ( FallbackColumnsTestCase, diff --git a/tests/integration/src/sqlbuild/compiler/attachments/helpers.py b/tests/integration/src/sqlbuild/compiler/attachments/helpers.py index c382bd4f56..ac846e8dba 100644 --- a/tests/integration/src/sqlbuild/compiler/attachments/helpers.py +++ b/tests/integration/src/sqlbuild/compiler/attachments/helpers.py @@ -9,12 +9,12 @@ import sqlbuild._native as _native from sqlbuild._native import ( - CompileInputError, SqlLexicalSyntax, SqlReferenceScan, find_macro_call_names, scan_sql_reference_calls, ) +from sqlbuild.compiler.compile.exceptions import CompileInputError from sqlbuild.compiler.compile.types import SqlTestMode _SQL_PIECES: tuple[str, ...] = ( diff --git a/tests/integration/src/sqlbuild/compiler/compile/helpers.py b/tests/integration/src/sqlbuild/compiler/compile/helpers.py index 926cba2c18..1079860f2b 100644 --- a/tests/integration/src/sqlbuild/compiler/compile/helpers.py +++ b/tests/integration/src/sqlbuild/compiler/compile/helpers.py @@ -16,11 +16,9 @@ from sqlbuild._native import ( CompileAdapterContext, - CompileInputError, CompilerStage, DiscoveredProjectInputs, LoadedMacro, - MacroContext, MacroExpansionResult, ResourceIdentity, ResourceKind, @@ -33,6 +31,8 @@ resolve_effective_collection_rendering, ) from sqlbuild.adapters.duckdb.classes.duckdb_adapter import DuckDbAdapter +from sqlbuild.compiler.compile.exceptions import CompileInputError +from sqlbuild.compiler.compile.models import MacroContext from sqlbuild.compiler.macro_bridge.classes.macro_bridge import MacroBridge from sqlbuild.compiler.macro_bridge.constants import ACTIVE_MACRO_BRIDGE diff --git a/tests/integration/src/sqlbuild/compiler/compiled_project/test_native_audits.py b/tests/integration/src/sqlbuild/compiler/compiled_project/test_native_audits.py index 3d593b327a..d10fcc3fb3 100644 --- a/tests/integration/src/sqlbuild/compiler/compiled_project/test_native_audits.py +++ b/tests/integration/src/sqlbuild/compiler/compiled_project/test_native_audits.py @@ -7,7 +7,7 @@ import pytest -from sqlbuild._native import CompileInputError +from sqlbuild.compiler.compile.exceptions import CompileInputError from sqlbuild.compiler.compile.types import CompiledAuditView from tests.integration.src.sqlbuild.compiler.compiled_project._test_types import ( NativeAuditAttachmentTestCase, diff --git a/tests/integration/src/sqlbuild/compiler/compiled_project/test_native_compiled_project.py b/tests/integration/src/sqlbuild/compiler/compiled_project/test_native_compiled_project.py index c6db982c6f..2c9d07b14f 100644 --- a/tests/integration/src/sqlbuild/compiler/compiled_project/test_native_compiled_project.py +++ b/tests/integration/src/sqlbuild/compiler/compiled_project/test_native_compiled_project.py @@ -11,7 +11,6 @@ import pytest from sqlbuild._native import ( - CompileInputError, InferredColumn, InvalidQualifiedIdentityError, build_compiled_project, @@ -19,6 +18,7 @@ ) from sqlbuild.adapters.duckdb.classes.duckdb_adapter import DuckDbAdapter from sqlbuild.cli.entry.main.entry import main +from sqlbuild.compiler.compile.exceptions import CompileInputError from sqlbuild.compiler.compile.types import CompiledProjectView from tests.integration.src.sqlbuild.compiler.compiled_project._test_types import ( ImmediateSyntaxErrorTestCase, diff --git a/tests/integration/src/sqlbuild/compiler/compiled_project/test_native_functions.py b/tests/integration/src/sqlbuild/compiler/compiled_project/test_native_functions.py index 631b4338db..bd2c76ed63 100644 --- a/tests/integration/src/sqlbuild/compiler/compiled_project/test_native_functions.py +++ b/tests/integration/src/sqlbuild/compiler/compiled_project/test_native_functions.py @@ -7,7 +7,7 @@ import pytest -from sqlbuild._native import CompileInputError +from sqlbuild.compiler.compile.exceptions import CompileInputError from sqlbuild.compiler.compile.types import CompiledFunctionView from tests.integration.src.sqlbuild.compiler.compiled_project._test_types import ( NativeFunctionAttachmentTestCase, diff --git a/tests/integration/src/sqlbuild/compiler/project_assembly/test_native_project_assembly.py b/tests/integration/src/sqlbuild/compiler/project_assembly/test_native_project_assembly.py index cbaa13dc99..e3bc480ef9 100644 --- a/tests/integration/src/sqlbuild/compiler/project_assembly/test_native_project_assembly.py +++ b/tests/integration/src/sqlbuild/compiler/project_assembly/test_native_project_assembly.py @@ -11,8 +11,9 @@ import pytest import sqlbuild._native as native_module -from sqlbuild._native import CompileInputError, assemble_project +from sqlbuild._native import assemble_project from sqlbuild.adapter.contract.models import ExpressionInferenceProfile +from sqlbuild.compiler.compile.exceptions import CompileInputError from sqlbuild.compiler.compile.types import CompiledProjectView from tests.integration.src.sqlbuild.compiler.helpers import ( GoldenEntry, diff --git a/tests/integration/src/sqlbuild/compiler/scopes/helpers.py b/tests/integration/src/sqlbuild/compiler/scopes/helpers.py index 19d13a53b1..02360d80eb 100644 --- a/tests/integration/src/sqlbuild/compiler/scopes/helpers.py +++ b/tests/integration/src/sqlbuild/compiler/scopes/helpers.py @@ -14,7 +14,6 @@ import sqlbuild._native as _native from sqlbuild._native import ( - CompileInputError, ConstantDeclaration, DeclarationIdentity, DeclarationKind, @@ -43,6 +42,7 @@ ) from sqlbuild.adapters.duckdb.classes.duckdb_adapter import DuckDbAdapter from sqlbuild.compiler.compile._helpers.render.user_macros import load_project_macros +from sqlbuild.compiler.compile.exceptions import CompileInputError from tests.integration.src.sqlbuild.compiler.helpers import GoldenEntry, golden_entry CURRENT_PATH_COMPONENT: str = "." diff --git a/tests/integration/src/sqlbuild/executor/diff/test_query_artifact_cleanup.py b/tests/integration/src/sqlbuild/executor/diff/test_query_artifact_cleanup.py index 3f37924d40..36acabc258 100644 --- a/tests/integration/src/sqlbuild/executor/diff/test_query_artifact_cleanup.py +++ b/tests/integration/src/sqlbuild/executor/diff/test_query_artifact_cleanup.py @@ -10,6 +10,7 @@ import pytest from sqlbuild._native import ( + CompiledProject, DiffCommandRequest, QueryDiffPreparation, build_janitor_plan, diff --git a/tests/integration/src/sqlbuild/rule_engine/host_payload/helpers.py b/tests/integration/src/sqlbuild/rule_engine/host_payload/helpers.py index ddaa1811cd..7f45fc3eba 100644 --- a/tests/integration/src/sqlbuild/rule_engine/host_payload/helpers.py +++ b/tests/integration/src/sqlbuild/rule_engine/host_payload/helpers.py @@ -6,7 +6,7 @@ import pytest -import sqlbuild._native as compile_models +import sqlbuild.compiler.compile.models as compile_models from sqlbuild._native import ( CompiledModel, CompiledProject, diff --git a/tests/unit/src/sqlbuild/cli/commands/_helpers/runtime/test_connection.py b/tests/unit/src/sqlbuild/cli/commands/_helpers/runtime/test_connection.py index e6de32c867..dbd92dbfe6 100644 --- a/tests/unit/src/sqlbuild/cli/commands/_helpers/runtime/test_connection.py +++ b/tests/unit/src/sqlbuild/cli/commands/_helpers/runtime/test_connection.py @@ -6,7 +6,6 @@ from sqlbuild import _native as native from sqlbuild._native import ( - CompileInputError, DiscoveredProjectInputs, LocalConfig, LocalTargetConfig, @@ -19,6 +18,7 @@ resolve_target_connection_config, ) from sqlbuild.adapter.contract.types import BuiltinAdapter +from sqlbuild.compiler.compile.exceptions import CompileInputError from tests.unit.src.sqlbuild.cli.commands._helpers.runtime._test_types import ( ExplicitMemoryConnectionTestCase, MissingConnectionTestCase, diff --git a/tests/unit/src/sqlbuild/cli/commands/main/entry/_test_types.py b/tests/unit/src/sqlbuild/cli/commands/main/entry/_test_types.py index 1bcf26a480..334d3df385 100644 --- a/tests/unit/src/sqlbuild/cli/commands/main/entry/_test_types.py +++ b/tests/unit/src/sqlbuild/cli/commands/main/entry/_test_types.py @@ -5,11 +5,11 @@ from sqlbuild._native import ( CliUserError, ColumnLineageMode, - CompileInputError, CompileLineageMode, ProjectConfigError, SpecConfigError, ) +from sqlbuild.compiler.compile.exceptions import CompileInputError @dataclass(frozen=True) diff --git a/tests/unit/src/sqlbuild/cli/commands/main/entry/test_main.py b/tests/unit/src/sqlbuild/cli/commands/main/entry/test_main.py index 5e82c8d4d6..b3fbacdb99 100644 --- a/tests/unit/src/sqlbuild/cli/commands/main/entry/test_main.py +++ b/tests/unit/src/sqlbuild/cli/commands/main/entry/test_main.py @@ -16,7 +16,6 @@ CloneCommandRequest, ColumnLineageMode, CompileCommandRequest, - CompileInputError, CompileLineageMode, CostCommandRequest, CursorOverrides, @@ -36,6 +35,7 @@ ) from sqlbuild._native import run_cli as _main_with_dependencies from sqlbuild._native import run_cli as main +from sqlbuild.compiler.compile.exceptions import CompileInputError from sqlbuild.executor.pipeline.exceptions import AuditExecutionError from sqlbuild.executor.pipeline.models import AuditExecutionFailure from sqlbuild.observability import EventDispatcher, ExecutionIdentity, current_execution_identity diff --git a/tests/unit/src/sqlbuild/compiler/compile/_helpers/config/test_table_type.py b/tests/unit/src/sqlbuild/compiler/compile/_helpers/config/test_table_type.py index e0dc676c2f..7b1e157a0c 100644 --- a/tests/unit/src/sqlbuild/compiler/compile/_helpers/config/test_table_type.py +++ b/tests/unit/src/sqlbuild/compiler/compile/_helpers/config/test_table_type.py @@ -3,7 +3,6 @@ import pytest from sqlbuild._native import ( - CompileInputError, CompileModelConfig, DefaultsConfig, MaterializationDefaultsConfig, @@ -13,6 +12,7 @@ TableTypeSource, TargetConfig, ) +from sqlbuild.compiler.compile.exceptions import CompileInputError from tests.unit.src.sqlbuild.compiler.compile._helpers.config._test_types import ( TableTypeResolutionErrorTestCase, TableTypeResolutionTestCase, diff --git a/tests/unit/src/sqlbuild/compiler/compile/_helpers/helpers.py b/tests/unit/src/sqlbuild/compiler/compile/_helpers/helpers.py index 65690b902f..904478c5df 100644 --- a/tests/unit/src/sqlbuild/compiler/compile/_helpers/helpers.py +++ b/tests/unit/src/sqlbuild/compiler/compile/_helpers/helpers.py @@ -31,7 +31,6 @@ DiscoveredMacroFile, DiscoveredProjectInputs, LoadedMacro, - MacroContext, MacroMetadata, NativeColumnQuery, NativeQueryColumns, @@ -63,6 +62,7 @@ from sqlbuild.adapters.snowflake.classes.snowflake_adapter import SnowflakeAdapter from sqlbuild.adapters.sqlserver.classes.sqlserver_adapter import SqlServerAdapter from sqlbuild.compiler.compile._helpers.render.user_macros import load_project_macros +from sqlbuild.compiler.compile.models import MacroContext from sqlbuild.compiler.compile.types import ( CompiledModelView, CompiledProjectView, diff --git a/tests/unit/src/sqlbuild/compiler/compile/_helpers/render/test_cursor_intrinsics.py b/tests/unit/src/sqlbuild/compiler/compile/_helpers/render/test_cursor_intrinsics.py index 0f38530fe4..49e2ce9725 100644 --- a/tests/unit/src/sqlbuild/compiler/compile/_helpers/render/test_cursor_intrinsics.py +++ b/tests/unit/src/sqlbuild/compiler/compile/_helpers/render/test_cursor_intrinsics.py @@ -3,7 +3,6 @@ import pytest from sqlbuild._native import ( - CompileInputError, InferredColumn, NativeColumnQuery, cursor_intrinsics_analysis_sql, @@ -12,6 +11,7 @@ validated_model_cursor_intrinsics, ) from sqlbuild.adapter.contract.models import ExpressionInferenceProfile +from sqlbuild.compiler.compile.exceptions import CompileInputError from sqlbuild.compiler.lineage.types import InferredNullability from tests.unit.src.sqlbuild.compiler.compile._helpers.render._test_types import ( CursorIntrinsicAnalysisTestCase, diff --git a/tests/unit/src/sqlbuild/compiler/compile/_helpers/test_audit_factories.py b/tests/unit/src/sqlbuild/compiler/compile/_helpers/test_audit_factories.py index 6f9c5666b7..8f541715e3 100644 --- a/tests/unit/src/sqlbuild/compiler/compile/_helpers/test_audit_factories.py +++ b/tests/unit/src/sqlbuild/compiler/compile/_helpers/test_audit_factories.py @@ -9,12 +9,12 @@ import pytest from sqlbuild._native import ( - CompileInputError, SchemaModelEntry, SourceLocation, build_compile_inputs, discover_project_inputs, ) +from sqlbuild.compiler.compile.exceptions import CompileInputError from tests.unit.src.sqlbuild.compiler.compile._helpers._test_types import ( AuditFactoryAttachmentTestCase, ) diff --git a/tests/unit/src/sqlbuild/compiler/compile/_helpers/test_declaration_scopes.py b/tests/unit/src/sqlbuild/compiler/compile/_helpers/test_declaration_scopes.py index 7e2007f1a7..beef2dd785 100644 --- a/tests/unit/src/sqlbuild/compiler/compile/_helpers/test_declaration_scopes.py +++ b/tests/unit/src/sqlbuild/compiler/compile/_helpers/test_declaration_scopes.py @@ -10,7 +10,6 @@ import pytest from sqlbuild._native import ( - CompileInputError, DeclarationIdentity, DeclarationKind, DiscoveredProjectInputs, @@ -26,6 +25,7 @@ discover_project_inputs, scope_metadata_projection, ) +from sqlbuild.compiler.compile.exceptions import CompileInputError from sqlbuild.compiler.compile.types import CompiledProjectView from tests.unit.src.sqlbuild.compiler.compile._helpers._test_types import ( ExpectedBooleanTestCase, diff --git a/tests/unit/src/sqlbuild/compiler/compile/_helpers/test_declarations.py b/tests/unit/src/sqlbuild/compiler/compile/_helpers/test_declarations.py index d629765405..c5000b10f7 100644 --- a/tests/unit/src/sqlbuild/compiler/compile/_helpers/test_declarations.py +++ b/tests/unit/src/sqlbuild/compiler/compile/_helpers/test_declarations.py @@ -8,7 +8,6 @@ import pytest from sqlbuild._native import ( - CompileInputError, DeclarationParseError, DiscoveredProjectInputs, ResourceIdentityError, @@ -19,6 +18,7 @@ compute_query_hash, discover_project_inputs, ) +from sqlbuild.compiler.compile.exceptions import CompileInputError from sqlbuild.compiler.compile.types import CompiledModelView, CompiledProjectView from tests.unit.src.sqlbuild.compiler.compile._helpers._test_types import ( CompileDeclarationsErrorTestCase, diff --git a/tests/unit/src/sqlbuild/compiler/compile/_helpers/test_expansion_spans.py b/tests/unit/src/sqlbuild/compiler/compile/_helpers/test_expansion_spans.py index 6f2bd96993..2660dfddf4 100644 --- a/tests/unit/src/sqlbuild/compiler/compile/_helpers/test_expansion_spans.py +++ b/tests/unit/src/sqlbuild/compiler/compile/_helpers/test_expansion_spans.py @@ -9,11 +9,11 @@ from sqlbuild._native import ( ExpansionSpan, LoadedMacro, - MacroContext, MappedOffset, expand_authored_sql_with_spans, map_expanded_offset, ) +from sqlbuild.compiler.compile.models import MacroContext from tests.unit.src.sqlbuild.compiler.compile._helpers._test_types import ExpandWithSpansTestCase from tests.unit.src.sqlbuild.compiler.compile._helpers.helpers import ( DUCKDB_COMPILE_ADAPTER_CONTEXT, diff --git a/tests/unit/src/sqlbuild/compiler/compile/_helpers/test_microbatch_redesign.py b/tests/unit/src/sqlbuild/compiler/compile/_helpers/test_microbatch_redesign.py index e3a637ac74..ea02a03e59 100644 --- a/tests/unit/src/sqlbuild/compiler/compile/_helpers/test_microbatch_redesign.py +++ b/tests/unit/src/sqlbuild/compiler/compile/_helpers/test_microbatch_redesign.py @@ -5,7 +5,6 @@ import pytest from sqlbuild._native import ( - CompileInputError, CompileModelConfig, CursorBounds, CursorInputRelation, @@ -16,6 +15,7 @@ resolve_effective_timestamp_grain, watermark_type_is_compatible, ) +from sqlbuild.compiler.compile.exceptions import CompileInputError from sqlbuild.executor.run._helpers.validation.cursor_bounds import ( resolve_effective_timestamp_grain as resolve_runtime_effective_timestamp_grain, ) diff --git a/tests/unit/src/sqlbuild/compiler/compile/_helpers/test_model_config_validation.py b/tests/unit/src/sqlbuild/compiler/compile/_helpers/test_model_config_validation.py index e72b1e8244..17b779ab5a 100644 --- a/tests/unit/src/sqlbuild/compiler/compile/_helpers/test_model_config_validation.py +++ b/tests/unit/src/sqlbuild/compiler/compile/_helpers/test_model_config_validation.py @@ -7,12 +7,12 @@ import pytest from sqlbuild._native import ( - CompileInputError, CompileModelConfig, ConfigValueTypeError, ResolvedTimeTravelRetention, TimeTravelRetentionSource, ) +from sqlbuild.compiler.compile.exceptions import CompileInputError from tests.unit.src.sqlbuild.compiler.compile._helpers._test_types import ( ChangePolicyHelpTestCase, ContractConfigErrorTestCase, diff --git a/tests/unit/src/sqlbuild/compiler/compile/_helpers/test_model_schemas.py b/tests/unit/src/sqlbuild/compiler/compile/_helpers/test_model_schemas.py index f0e3257417..bc73d41f34 100644 --- a/tests/unit/src/sqlbuild/compiler/compile/_helpers/test_model_schemas.py +++ b/tests/unit/src/sqlbuild/compiler/compile/_helpers/test_model_schemas.py @@ -8,7 +8,6 @@ import pytest from sqlbuild._native import ( - CompileInputError, CompilerDiagnostic, DiscoveredProjectInputs, SchemaAuditInstance, @@ -21,6 +20,7 @@ evaluate_model_contracts, ) from sqlbuild.adapters.duckdb.classes.duckdb_adapter import DuckDbAdapter +from sqlbuild.compiler.compile.exceptions import CompileInputError from sqlbuild.compiler.compile.types import CompiledModelView, CompiledProjectView from tests.unit.src.sqlbuild.compiler.compile._helpers._test_types import ( DynamicModelSchemaTestCase, diff --git a/tests/unit/src/sqlbuild/compiler/compile/_helpers/test_parameterized_sql_tests.py b/tests/unit/src/sqlbuild/compiler/compile/_helpers/test_parameterized_sql_tests.py index 719fc001eb..8ce9866e21 100644 --- a/tests/unit/src/sqlbuild/compiler/compile/_helpers/test_parameterized_sql_tests.py +++ b/tests/unit/src/sqlbuild/compiler/compile/_helpers/test_parameterized_sql_tests.py @@ -10,7 +10,6 @@ from sqlbuild._native import ( CompiledObjectKey, CompiledSqlTest, - CompileInputError, SqlLogicalType, SqlValue, SqlValueKind, @@ -19,6 +18,7 @@ ) from sqlbuild.adapters.bigquery.classes.bigquery_adapter import BigQueryAdapter from sqlbuild.adapters.sqlserver.classes.sqlserver_adapter import SqlServerAdapter +from sqlbuild.compiler.compile.exceptions import CompileInputError from sqlbuild.compiler.compile.types import ( CompiledProjectView, CompiledResourceType, diff --git a/tests/unit/src/sqlbuild/compiler/discovery/_helpers/test_filesystem_audit_factories.py b/tests/unit/src/sqlbuild/compiler/discovery/_helpers/test_filesystem_audit_factories.py index 18b7540228..d78d981546 100644 --- a/tests/unit/src/sqlbuild/compiler/discovery/_helpers/test_filesystem_audit_factories.py +++ b/tests/unit/src/sqlbuild/compiler/discovery/_helpers/test_filesystem_audit_factories.py @@ -7,9 +7,10 @@ import pytest -from sqlbuild._native import PythonNodeDiscoveryError, discover_python_node_functions -from sqlbuild.compiler.discovery.models import ( +from sqlbuild._native import ( DiscoveredPythonNodeFunctions, + PythonNodeDiscoveryError, + discover_python_node_functions, ) from tests.unit.src.sqlbuild.compiler.discovery._helpers._test_types import ( AuditFactoryDiscoveryTestCase, diff --git a/tests/unit/src/sqlbuild/compiler/discovery/_helpers/test_filesystem_command_output_sinks.py b/tests/unit/src/sqlbuild/compiler/discovery/_helpers/test_filesystem_command_output_sinks.py index 48cc3d5cb7..e5bd1975e0 100644 --- a/tests/unit/src/sqlbuild/compiler/discovery/_helpers/test_filesystem_command_output_sinks.py +++ b/tests/unit/src/sqlbuild/compiler/discovery/_helpers/test_filesystem_command_output_sinks.py @@ -6,16 +6,14 @@ import pytest from sqlbuild._native import ( + DiscoveredCommandOutputSink, + DiscoveredProvider, EventExporterDiscoveryError, discover_provider_classes, ) from sqlbuild.compiler.discovery._helpers.filesystem.command_output_sinks import ( discover_command_output_sink_functions, ) -from sqlbuild.compiler.discovery.models import ( - DiscoveredCommandOutputSink, - DiscoveredProvider, -) from sqlbuild.sinks import CommandOutputStream from tests.unit.src.sqlbuild.compiler.discovery._helpers._test_types import ( CommandOutputSinkDiscoveryTestCase, diff --git a/tests/unit/src/sqlbuild/compiler/discovery/_helpers/test_filesystem_event_exporters.py b/tests/unit/src/sqlbuild/compiler/discovery/_helpers/test_filesystem_event_exporters.py index 7244ce5182..419fe53c88 100644 --- a/tests/unit/src/sqlbuild/compiler/discovery/_helpers/test_filesystem_event_exporters.py +++ b/tests/unit/src/sqlbuild/compiler/discovery/_helpers/test_filesystem_event_exporters.py @@ -6,14 +6,12 @@ import pytest from sqlbuild._native import ( + DiscoveredEventExporter, + DiscoveredProvider, EventExporterDiscoveryError, discover_event_exporter_functions, discover_provider_classes, ) -from sqlbuild.compiler.discovery.models import ( - DiscoveredEventExporter, - DiscoveredProvider, -) from tests.unit.src.sqlbuild.compiler.discovery._helpers._test_types import ( EventExporterDiscoveryTestCase, EventExporterSignatureErrorTestCase, diff --git a/tests/unit/src/sqlbuild/compiler/discovery/_helpers/test_filesystem_factories.py b/tests/unit/src/sqlbuild/compiler/discovery/_helpers/test_filesystem_factories.py index ce2714758c..4b248fb85d 100644 --- a/tests/unit/src/sqlbuild/compiler/discovery/_helpers/test_filesystem_factories.py +++ b/tests/unit/src/sqlbuild/compiler/discovery/_helpers/test_filesystem_factories.py @@ -7,9 +7,10 @@ import pytest -from sqlbuild._native import PythonNodeDiscoveryError, discover_python_node_functions -from sqlbuild.compiler.discovery.models import ( +from sqlbuild._native import ( DiscoveredPythonNodeFunctions, + PythonNodeDiscoveryError, + discover_python_node_functions, ) from tests.unit.src.sqlbuild.compiler.discovery._helpers._test_types import ( DiscoverPythonNodeFactoriesTestCase, diff --git a/tests/unit/src/sqlbuild/compiler/discovery/_helpers/test_filesystem_hooks.py b/tests/unit/src/sqlbuild/compiler/discovery/_helpers/test_filesystem_hooks.py index 02f1f92e7e..89e0f856cc 100644 --- a/tests/unit/src/sqlbuild/compiler/discovery/_helpers/test_filesystem_hooks.py +++ b/tests/unit/src/sqlbuild/compiler/discovery/_helpers/test_filesystem_hooks.py @@ -6,17 +6,10 @@ import pytest from sqlbuild._native import ( - PythonNodeDiscoveryError, - discover_hook_functions, - discover_provider_classes, -) -from sqlbuild.compiler.discovery.models import ( DiscoveredHookFunction, DiscoveredProvider, DiscoveredProviderUsage, PythonNodeDiscoveryError, -) -from sqlbuild.compiler.discovery._helpers.filesystem.core import ( discover_hook_functions, discover_provider_classes, ) diff --git a/tests/unit/src/sqlbuild/compiler/discovery/_helpers/test_filesystem_loaders.py b/tests/unit/src/sqlbuild/compiler/discovery/_helpers/test_filesystem_loaders.py index 74c3d57a2f..735972e909 100644 --- a/tests/unit/src/sqlbuild/compiler/discovery/_helpers/test_filesystem_loaders.py +++ b/tests/unit/src/sqlbuild/compiler/discovery/_helpers/test_filesystem_loaders.py @@ -6,9 +6,10 @@ import pytest -from sqlbuild._native import PythonNodeDiscoveryError, discover_python_node_functions -from sqlbuild.compiler.discovery.models import ( +from sqlbuild._native import ( DiscoveredLoaderFunction, + PythonNodeDiscoveryError, + discover_python_node_functions, ) from tests.unit.src.sqlbuild.compiler.discovery._helpers._test_types import ( DiscoverLoaderFunctionsTestCase, diff --git a/tests/unit/src/sqlbuild/compiler/discovery/_helpers/test_filesystem_materializations.py b/tests/unit/src/sqlbuild/compiler/discovery/_helpers/test_filesystem_materializations.py index 110165c40a..4943308fea 100644 --- a/tests/unit/src/sqlbuild/compiler/discovery/_helpers/test_filesystem_materializations.py +++ b/tests/unit/src/sqlbuild/compiler/discovery/_helpers/test_filesystem_materializations.py @@ -7,15 +7,9 @@ import pytest from sqlbuild._native import ( - discover_materialization_files, - discover_provider_classes, -) -from sqlbuild.compiler.discovery.models import ( DiscoveredMaterializationFile, DiscoveredProvider, DiscoveredProviderUsage, -) -from sqlbuild.compiler.discovery._helpers.filesystem.core import ( discover_materialization_files, discover_provider_classes, ) diff --git a/tests/unit/src/sqlbuild/compiler/discovery/_helpers/test_filesystem_providers.py b/tests/unit/src/sqlbuild/compiler/discovery/_helpers/test_filesystem_providers.py index d599d561f7..b7f2416dfd 100644 --- a/tests/unit/src/sqlbuild/compiler/discovery/_helpers/test_filesystem_providers.py +++ b/tests/unit/src/sqlbuild/compiler/discovery/_helpers/test_filesystem_providers.py @@ -6,13 +6,11 @@ import pytest from sqlbuild._native import ( + DiscoveredProvider, ProviderDiscoveryError, ResourceIdentityError, discover_provider_classes, ) -from sqlbuild.compiler.discovery.models import ( - DiscoveredProvider, -) from tests.unit.src.sqlbuild.compiler.discovery._helpers._test_types import ( DiscoverProviderCacheIsolationTestCase, DiscoverProviderClassesErrorTestCase, diff --git a/tests/unit/src/sqlbuild/compiler/discovery/_helpers/test_filesystem_python_nodes.py b/tests/unit/src/sqlbuild/compiler/discovery/_helpers/test_filesystem_python_nodes.py index 05bd85a827..b88b747bd4 100644 --- a/tests/unit/src/sqlbuild/compiler/discovery/_helpers/test_filesystem_python_nodes.py +++ b/tests/unit/src/sqlbuild/compiler/discovery/_helpers/test_filesystem_python_nodes.py @@ -11,18 +11,12 @@ import pytest from sqlbuild._native import ( - PythonNodeDiscoveryError, - discover_python_node_functions, -) -from sqlbuild.compiler.discovery.models import ( DiscoveredAssetFunction, DiscoveredCheckFunction, DiscoveredPythonNodeFunctions, DiscoveredTaskFunction, PythonNodeDiscoveryError, PythonNodeIdentity, -) -from sqlbuild.compiler.discovery._helpers.filesystem.core import ( discover_python_node_functions, ) from sqlbuild.compiler.python_nodes.main.identity import build_python_node_identity diff --git a/tests/unit/src/sqlbuild/compiler/discovery/_helpers/test_filesystem_scoped_declarations.py b/tests/unit/src/sqlbuild/compiler/discovery/_helpers/test_filesystem_scoped_declarations.py index 7e94edd8b4..6058740b9c 100644 --- a/tests/unit/src/sqlbuild/compiler/discovery/_helpers/test_filesystem_scoped_declarations.py +++ b/tests/unit/src/sqlbuild/compiler/discovery/_helpers/test_filesystem_scoped_declarations.py @@ -25,10 +25,6 @@ discover_sql_hook_files, discover_test_files, ) -from sqlbuild.compiler.discovery.models import ( - DiscoveredMacroFile, - DiscoveredProjectInputs, -) from tests.unit.src.sqlbuild.compiler.discovery._helpers._test_types import ( DiscoverGlobalDeclarationTestCase, DiscoverGroupedDeclarationTestCase, diff --git a/tests/unit/src/sqlbuild/compiler/macro_bridge/main/helpers.py b/tests/unit/src/sqlbuild/compiler/macro_bridge/main/helpers.py index 5d10440e96..f51152461b 100644 --- a/tests/unit/src/sqlbuild/compiler/macro_bridge/main/helpers.py +++ b/tests/unit/src/sqlbuild/compiler/macro_bridge/main/helpers.py @@ -5,7 +5,8 @@ from collections.abc import Callable from typing import cast -from sqlbuild._native import CompileInputError, NativeStageMismatchError +from sqlbuild._native import NativeStageMismatchError +from sqlbuild.compiler.compile.exceptions import CompileInputError from sqlbuild.compiler.macro_bridge.classes.macro_bridge import MacroBridge from sqlbuild.compiler.macro_bridge.main.active_macro_bridge import active_macro_bridge diff --git a/tests/unit/src/sqlbuild/compiler/planner/_helpers/sql_test_assembly/helpers.py b/tests/unit/src/sqlbuild/compiler/planner/_helpers/sql_test_assembly/helpers.py index 6f04361318..cfe2db5d8b 100644 --- a/tests/unit/src/sqlbuild/compiler/planner/_helpers/sql_test_assembly/helpers.py +++ b/tests/unit/src/sqlbuild/compiler/planner/_helpers/sql_test_assembly/helpers.py @@ -20,7 +20,6 @@ DiscoveredSqlTestBlock, DiscoveredSqlTestFile, LoadedMacro, - MacroContext, NativeSqlTestPlan, PlanWarning, SqlLexicalSyntax, @@ -38,6 +37,7 @@ from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter from sqlbuild.adapters.duckdb.classes.duckdb_adapter import DuckDbAdapter from sqlbuild.adapters.snowflake.classes.snowflake_adapter import SnowflakeAdapter +from sqlbuild.compiler.compile.models import MacroContext from sqlbuild.compiler.compile.types import ( CompiledFunctionView, CompiledModelView, diff --git a/tests/unit/src/sqlbuild/compiler/planner/_helpers/sql_test_assembly/test_plan_test.py b/tests/unit/src/sqlbuild/compiler/planner/_helpers/sql_test_assembly/test_plan_test.py index 7d89e3ffbb..b79cb1a6ec 100644 --- a/tests/unit/src/sqlbuild/compiler/planner/_helpers/sql_test_assembly/test_plan_test.py +++ b/tests/unit/src/sqlbuild/compiler/planner/_helpers/sql_test_assembly/test_plan_test.py @@ -14,7 +14,6 @@ CompiledProject, CompiledRelationLocation, CompiledSqlTest, - CompileInputError, CompileSqlTestCte, DiscoveredSqlTestBlock, DiscoveredSqlTestFile, @@ -39,6 +38,7 @@ from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter from sqlbuild.adapters.duckdb.classes.duckdb_adapter import DuckDbAdapter from sqlbuild.adapters.sqlserver.classes.sqlserver_adapter import SqlServerAdapter +from sqlbuild.compiler.compile.exceptions import CompileInputError from sqlbuild.compiler.compile.types import CompiledProjectView, CompiledResourceType, SqlTestMode from tests.unit.src.sqlbuild.compiler.planner._helpers.sql_test_assembly._test_types import ( AssertionChainCteErrorTestCase, diff --git a/tests/unit/src/sqlbuild/integrations/dlt/test_dlt_loaders.py b/tests/unit/src/sqlbuild/integrations/dlt/test_dlt_loaders.py index 564aa3cf45..0be18f8519 100644 --- a/tests/unit/src/sqlbuild/integrations/dlt/test_dlt_loaders.py +++ b/tests/unit/src/sqlbuild/integrations/dlt/test_dlt_loaders.py @@ -8,9 +8,7 @@ import pytest -from sqlbuild._native import discover_project_inputs -from sqlbuild.compiler.discovery.models import DiscoveredProjectInputs -from sqlbuild.compiler.discovery.types import LoaderConnectionMode +from sqlbuild._native import DiscoveredProjectInputs, LoaderConnectionMode, discover_project_inputs from tests.unit.src.sqlbuild.integrations.dlt._test_types import DltLoaderDiscoveryTestCase diff --git a/tests/unit/src/sqlbuild/integrations/ingestr/test_loaders.py b/tests/unit/src/sqlbuild/integrations/ingestr/test_loaders.py index 29d71abcd3..466917cf40 100644 --- a/tests/unit/src/sqlbuild/integrations/ingestr/test_loaders.py +++ b/tests/unit/src/sqlbuild/integrations/ingestr/test_loaders.py @@ -8,9 +8,7 @@ import pytest -from sqlbuild._native import discover_project_inputs -from sqlbuild.compiler.discovery.models import DiscoveredProjectInputs -from sqlbuild.compiler.discovery.types import LoaderConnectionMode +from sqlbuild._native import DiscoveredProjectInputs, LoaderConnectionMode, discover_project_inputs from tests.unit.src.sqlbuild.integrations.ingestr._test_types import ( IngestrLoaderDiscoveryTestCase, ) diff --git a/tests/unit/src/sqlbuild/runtime/event_exporting/classes/helpers.py b/tests/unit/src/sqlbuild/runtime/event_exporting/classes/helpers.py index 9ef062f5d0..af644e7136 100644 --- a/tests/unit/src/sqlbuild/runtime/event_exporting/classes/helpers.py +++ b/tests/unit/src/sqlbuild/runtime/event_exporting/classes/helpers.py @@ -3,12 +3,12 @@ from threading import Event from typing import ClassVar -from sqlbuild._native import BoundEventExporter, EventExporterDispatcher -from sqlbuild.compiler.discovery.models import ( +from sqlbuild._native import ( + BoundEventExporter, DiscoveredEventExporter, DiscoveredProvider, DiscoveredProviderUsage, - EventExportSeverity, + EventExporterDispatcher, ) from sqlbuild.observability import LifecycleEvent from sqlbuild.providers import Provider diff --git a/tests/unit/src/sqlbuild/runtime/event_exporting/classes/test_command_scope.py b/tests/unit/src/sqlbuild/runtime/event_exporting/classes/test_command_scope.py index d509b8f65a..4f40adb060 100644 --- a/tests/unit/src/sqlbuild/runtime/event_exporting/classes/test_command_scope.py +++ b/tests/unit/src/sqlbuild/runtime/event_exporting/classes/test_command_scope.py @@ -8,15 +8,13 @@ import pytest from sqlbuild._native import ( - EventExporterCommandScope, - EventExporterDispatcher, - EventExportSummary, -) -from sqlbuild.compiler.discovery.models import ( DiscoveredCommandOutputSink, DiscoveredEventExporter, DiscoveredProvider, DiscoveredProviderUsage, + EventExporterCommandScope, + EventExporterDispatcher, + EventExportSummary, ) from sqlbuild.providers import Provider from sqlbuild.sinks import CommandOutputRecord, CommandOutputStream diff --git a/tests/unit/src/sqlbuild/runtime/event_exporting/classes/test_dispatcher.py b/tests/unit/src/sqlbuild/runtime/event_exporting/classes/test_dispatcher.py index 47964dcdd6..8a0b9e0176 100644 --- a/tests/unit/src/sqlbuild/runtime/event_exporting/classes/test_dispatcher.py +++ b/tests/unit/src/sqlbuild/runtime/event_exporting/classes/test_dispatcher.py @@ -12,6 +12,7 @@ BoundEventExporter, EventExporterCounts, EventExporterDispatcher, + EventExportSeverity, EventExportSummary, ) from sqlbuild.observability import LifecycleEvent diff --git a/tests/unit/src/sqlbuild/runtime/event_exporting/test_policy.py b/tests/unit/src/sqlbuild/runtime/event_exporting/test_policy.py index 4f82779a20..9ddf36bb45 100644 --- a/tests/unit/src/sqlbuild/runtime/event_exporting/test_policy.py +++ b/tests/unit/src/sqlbuild/runtime/event_exporting/test_policy.py @@ -1,6 +1,7 @@ import pytest from sqlbuild._native import ( + EventExportSeverity, event_export_severity_at_least, lifecycle_export_policy_catalog, stricter_event_export_severity, @@ -8,7 +9,6 @@ from sqlbuild.runtime.event_exporting.constants import EVENT_EXPORT_KINDS from sqlbuild.runtime.event_exporting.types import LifecycleEventKind from sqlbuild.runtime.observability.constants import LIFECYCLE_EVENT_CATALOG -from sqlbuild.spec.contracts.types import EventExportSeverity from tests.unit.src.sqlbuild.runtime.event_exporting._test_types import ( EventExportPolicyTestCase, SeverityNarrowingTestCase, From 2e904c91bbd2d475f972c37bdd96a19fd4e8ad76 Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Thu, 15 Oct 2026 21:02:07 +0100 Subject: [PATCH 13/48] test: point remaining test scripts at natively hosted shapes --- .../sqlbuild-python/src/bindings/_helpers/cursor/conversion.rs | 2 +- .../src/bindings/_helpers/discovery/imported_modules.rs | 2 +- .../src/sqlbuild/rule_engine/host_payload/helpers.py | 2 +- .../compiler/planner/classes/test_sql_test_planning_exit.py | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/crates/sqlbuild-python/src/bindings/_helpers/cursor/conversion.rs b/crates/sqlbuild-python/src/bindings/_helpers/cursor/conversion.rs index f78954520c..0bbace1b4c 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/cursor/conversion.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/cursor/conversion.rs @@ -1,4 +1,4 @@ -//! One call's cursor values read from and built as `sqlbuild.cursor_algebra.models` objects. +//! One call's cursor values read from and built as the native cursor value records. use std::cell::RefCell; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/discovery/imported_modules.rs b/crates/sqlbuild-python/src/bindings/_helpers/discovery/imported_modules.rs index 939a4613b6..5257ca286a 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/discovery/imported_modules.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/discovery/imported_modules.rs @@ -115,7 +115,7 @@ pub(crate) fn module_functions<'py>( .collect() } -/// `sqlbuild.compiler.discovery.models.(**pairs)`. +/// `sqlbuild._native.(**pairs)`. pub(crate) fn discovered<'py>( py: Python<'py>, class_name: &str, diff --git a/tests/integration/src/sqlbuild/rule_engine/host_payload/helpers.py b/tests/integration/src/sqlbuild/rule_engine/host_payload/helpers.py index 7f45fc3eba..ddaa1811cd 100644 --- a/tests/integration/src/sqlbuild/rule_engine/host_payload/helpers.py +++ b/tests/integration/src/sqlbuild/rule_engine/host_payload/helpers.py @@ -6,7 +6,7 @@ import pytest -import sqlbuild.compiler.compile.models as compile_models +import sqlbuild._native as compile_models from sqlbuild._native import ( CompiledModel, CompiledProject, diff --git a/tests/unit/src/sqlbuild/compiler/planner/classes/test_sql_test_planning_exit.py b/tests/unit/src/sqlbuild/compiler/planner/classes/test_sql_test_planning_exit.py index a1686d952e..fc73db0258 100644 --- a/tests/unit/src/sqlbuild/compiler/planner/classes/test_sql_test_planning_exit.py +++ b/tests/unit/src/sqlbuild/compiler/planner/classes/test_sql_test_planning_exit.py @@ -39,7 +39,7 @@ def slow_plan_selected_sql_tests(**arguments): _FAILED_BUILD: str = dedent( """ from sqlbuild.cli.entry.main.entry import main - from sqlbuild.compiler.planner.exceptions import PlannerInputError + from sqlbuild._native import PlannerInputError def failing_reconciliation(**arguments): From f62e56437c0c6e72766da831ac32e15bbd7f0a6e Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Thu, 15 Oct 2026 16:57:41 +0100 Subject: [PATCH 14/48] fix: accept path-like Dagster CLI arguments and match Python isalnum in check names --- .../main/normalize_check_name.rs | 5 +- .../dagster_invocation/tests/test_texts.rs | 30 ++++++++++ .../dagster/_helpers/invocation_factory.py | 20 ++++++- .../integrations/dagster/_test_types.py | 14 +++++ .../integrations/dagster/test_resource.py | 58 +++++++++++++++++++ 5 files changed, 123 insertions(+), 4 deletions(-) diff --git a/crates/sqlbuild-core/src/dagster_invocation/main/normalize_check_name.rs b/crates/sqlbuild-core/src/dagster_invocation/main/normalize_check_name.rs index 2ebe69f43b..41b13b95f8 100644 --- a/crates/sqlbuild-core/src/dagster_invocation/main/normalize_check_name.rs +++ b/crates/sqlbuild-core/src/dagster_invocation/main/normalize_check_name.rs @@ -1,14 +1,15 @@ //! Dagster-safe check name segments. use crate::dagster_invocation::constants::{CHECK_NAME_SEPARATOR, DEFAULT_CHECK_NAME}; +use crate::text::main::python_isalnum::python_isalnum; -/// `value` with every non-alphanumeric character replaced and outer separators trimmed. +/// `value` with every character Python's `str.isalnum` rejects replaced and outer separators trimmed. #[must_use] pub fn normalize_check_name(value: &str) -> String { let normalized: String = value .chars() .map(|character| { - if character.is_alphanumeric() || character == CHECK_NAME_SEPARATOR { + if python_isalnum(character) || character == CHECK_NAME_SEPARATOR { character } else { CHECK_NAME_SEPARATOR diff --git a/crates/sqlbuild-core/src/dagster_invocation/tests/test_texts.rs b/crates/sqlbuild-core/src/dagster_invocation/tests/test_texts.rs index 8ee21180a9..573826ed85 100644 --- a/crates/sqlbuild-core/src/dagster_invocation/tests/test_texts.rs +++ b/crates/sqlbuild-core/src/dagster_invocation/tests/test_texts.rs @@ -21,6 +21,36 @@ fn given_raw_check_name_when_normalizing_then_replaces_punctuation() { value: "--", expected_name: "check", }, + CheckNameTestCase { + description: "combining vowel sign is not alphanumeric", + value: "\u{915}\u{93f}1", + expected_name: "\u{915}_1", + }, + CheckNameTestCase { + description: "trailing combining mark is trimmed as a separator", + value: "\u{915}\u{93f}", + expected_name: "\u{915}", + }, + CheckNameTestCase { + description: "combining acute accent becomes a separator", + value: "e\u{301}tat", + expected_name: "e_tat", + }, + CheckNameTestCase { + description: "precomposed letters and Arabic-Indic digits are kept", + value: "\u{dc}n\u{ef}c\u{f6}d\u{e9}_\u{663}\u{664}", + expected_name: "\u{dc}n\u{ef}c\u{f6}d\u{e9}_\u{663}\u{664}", + }, + CheckNameTestCase { + description: "superscript, fraction and roman numeral are numeric", + value: "x\u{b2}\u{bd}\u{216b}", + expected_name: "x\u{b2}\u{bd}\u{216b}", + }, + CheckNameTestCase { + description: "ideographs and kana are kept around a separator", + value: "\u{65e5}\u{672c}\u{8a9e}-\u{30c1}\u{30a7}\u{30c3}\u{30af}", + expected_name: "\u{65e5}\u{672c}\u{8a9e}_\u{30c1}\u{30a7}\u{30c3}\u{30af}", + }, ]; for test_case in test_cases { assert_eq!( diff --git a/src/sqlbuild/integrations/dagster/_helpers/invocation_factory.py b/src/sqlbuild/integrations/dagster/_helpers/invocation_factory.py index 8c35568cff..7206d01408 100644 --- a/src/sqlbuild/integrations/dagster/_helpers/invocation_factory.py +++ b/src/sqlbuild/integrations/dagster/_helpers/invocation_factory.py @@ -7,7 +7,7 @@ import subprocess from collections.abc import Mapping, Sequence from pathlib import Path -from typing import Any +from typing import Any, cast from sqlbuild._native import dagster_cli_arguments from sqlbuild.cli.output.constants import INTEGRATION_RESULT_PATH_ENV @@ -34,7 +34,7 @@ def start_sqlbuild_cli_invocation( """Start a SQLBuild CLI subprocess and return its invocation wrapper.""" planned: dict[str, Any] = dagster_cli_arguments( - args=list(args), + args=_cli_arguments(args), dag=dag, has_context=context is not None, read_selection=lambda: ( @@ -83,3 +83,19 @@ def start_sqlbuild_cli_invocation( ) invocation.execution_json_owned = execution_json_owned return invocation + + +def _cli_arguments(args: Sequence[object]) -> list[str]: + """Arguments as strings: path-like values pass through, others fail as `str` methods would.""" + + arguments: list[str] = [] + for argument in args: + if isinstance(argument, os.PathLike): + arguments.append(os.fsdecode(cast("os.PathLike[str]", argument))) + elif isinstance(argument, str): + arguments.append(argument) + else: + raise AttributeError( + f"{type(argument).__name__!r} object has no attribute 'startswith'" + ) + return arguments diff --git a/tests/unit/src/sqlbuild/integrations/dagster/_test_types.py b/tests/unit/src/sqlbuild/integrations/dagster/_test_types.py index ce9886ddbd..620bbf8231 100644 --- a/tests/unit/src/sqlbuild/integrations/dagster/_test_types.py +++ b/tests/unit/src/sqlbuild/integrations/dagster/_test_types.py @@ -124,6 +124,20 @@ class DagsterAssetCheckFilterTestCase: unexpected_check_names: tuple[str, ...] +@dataclass(frozen=True) +class DagsterCliArgumentTypeTestCase: + description: str + args: tuple[object, ...] + expected_args: tuple[str, ...] + + +@dataclass(frozen=True) +class DagsterCliArgumentErrorTestCase: + description: str + args: tuple[object, ...] + expected_error: str + + @dataclass(frozen=True) class DagsterCliInvocationTestCase: description: str diff --git a/tests/unit/src/sqlbuild/integrations/dagster/test_resource.py b/tests/unit/src/sqlbuild/integrations/dagster/test_resource.py index 492cdcaba0..328b01665b 100644 --- a/tests/unit/src/sqlbuild/integrations/dagster/test_resource.py +++ b/tests/unit/src/sqlbuild/integrations/dagster/test_resource.py @@ -30,6 +30,8 @@ from sqlbuild.integrations.dagster.classes.sqlbuild_cli_invocation import SqlBuildCliInvocation from tests.unit.src.sqlbuild.integrations.dagster._test_types import ( DagsterAuditIdentityTestCase, + DagsterCliArgumentErrorTestCase, + DagsterCliArgumentTypeTestCase, DagsterCliCloneFailureTestCase, DagsterCliCloneStreamTestCase, DagsterCliFailureTestCase, @@ -809,6 +811,62 @@ def test_given_sqlbuild_cli_resource_when_waiting_invocation_then_captures_proce assert invocation.stderr == test_case.expected_stderr +@pytest.mark.parametrize( + "test_case", + [ + DagsterCliArgumentTypeTestCase( + description="path-like arguments reach the subprocess as paths", + args=("build", "--select-file", Path("selectors/orders.txt")), + expected_args=("build", "--select-file", "selectors/orders.txt"), + ), + ], + ids=lambda case: case.description, +) +def test_given_path_like_cli_arguments_when_starting_without_context_then_paths_pass_through( + test_case: DagsterCliArgumentTypeTestCase, tmp_path: Path +) -> None: + project_dir: Path = tmp_path / "project" + project_dir.mkdir() + resource: SqlBuildCliResource = SqlBuildCliResource( + project_dir=str(project_dir), + sqb_command=write_fake_sqb_command(root=tmp_path, expected_args=test_case.expected_args), + ) + + invocation: SqlBuildCliInvocation = resource.cli( + cast(list[str], list(test_case.args)), raise_on_error=False + ).wait() + + assert invocation.is_successful(), invocation.stderr + assert invocation.command[-len(test_case.args) :] == test_case.expected_args + + +@pytest.mark.parametrize( + "test_case", + [ + DagsterCliArgumentErrorTestCase( + description="integer argument fails as string methods would", + args=("build", 3), + expected_error="'int' object has no attribute 'startswith'", + ), + ], + ids=lambda case: case.description, +) +def test_given_other_non_string_cli_arguments_when_starting_then_attribute_error_is_raised( + test_case: DagsterCliArgumentErrorTestCase, tmp_path: Path +) -> None: + project_dir: Path = tmp_path / "project" + project_dir.mkdir() + resource: SqlBuildCliResource = SqlBuildCliResource( + project_dir=str(project_dir), + sqb_command=write_fake_sqb_command(root=tmp_path), + ) + + with pytest.raises(AttributeError) as raised: + resource.cli(cast(list[str], list(test_case.args))) + + assert str(raised.value) == test_case.expected_error + + @pytest.mark.parametrize( "test_case", ( From 48cef6bcc4dde6b0c7b8ac49cfa9d7a182d1c99f Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Thu, 15 Oct 2026 17:14:51 +0100 Subject: [PATCH 15/48] refactor: reuse the shared integration loader name rule in project validation --- .../src/project_validation/_helpers/loaders.rs | 11 +---------- .../src/project_validation/constants.rs | 2 -- 2 files changed, 1 insertion(+), 12 deletions(-) diff --git a/crates/sqlbuild-discovery/src/project_validation/_helpers/loaders.rs b/crates/sqlbuild-discovery/src/project_validation/_helpers/loaders.rs index 47ebf99d0a..91ea0a3256 100644 --- a/crates/sqlbuild-discovery/src/project_validation/_helpers/loaders.rs +++ b/crates/sqlbuild-discovery/src/project_validation/_helpers/loaders.rs @@ -6,10 +6,10 @@ use crate::project_validation::_helpers::conflicts::{conflict, node_path, unique use crate::project_validation::_helpers::python_nodes::{ validate_check_dependencies, validate_loader_dependencies, validate_python_node_dependencies, }; -use crate::project_validation::constants::DLT_LOADER_KIND; use crate::project_validation::models::{ NamedPath, NodeFact, NodeKind, ProjectFacts, SourceFileFact, ValidationFailure, }; +use crate::yaml_entries::main::integration_loader_name::integration_loader_name; /// Validate loader and node names, node dependencies, then the loaders sources name. pub(crate) fn validate_loaders(facts: &ProjectFacts) -> Result<(), ValidationFailure> { @@ -64,15 +64,6 @@ fn source_loader_name_collisions( Ok(()) } -/// Python's `integration_loader_name`. -fn integration_loader_name(kind: &str, source_name: &str) -> String { - if kind == DLT_LOADER_KIND { - source_name.to_owned() - } else { - format!("{kind}__{source_name}") - } -} - fn source_loader_references( sources: &[SourceFileFact], loaders: &[&NodeFact], diff --git a/crates/sqlbuild-discovery/src/project_validation/constants.rs b/crates/sqlbuild-discovery/src/project_validation/constants.rs index 8ee3042fb3..6122e20503 100644 --- a/crates/sqlbuild-discovery/src/project_validation/constants.rs +++ b/crates/sqlbuild-discovery/src/project_validation/constants.rs @@ -1,7 +1,5 @@ //! Names and limits cross-file discovery validation checks against. -/// The loader kind whose generated loader takes the source's own name. -pub(crate) const DLT_LOADER_KIND: &str = "dlt"; /// The prefix discovery strips from model paths before matching path defaults. pub(crate) const MODELS_PREFIX: &str = "models/"; /// `str(Path(...).parent)` of a path with no folder. From 3f81565022a95b4274a6d0f4ef6fc72e6475f0e8 Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Thu, 15 Oct 2026 17:22:28 +0100 Subject: [PATCH 16/48] refactor: use the native query-diff artifact name grammar in the executor --- .../src/janitor_planning/main/mod.rs | 1 + .../main/query_diff_artifact_name.rs | 9 ++++++ .../tests/test_archive_names.rs | 32 ++++++++++++++++++- .../src/janitor_planning/tests/test_types.rs | 7 ++++ .../_helpers/janitor/janitor_planning.rs | 15 +++++++++ .../bindings/_helpers/janitor/registration.rs | 7 ++-- src/sqlbuild/_native.pyi | 2 ++ .../executor/diff/_helpers/query_artifacts.py | 29 +++++++---------- .../diff/classes/query_artifact_lifecycle.py | 4 +-- 9 files changed, 83 insertions(+), 23 deletions(-) create mode 100644 crates/sqlbuild-analysis/src/janitor_planning/main/query_diff_artifact_name.rs diff --git a/crates/sqlbuild-analysis/src/janitor_planning/main/mod.rs b/crates/sqlbuild-analysis/src/janitor_planning/main/mod.rs index 69c555cdcd..dd88d0ff6b 100644 --- a/crates/sqlbuild-analysis/src/janitor_planning/main/mod.rs +++ b/crates/sqlbuild-analysis/src/janitor_planning/main/mod.rs @@ -6,6 +6,7 @@ pub mod matching_exclude_pattern; pub mod migration_origin_keys; pub mod old_name_request_matches; pub mod parse_archive_name; +pub mod query_diff_artifact_name; pub mod query_diff_artifact_parts; pub mod resolve_retention_days; pub mod unaddressable_relation_reason; diff --git a/crates/sqlbuild-analysis/src/janitor_planning/main/query_diff_artifact_name.rs b/crates/sqlbuild-analysis/src/janitor_planning/main/query_diff_artifact_name.rs new file mode 100644 index 0000000000..0bccd96cd2 --- /dev/null +++ b/crates/sqlbuild-analysis/src/janitor_planning/main/query_diff_artifact_name.rs @@ -0,0 +1,9 @@ +//! The reserved relation name of one run's query-diff artifact. + +use crate::janitor_planning::constants::QUERY_DIFF_ARTIFACT_PREFIX; + +/// The artifact relation name for `run_id` and `side`, which `query_diff_artifact_parts` parses. +#[must_use] +pub fn query_diff_artifact_name(run_id: &str, side: &str) -> String { + format!("{QUERY_DIFF_ARTIFACT_PREFIX}{run_id}_{side}") +} diff --git a/crates/sqlbuild-analysis/src/janitor_planning/tests/test_archive_names.rs b/crates/sqlbuild-analysis/src/janitor_planning/tests/test_archive_names.rs index a322336757..9273ff4854 100644 --- a/crates/sqlbuild-analysis/src/janitor_planning/tests/test_archive_names.rs +++ b/crates/sqlbuild-analysis/src/janitor_planning/tests/test_archive_names.rs @@ -2,10 +2,12 @@ use crate::janitor_planning::main::build_archive_name::build_archive_name; use crate::janitor_planning::main::parse_archive_name::parse_archive_name; +use crate::janitor_planning::main::query_diff_artifact_name::query_diff_artifact_name; use crate::janitor_planning::main::query_diff_artifact_parts::query_diff_artifact_parts; use crate::janitor_planning::models::ArchiveTimestamp; use crate::janitor_planning::tests::test_types::{ - ArtifactNameTestCase, BuildArchiveNameTestCase, ParseArchiveNameTestCase, + ArtifactNameBuildTestCase, ArtifactNameTestCase, BuildArchiveNameTestCase, + ParseArchiveNameTestCase, }; const ARCHIVED_AT: ArchiveTimestamp = ArchiveTimestamp { @@ -148,3 +150,31 @@ fn given_relation_name_when_checking_query_diff_artifact_then_returns_run_and_si ); } } + +#[test] +fn given_run_and_side_when_building_query_diff_artifact_name_then_parses_back() { + let test_cases = [ + ArtifactNameBuildTestCase { + description: "generated run id round trips", + run_id: "20260115T123005Z_0123456789ab", + side: "right", + expected_parts: Some(("20260115T123005Z_0123456789ab", "right")), + }, + ArtifactNameBuildTestCase { + description: "free-form run id is not reserved", + run_id: "manual", + side: "left", + expected_parts: None, + }, + ]; + for test_case in test_cases { + assert_eq!( + query_diff_artifact_parts(&query_diff_artifact_name(test_case.run_id, test_case.side)), + test_case + .expected_parts + .map(|(run_id, side)| (run_id.to_owned(), side.to_owned())), + "{}", + test_case.description + ); + } +} diff --git a/crates/sqlbuild-analysis/src/janitor_planning/tests/test_types.rs b/crates/sqlbuild-analysis/src/janitor_planning/tests/test_types.rs index 1dde854e76..1ad44509e9 100644 --- a/crates/sqlbuild-analysis/src/janitor_planning/tests/test_types.rs +++ b/crates/sqlbuild-analysis/src/janitor_planning/tests/test_types.rs @@ -50,6 +50,13 @@ pub(super) struct ArtifactNameTestCase { pub expected_parts: Option<(&'static str, &'static str)>, } +pub(super) struct ArtifactNameBuildTestCase { + pub description: &'static str, + pub run_id: &'static str, + pub side: &'static str, + pub expected_parts: Option<(&'static str, &'static str)>, +} + pub(super) struct RetentionTestCase { pub description: &'static str, pub override_days: Option, diff --git a/crates/sqlbuild-python/src/bindings/_helpers/janitor/janitor_planning.rs b/crates/sqlbuild-python/src/bindings/_helpers/janitor/janitor_planning.rs index 6ddfb15348..b16318279e 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/janitor/janitor_planning.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/janitor/janitor_planning.rs @@ -7,6 +7,7 @@ use pyo3::pyfunction; use pyo3::types::{PyBool, PyDict, PyDictMethods}; use sqlbuild_analysis::janitor_planning::constants::OLD_NAME_VIEW_REASON; use sqlbuild_analysis::janitor_planning::main::build_archive_name::build_archive_name; +use sqlbuild_analysis::janitor_planning::main::query_diff_artifact_name::query_diff_artifact_name; use sqlbuild_analysis::janitor_planning::main::query_diff_artifact_parts::query_diff_artifact_parts; use crate::bindings::_helpers::compile_driver::python_calls::{KeywordArguments, imported, within}; @@ -399,3 +400,17 @@ pub(crate) fn build_janitor_archive_name( ) .map_err(|message| shared_error(archived_at.py(), message)) } + +/// The reserved relation name of one run's query-diff artifact side. +#[pyfunction] +#[pyo3(signature = (*, run_id, side))] +pub(crate) fn build_query_diff_artifact_name(run_id: &str, side: &str) -> String { + query_diff_artifact_name(run_id, side) +} + +/// The run id and folded side of a strict query-diff artifact name, or `None`. +#[pyfunction] +#[pyo3(signature = (name, /))] +pub(crate) fn parse_query_diff_artifact_name(name: &str) -> Option<(String, String)> { + query_diff_artifact_parts(name) +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/janitor/registration.rs b/crates/sqlbuild-python/src/bindings/_helpers/janitor/registration.rs index d68f7e7003..ae838bfb40 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/janitor/registration.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/janitor/registration.rs @@ -4,7 +4,8 @@ use pyo3::prelude::{Bound, PyModule, PyModuleMethods, PyResult}; use pyo3::wrap_pyfunction; use crate::bindings::_helpers::janitor::janitor_planning::{ - build_janitor_archive_name, build_janitor_plan_binding, + build_janitor_archive_name, build_janitor_plan_binding, build_query_diff_artifact_name, + parse_query_diff_artifact_name, }; use crate::bindings::_helpers::janitor::{janitor_command, janitor_plan}; @@ -12,5 +13,7 @@ pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { janitor_command::register(module)?; janitor_plan::register(module)?; module.add_function(wrap_pyfunction!(build_janitor_plan_binding, module)?)?; - module.add_function(wrap_pyfunction!(build_janitor_archive_name, module)?) + module.add_function(wrap_pyfunction!(build_janitor_archive_name, module)?)?; + module.add_function(wrap_pyfunction!(build_query_diff_artifact_name, module)?)?; + module.add_function(wrap_pyfunction!(parse_query_diff_artifact_name, module)?) } diff --git a/src/sqlbuild/_native.pyi b/src/sqlbuild/_native.pyi index ae9a723b5a..744e262d19 100644 --- a/src/sqlbuild/_native.pyi +++ b/src/sqlbuild/_native.pyi @@ -6200,6 +6200,8 @@ def build_janitor_plan( def build_janitor_archive_name( *, original_name: str, archived_at: datetime, identifier_limit: int, kind: str | None = None ) -> str: ... +def build_query_diff_artifact_name(*, run_id: str, side: str) -> str: ... +def parse_query_diff_artifact_name(name: str, /) -> tuple[str, str] | None: ... def render_janitor_completion( *, result: JanitorExecutionResult, use_color: bool = False ) -> str: ... diff --git a/src/sqlbuild/executor/diff/_helpers/query_artifacts.py b/src/sqlbuild/executor/diff/_helpers/query_artifacts.py index fe57c3fbbc..8eca81bac8 100644 --- a/src/sqlbuild/executor/diff/_helpers/query_artifacts.py +++ b/src/sqlbuild/executor/diff/_helpers/query_artifacts.py @@ -4,14 +4,15 @@ import hashlib import json -import re from datetime import UTC, datetime from typing import Any from sqlbuild._native import ( + build_query_diff_artifact_name, + compute_query_hash, Fingerprint, FingerprintSet, - compute_query_hash, + parse_query_diff_artifact_name, read_latest_fingerprints, write_fingerprint, ) @@ -30,10 +31,6 @@ QueryDiffArtifactInspection, ) -_ARTIFACT_PATTERN: re.Pattern[str] = re.compile( - r"^__sqlbuild_query_diff_(?P\d{8}T\d{6}Z_[0-9a-f]{12})_(?Pleft|right)$", - flags=re.IGNORECASE, -) _ARTIFACT_VERSION: int = 1 @@ -47,7 +44,7 @@ def build_query_diff_artifact( ) -> QueryDiffArtifact: """Build one collision-resistant artifact identity in the active target schema.""" - name: str = f"__sqlbuild_query_diff_{run_id}_{side}" + name: str = build_query_diff_artifact_name(run_id=run_id, side=side) relation: str | None = adapter.render_qualified_name( database=database, schema=schema, @@ -66,12 +63,6 @@ def build_query_diff_artifact( ) -def is_query_diff_artifact_name(name: str) -> bool: - """Return whether a relation uses the strict reserved query-diff name format.""" - - return _ARTIFACT_PATTERN.fullmatch(name) is not None - - def materialize_query_diff_artifact( *, adapter: BaseAdapter, @@ -201,7 +192,9 @@ def inspect_query_diff_artifacts( schemas=(schema,), ) candidates: tuple[RelationInfo, ...] = tuple( - relation for relation in relations if _ARTIFACT_PATTERN.fullmatch(relation.name) + relation + for relation in relations + if parse_query_diff_artifact_name(relation.name) is not None ) if not candidates: return QueryDiffArtifactInspection() @@ -232,13 +225,13 @@ def inspect_query_diff_artifacts( untracked: list[RelationInfo] = [] for relation in candidates: fingerprint: Fingerprint | None = owned.get(relation.name.lower()) - match: re.Match[str] | None = _ARTIFACT_PATTERN.fullmatch(relation.name) - if match is None: + parts: tuple[str, str] | None = parse_query_diff_artifact_name(relation.name) + if parts is None: continue artifact: QueryDiffArtifact = build_query_diff_artifact( adapter=adapter, - run_id=match.group("run_id"), - side=match.group("side").lower(), + run_id=parts[0], + side=parts[1], database=database, schema=schema, ) diff --git a/src/sqlbuild/executor/diff/classes/query_artifact_lifecycle.py b/src/sqlbuild/executor/diff/classes/query_artifact_lifecycle.py index 73f32405da..0ef13a9526 100644 --- a/src/sqlbuild/executor/diff/classes/query_artifact_lifecycle.py +++ b/src/sqlbuild/executor/diff/classes/query_artifact_lifecycle.py @@ -5,13 +5,13 @@ from datetime import datetime from typing import Any +from sqlbuild._native import parse_query_diff_artifact_name from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter from sqlbuild.executor.diff._helpers.query_artifacts import ( build_query_diff_artifact, cleanup_expired_query_diff_artifacts, cleanup_query_diff_artifact, inspect_query_diff_artifacts, - is_query_diff_artifact_name, materialize_query_diff_artifact, ) from sqlbuild.executor.diff.models import ( @@ -102,4 +102,4 @@ def inspect( @staticmethod def is_artifact_name(name: str) -> bool: - return is_query_diff_artifact_name(name) + return parse_query_diff_artifact_name(name) is not None From 22b61f946cfd844b5862f4bf06de1929149b5e94 Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Thu, 15 Oct 2026 18:01:49 +0100 Subject: [PATCH 17/48] refactor: classify executed statements for relation cache invalidation natively --- .../bindings/_helpers/sqltext/ddl_effects.rs | 73 ++++ .../src/bindings/_helpers/sqltext/mod.rs | 1 + .../bindings/_helpers/sqltext/registration.rs | 3 +- .../src/ddl_effects/_helpers/code_text.rs | 114 ++++++ .../src/ddl_effects/_helpers/matching.rs | 86 +++++ .../src/ddl_effects/_helpers/mod.rs | 4 + .../src/ddl_effects/_helpers/names.rs | 128 +++++++ .../src/ddl_effects/_helpers/relations.rs | 227 ++++++++++++ .../src/ddl_effects/constants.rs | 135 +++++++ .../src/ddl_effects/main/mod.rs | 3 + .../main/qualified_relation_name_key.rs | 15 + .../src/ddl_effects/main/relation_name_key.rs | 9 + .../main/statement_metadata_effect.rs | 37 ++ .../sqlbuild-sqltext/src/ddl_effects/mod.rs | 7 + .../src/ddl_effects/models.rs | 14 + .../sqlbuild-sqltext/src/ddl_effects/tests.rs | 4 + .../src/ddl_effects/tests/test_ddl_effects.rs | 223 ++++++++++++ .../src/ddl_effects/tests/test_types.rs | 13 + crates/sqlbuild-sqltext/src/lib.rs | 1 + python-boundary.toml | 2 +- src/sqlbuild/_native.pyi | 5 +- .../adapter/relations/_helpers/ddl_effects.py | 343 ------------------ .../classes/relation_metadata_cache.py | 2 +- src/sqlbuild/adapter/relations/constants.py | 5 - .../test_adversarial_input_runtime.py | 2 +- .../test_statement_metadata_effect.py | 2 +- 26 files changed, 1104 insertions(+), 354 deletions(-) create mode 100644 crates/sqlbuild-python/src/bindings/_helpers/sqltext/ddl_effects.rs create mode 100644 crates/sqlbuild-sqltext/src/ddl_effects/_helpers/code_text.rs create mode 100644 crates/sqlbuild-sqltext/src/ddl_effects/_helpers/matching.rs create mode 100644 crates/sqlbuild-sqltext/src/ddl_effects/_helpers/mod.rs create mode 100644 crates/sqlbuild-sqltext/src/ddl_effects/_helpers/names.rs create mode 100644 crates/sqlbuild-sqltext/src/ddl_effects/_helpers/relations.rs create mode 100644 crates/sqlbuild-sqltext/src/ddl_effects/constants.rs create mode 100644 crates/sqlbuild-sqltext/src/ddl_effects/main/mod.rs create mode 100644 crates/sqlbuild-sqltext/src/ddl_effects/main/qualified_relation_name_key.rs create mode 100644 crates/sqlbuild-sqltext/src/ddl_effects/main/relation_name_key.rs create mode 100644 crates/sqlbuild-sqltext/src/ddl_effects/main/statement_metadata_effect.rs create mode 100644 crates/sqlbuild-sqltext/src/ddl_effects/mod.rs create mode 100644 crates/sqlbuild-sqltext/src/ddl_effects/models.rs create mode 100644 crates/sqlbuild-sqltext/src/ddl_effects/tests.rs create mode 100644 crates/sqlbuild-sqltext/src/ddl_effects/tests/test_ddl_effects.rs create mode 100644 crates/sqlbuild-sqltext/src/ddl_effects/tests/test_types.rs delete mode 100644 src/sqlbuild/adapter/relations/_helpers/ddl_effects.py diff --git a/crates/sqlbuild-python/src/bindings/_helpers/sqltext/ddl_effects.rs b/crates/sqlbuild-python/src/bindings/_helpers/sqltext/ddl_effects.rs new file mode 100644 index 0000000000..3b48467554 --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/sqltext/ddl_effects.rs @@ -0,0 +1,73 @@ +//! Executed-statement effects on the executor's cached relation metadata. + +use pyo3::prelude::{Bound, Py, PyAny, PyModule, PyModuleMethods, PyResult, Python}; +use pyo3::sync::PyOnceLock; +use pyo3::types::{PyBool, PyFrozenSet}; +use pyo3::{pyfunction, wrap_pyfunction}; +use sqlbuild_sqltext::ddl_effects::main::qualified_relation_name_key::qualified_relation_name_key as native_qualified_relation_name_key; +use sqlbuild_sqltext::ddl_effects::main::relation_name_key::relation_name_key as native_relation_name_key; +use sqlbuild_sqltext::ddl_effects::main::statement_metadata_effect::statement_metadata_effect as native_statement_metadata_effect; +use sqlbuild_sqltext::ddl_effects::models::StatementMetadataEffect; + +use crate::bindings::_helpers::compile_driver::python_calls::{call_with, imported}; + +const EFFECT_MODEL: (&str, &str) = ( + "sqlbuild.adapter.relations.models", + "StatementMetadataEffect", +); + +static EFFECT_CLASS: PyOnceLock> = PyOnceLock::new(); + +/// Which cached relation names `sql` may have changed, as a `StatementMetadataEffect`. +#[pyfunction] +#[pyo3(signature = (sql, /))] +fn statement_metadata_effect<'py>(py: Python<'py>, sql: &str) -> PyResult> { + let effect: StatementMetadataEffect = py.detach(|| native_statement_metadata_effect(sql)); + let class: &Bound<'py, PyAny> = EFFECT_CLASS + .get_or_try_init(py, || { + imported(py, EFFECT_MODEL.0, EFFECT_MODEL.1).map(Bound::unbind) + })? + .bind(py); + call_with( + class, + &[ + ( + "relation_names", + PyFrozenSet::new(py, &effect.relation_names)?.into_any(), + ), + ( + "invalidates_all", + PyBool::new(py, effect.invalidates_all) + .to_owned() + .into_any(), + ), + ( + "ends_transaction", + PyBool::new(py, effect.ends_transaction) + .to_owned() + .into_any(), + ), + ], + ) +} + +/// The invalidation key for one unquoted or quoted unqualified relation name. +#[pyfunction] +#[pyo3(signature = (name, /))] +fn relation_name_key(name: &str) -> String { + native_relation_name_key(name) +} + +/// The key for the last part of a qualified name, or `None` when it cannot be parsed. +#[pyfunction] +#[pyo3(signature = (qualified, /))] +fn qualified_relation_name_key(qualified: &str) -> Option { + native_qualified_relation_name_key(qualified) +} + +pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { + module.add_function(wrap_pyfunction!(statement_metadata_effect, module)?)?; + module.add_function(wrap_pyfunction!(relation_name_key, module)?)?; + module.add_function(wrap_pyfunction!(qualified_relation_name_key, module)?)?; + Ok(()) +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/sqltext/mod.rs b/crates/sqlbuild-python/src/bindings/_helpers/sqltext/mod.rs index 52ccc56e39..670af7fcaa 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/sqltext/mod.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/sqltext/mod.rs @@ -1,5 +1,6 @@ pub(crate) mod authored_values; pub(crate) mod code_scanner; +pub(crate) mod ddl_effects; pub(crate) mod declaration_references; pub(crate) mod lexical_syntax; pub(crate) mod model_headers; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/sqltext/registration.rs b/crates/sqlbuild-python/src/bindings/_helpers/sqltext/registration.rs index 2b9fdf77b5..655c754951 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/sqltext/registration.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/sqltext/registration.rs @@ -3,7 +3,7 @@ use pyo3::prelude::{Bound, PyModule, PyResult}; use crate::bindings::_helpers::sqltext::{ - code_scanner, model_headers, placeholders, sql_references, + code_scanner, ddl_effects, model_headers, placeholders, sql_references, }; pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { @@ -11,5 +11,6 @@ pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { code_scanner::register(module)?; sql_references::register(module)?; placeholders::register(module)?; + ddl_effects::register(module)?; Ok(()) } diff --git a/crates/sqlbuild-sqltext/src/ddl_effects/_helpers/code_text.rs b/crates/sqlbuild-sqltext/src/ddl_effects/_helpers/code_text.rs new file mode 100644 index 0000000000..a9cfd1dbe1 --- /dev/null +++ b/crates/sqlbuild-sqltext/src/ddl_effects/_helpers/code_text.rs @@ -0,0 +1,114 @@ +//! SQL code with comments blanked and literals emptied, under one escape and bracket reading. + +/// How ambiguous lexemes are read. +#[derive(Clone, Copy)] +pub(crate) struct LexicalMode { + /// A backslash escapes the next character inside single-quoted literals. + pub(crate) backslash_escapes: bool, + /// `[...]` delimits a quoted identifier rather than plain code. + pub(crate) bracket_identifiers: bool, +} + +/// The next lexeme start at or after `position`: `--`, `/*`, a quote, `$` or `[`. +fn lexeme_start(bytes: &[u8], position: usize) -> Option { + (position..bytes.len()).find(|index| match bytes[*index] { + b'\'' | b'"' | b'`' | b'$' | b'[' => true, + b'-' => bytes.get(index + 1) == Some(&b'-'), + b'/' => bytes.get(index + 1) == Some(&b'*'), + _ => false, + }) +} + +/// Python's `text.find(needle, position)` as a byte offset. +fn find(text: &str, needle: &str, position: usize) -> Option { + text.get(position..)? + .find(needle) + .map(|offset| position + offset) +} + +/// The closing quote of a single-quoted literal opened at `begin`. +fn literal_end(text: &str, begin: usize, backslash_escapes: bool) -> Option { + let bytes: &[u8] = text.as_bytes(); + let mut position: usize = begin + 1; + loop { + let rest: &[u8] = bytes.get(position..)?; + let offset: usize = rest + .iter() + .position(|byte| *byte == b'\'' || (backslash_escapes && *byte == b'\\'))?; + let stop: usize = position + offset; + if bytes[stop] == b'\\' { + let escaped: char = text[stop + 1..].chars().next()?; + position = stop + 1 + escaped.len_utf8(); + continue; + } + if bytes.get(stop + 1) == Some(&b'\'') { + position = stop + 2; + continue; + } + return Some(stop); + } +} + +/// The closing `quote` of quoted text opened at `begin`; doubled quotes escape except `]`. +fn quoted_end(text: &str, begin: usize, quote: u8) -> Option { + let bytes: &[u8] = text.as_bytes(); + let mut position: usize = begin + 1; + loop { + let offset: usize = bytes + .get(position..)? + .iter() + .position(|byte| *byte == quote)?; + let end: usize = position + offset; + if quote == b']' || bytes.get(end + 1) != Some("e) { + return Some(end); + } + position = end + 2; + } +} + +/// The end of a `$tag$` opener at `begin`, or `None` when `$` starts no tag. +fn dollar_tag_end(bytes: &[u8], begin: usize) -> Option { + let length: usize = bytes[begin + 1..] + .iter() + .take_while(|byte| byte.is_ascii_alphabetic() || **byte == b'_') + .count(); + let close: usize = begin + 1 + length; + (bytes.get(close) == Some(&b'$')).then_some(close + 1) +} + +/// One lexeme's code replacement and the position after it; `None` when it never closes. +fn read_lexeme(text: &str, begin: usize, mode: LexicalMode) -> Option<(&str, usize)> { + let bytes: &[u8] = text.as_bytes(); + match bytes[begin] { + b'-' => Some((" ", find(text, "\n", begin).unwrap_or(text.len()))), + b'/' => find(text, "*/", begin + 2).map(|end| (" ", end + 2)), + b'\'' => literal_end(text, begin, mode.backslash_escapes).map(|end| ("''", end + 1)), + b'$' => match dollar_tag_end(bytes, begin) { + None => Some(("$", begin + 1)), + Some(tag_end) => { + let tag: &str = &text[begin..tag_end]; + find(text, tag, tag_end).map(|end| ("''", end + tag.len())) + } + }, + b'[' if !mode.bracket_identifiers => Some(("[", begin + 1)), + quote => { + let closing: u8 = if quote == b'[' { b']' } else { quote }; + quoted_end(text, begin, closing).map(|end| (&text[begin..=end], end + 1)) + } + } +} + +/// `sql` with comments as spaces and literals as `''`, or `None` when a lexeme never closes. +pub(crate) fn code_text(sql: &str, mode: LexicalMode) -> Option { + let bytes: &[u8] = sql.as_bytes(); + let mut code: String = String::with_capacity(sql.len()); + let mut position: usize = 0; + while let Some(start) = lexeme_start(bytes, position) { + code.push_str(&sql[position..start]); + let (replacement, next) = read_lexeme(sql, start, mode)?; + code.push_str(replacement); + position = next; + } + code.push_str(&sql[position..]); + Some(code) +} diff --git a/crates/sqlbuild-sqltext/src/ddl_effects/_helpers/matching.rs b/crates/sqlbuild-sqltext/src/ddl_effects/_helpers/matching.rs new file mode 100644 index 0000000000..216e517e53 --- /dev/null +++ b/crates/sqlbuild-sqltext/src/ddl_effects/_helpers/matching.rs @@ -0,0 +1,86 @@ +//! Anchored matching primitives with Python `re` whitespace and word-boundary semantics. + +use sqlbuild_core::text::main::python_isalnum::python_isalnum; +use sqlbuild_core::text::main::python_isspace::python_isspace; + +/// Python's `\w`: alphanumeric or `_`. +fn is_word(character: char) -> bool { + character == '_' || python_isalnum(character) +} + +/// The position after any whitespace at `position` (`\s*+`). +pub(crate) fn skip_whitespace(text: &str, position: usize) -> usize { + let rest: &str = &text[position..]; + position + (rest.len() - rest.trim_start_matches(python_isspace).len()) +} + +/// The position after at least one whitespace character at `position` (`\s++`). +pub(crate) fn whitespace(text: &str, position: usize) -> Option { + let end: usize = skip_whitespace(text, position); + (end > position).then_some(end) +} + +/// Whether only whitespace remains from `position` (`\s*+$`). +pub(crate) fn only_whitespace(text: &str, position: usize) -> bool { + skip_whitespace(text, position) == text.len() +} + +/// Whether `position` is a word boundary (`\b`). +pub(crate) fn word_boundary(text: &str, position: usize) -> bool { + let before: bool = text[..position].chars().next_back().is_some_and(is_word); + let after: bool = text[position..].chars().next().is_some_and(is_word); + before != after +} + +/// The position after an ASCII case-insensitive `word` at `position`. +pub(crate) fn word(text: &str, position: usize, word: &str) -> Option { + let end: usize = position + word.len(); + text.as_bytes() + .get(position..end) + .filter(|candidate| candidate.eq_ignore_ascii_case(word.as_bytes())) + .map(|_| end) +} + +/// The position after `words` separated by whitespace (`A\s++B`). +pub(crate) fn phrase(text: &str, position: usize, words: &[&str]) -> Option { + let mut end: usize = position; + for (index, expected) in words.iter().enumerate() { + if index > 0 { + end = whitespace(text, end)?; + } + end = word(text, end, expected)?; + } + Some(end) +} + +/// The position after the first of `phrases` that matches and is followed by whitespace. +pub(crate) fn phrase_then_whitespace( + text: &str, + position: usize, + phrases: &[&[&str]], +) -> Option { + phrases + .iter() + .find_map(|words| whitespace(text, phrase(text, position, words)?)) +} + +/// The position after `words` when it ends at a word boundary (`A\b`). +fn bounded_words(text: &str, position: usize, words: &[&str]) -> Option { + phrase(text, position, words).filter(|end| word_boundary(text, *end)) +} + +/// The position after the first of `phrases` that ends at a word boundary (`(?:A|B)\b`). +pub(crate) fn bounded_phrase(text: &str, position: usize, phrases: &[&[&str]]) -> Option { + phrases + .iter() + .find_map(|words| bounded_words(text, position, words)) +} + +/// The position after any repetitions of `phrases`, each followed by whitespace (`(?:..\s++)*+`). +pub(crate) fn repeated_modifiers(text: &str, position: usize, phrases: &[&[&str]]) -> usize { + let mut end: usize = position; + while let Some(next) = phrase_then_whitespace(text, end, phrases) { + end = next; + } + end +} diff --git a/crates/sqlbuild-sqltext/src/ddl_effects/_helpers/mod.rs b/crates/sqlbuild-sqltext/src/ddl_effects/_helpers/mod.rs new file mode 100644 index 0000000000..a979b2d312 --- /dev/null +++ b/crates/sqlbuild-sqltext/src/ddl_effects/_helpers/mod.rs @@ -0,0 +1,4 @@ +pub(crate) mod code_text; +pub(crate) mod matching; +pub(crate) mod names; +pub(crate) mod relations; diff --git a/crates/sqlbuild-sqltext/src/ddl_effects/_helpers/names.rs b/crates/sqlbuild-sqltext/src/ddl_effects/_helpers/names.rs new file mode 100644 index 0000000000..d911ba183e --- /dev/null +++ b/crates/sqlbuild-sqltext/src/ddl_effects/_helpers/names.rs @@ -0,0 +1,128 @@ +//! Quoted and unquoted relation name parts and their invalidation keys. + +use sqlbuild_core::text::main::python_casefold::python_casefold; + +use crate::ddl_effects::_helpers::matching::skip_whitespace; +use crate::ddl_effects::constants::QUOTED_IDENTIFIER_MIN_LENGTH; + +/// Which characters an unquoted name part may contain. +#[derive(Clone, Copy)] +pub(crate) enum UnquotedPart { + /// SQL text: starts with a letter, digit or `_`, then may also contain `$`. + Statement, + /// Qualified names from callers: letters, digits, `_`, `$` and `-`. + Loose, +} + +/// A qualified name: where it ends and its last part. +pub(crate) struct QualifiedName<'a> { + pub(crate) end: usize, + pub(crate) last_part: &'a str, +} + +/// The end of the text after `open` up to and including the next `close`. +fn closed_by(text: &str, open: usize, close: u8) -> Option { + let rest: &[u8] = text.as_bytes().get(open + 1..)?; + rest.iter() + .position(|byte| *byte == close) + .map(|offset| open + offset + 2) +} + +/// The end of a double-quoted part whose doubled quotes are escapes. +fn double_quoted(text: &str, open: usize) -> Option { + let bytes: &[u8] = text.as_bytes(); + let mut position: usize = open + 1; + loop { + match bytes.get(position) { + None => return None, + Some(b'"') if bytes.get(position + 1) == Some(&b'"') => position += 2, + Some(b'"') => return Some(position + 1), + Some(_) => position += 1, + } + } +} + +fn unquoted(text: &str, start: usize, kind: UnquotedPart) -> Option { + let bytes: &[u8] = text.as_bytes(); + let first_allowed = |byte: u8| match kind { + UnquotedPart::Statement => byte.is_ascii_alphanumeric() || byte == b'_', + UnquotedPart::Loose => byte.is_ascii_alphanumeric() || matches!(byte, b'_' | b'$' | b'-'), + }; + let next_allowed = |byte: u8| match kind { + UnquotedPart::Statement => byte.is_ascii_alphanumeric() || matches!(byte, b'_' | b'$'), + UnquotedPart::Loose => first_allowed(byte), + }; + if !first_allowed(*bytes.get(start)?) { + return None; + } + let length: usize = bytes[start + 1..] + .iter() + .take_while(|byte| next_allowed(**byte)) + .count(); + Some(start + 1 + length) +} + +/// The end of one name part starting at `start`. +pub(crate) fn name_part(text: &str, start: usize, kind: UnquotedPart) -> Option { + match text.as_bytes().get(start)? { + b'"' => double_quoted(text, start), + b'`' => closed_by(text, start, b'`'), + b'[' => closed_by(text, start, b']'), + _ => unquoted(text, start, kind), + } +} + +/// One or more parts separated by dots with optional whitespace around them. +pub(crate) fn qualified_name( + text: &str, + start: usize, + kind: UnquotedPart, +) -> Option> { + let mut part_start: usize = start; + let mut end: usize = name_part(text, start, kind)?; + loop { + let dot: usize = skip_whitespace(text, end); + if text.as_bytes().get(dot) != Some(&b'.') { + break; + } + let next_start: usize = skip_whitespace(text, dot + 1); + let Some(next_end) = name_part(text, next_start, kind) else { + break; + }; + part_start = next_start; + end = next_end; + } + Some(QualifiedName { + end, + last_part: &text[part_start..end], + }) +} + +/// A part without its surrounding identifier quotes, unescaping doubled double quotes. +fn strip_identifier_quotes(part: &str) -> String { + let bytes: &[u8] = part.as_bytes(); + if bytes.len() < QUOTED_IDENTIFIER_MIN_LENGTH { + return part.to_owned(); + } + let inner = || &part[1..part.len() - 1]; + match (bytes[0], bytes[bytes.len() - 1]) { + (b'"', b'"') => inner().replace("\"\"", "\""), + (b'`', b'`') | (b'[', b']') => inner().to_owned(), + _ => part.to_owned(), + } +} + +/// The invalidation key of one unquoted or quoted unqualified relation name. +pub(crate) fn name_key(name: &str) -> String { + python_casefold(&strip_identifier_quotes(name)) +} + +/// The key of a qualified name's last part; a backticked part keys its last dotted segment. +pub(crate) fn part_name_key(part: &str) -> String { + if !part.starts_with('`') { + return name_key(part); + } + let stripped: String = strip_identifier_quotes(part); + let segment: &str = stripped.rsplit('.').next().unwrap_or_default(); + python_casefold(segment) +} diff --git a/crates/sqlbuild-sqltext/src/ddl_effects/_helpers/relations.rs b/crates/sqlbuild-sqltext/src/ddl_effects/_helpers/relations.rs new file mode 100644 index 0000000000..c33a15d955 --- /dev/null +++ b/crates/sqlbuild-sqltext/src/ddl_effects/_helpers/relations.rs @@ -0,0 +1,227 @@ +//! Relation names a statement's code text creates, drops, alters or loads. + +use std::collections::BTreeSet; + +use sqlbuild_core::text::main::python_isspace::python_isspace; +use sqlbuild_core::text::main::python_strip::python_strip; + +use crate::ddl_effects::_helpers::matching::{ + bounded_phrase, only_whitespace, phrase, phrase_then_whitespace, repeated_modifiers, + skip_whitespace, whitespace, word, word_boundary, +}; +use crate::ddl_effects::_helpers::names::{UnquotedPart, part_name_key, qualified_name}; +use crate::ddl_effects::constants::{ + ALTER_FOLLOWING_WORDS, COPY_INTO, CREATE_FOLLOWING_WORDS, CREATE_SCHEMA_IF_NOT_EXISTS, + DROP_FOLLOWING_WORDS, FROM_WORD, IF_EXISTS, IF_NOT_EXISTS, INTO, NEUTRAL_OBJECT_KINDS, + NEUTRAL_OBJECT_LEADS, NEUTRAL_OBJECT_MODIFIERS, OR_REPLACE, QUERY_STATEMENTS, + READ_OR_DATA_STATEMENTS, RELATION_DDL_STATEMENTS, RELATION_KIND_MODIFIERS, RELATION_KINDS, + RENAMING_ALTERS, STATEMENT_SEPARATOR, TRANSACTION_END_STATEMENTS, +}; +use crate::ddl_effects::models::StatementMetadataEffect; + +/// Which statement form named the relation, deciding what may follow its name. +#[derive(Clone, Copy, PartialEq, Eq)] +enum Form { + Create, + Drop, + Alter, + CopyInto, + SelectInto, + SecondRelation, +} + +/// A matched relation name's key and the text after the name. +struct NamedRelation<'a> { + key: String, + rest: &'a str, +} + +/// The effect of every relation in the statement possibly changing. +pub(crate) fn all_relations() -> StatementMetadataEffect { + StatementMetadataEffect { + invalidates_all: true, + ..StatementMetadataEffect::default() + } +} + +/// A qualified name at `start` followed by whitespace, `(` or the end. +fn named_relation(text: &str, start: usize) -> Option> { + let name = qualified_name(text, start, UnquotedPart::Statement)?; + let rest: &str = &text[name.end..]; + let delimited: bool = rest + .chars() + .next() + .is_none_or(|next| next == '(' || python_isspace(next)); + delimited.then(|| NamedRelation { + key: part_name_key(name.last_part), + rest, + }) +} + +/// `LEAD\s++` modifiers `(TABLE|VIEW)\s++` and an optional `guard\s++` before the name. +fn guarded_relation<'a>(text: &'a str, lead: &str, guard: &[&str]) -> Option> { + let after_lead: usize = whitespace(text, word(text, 0, lead)?)?; + let after_modifiers: usize = repeated_modifiers(text, after_lead, RELATION_KIND_MODIFIERS); + let name_start: usize = phrase_then_whitespace(text, after_modifiers, RELATION_KINDS)?; + phrase(text, name_start, guard) + .and_then(|end| whitespace(text, end)) + .and_then(|guarded| named_relation(text, guarded)) + .or_else(|| named_relation(text, name_start)) +} + +fn copy_into_relation(text: &str) -> Option> { + named_relation(text, whitespace(text, phrase(text, 0, COPY_INTO)?)?) +} + +fn second_relation(rest: &str) -> Option> { + let clause: usize = whitespace(rest, 0)?; + named_relation(rest, phrase_then_whitespace(rest, clause, RENAMING_ALTERS)?) +} + +/// The first `\bINTO\s++name` in the statement. +fn select_into_relation(text: &str) -> Option> { + text.char_indices() + .filter(|(index, _)| word(text, *index, INTO).is_some() && word_boundary(text, *index)) + .find_map(|(index, _)| named_relation(text, whitespace(text, index + INTO.len())?)) +} + +fn has_into_keyword(text: &str) -> bool { + text.char_indices().any(|(index, _)| { + word(text, index, INTO) + .is_some_and(|end| word_boundary(text, index) && word_boundary(text, end)) + }) +} + +/// Whether `rest` is text the form allows after the relation name. +fn allowed_following(form: Form, rest: &str) -> bool { + match form { + Form::Create => { + let start: usize = skip_whitespace(rest, 0); + start == rest.len() + || rest[start..].starts_with('(') + || bounded_phrase(rest, start, CREATE_FOLLOWING_WORDS).is_some() + } + Form::Drop => { + let start: usize = skip_whitespace(rest, 0); + let end: usize = DROP_FOLLOWING_WORDS + .iter() + .find_map(|words| phrase(rest, start, words)) + .unwrap_or(start); + only_whitespace(rest, end) + } + Form::Alter => whitespace(rest, 0) + .is_some_and(|start| bounded_phrase(rest, start, ALTER_FOLLOWING_WORDS).is_some()), + Form::CopyInto | Form::SelectInto => whitespace(rest, 0) + .is_some_and(|start| bounded_phrase(rest, start, &[FROM_WORD]).is_some()), + Form::SecondRelation => only_whitespace(rest, 0), + } +} + +/// The relation keys a matched name names, or `None` when what follows is ambiguous. +fn unambiguous_names(form: Form, relation: Option>) -> Option> { + let relation: NamedRelation<'_> = relation?; + if !allowed_following(form, relation.rest) { + return None; + } + let renaming: bool = form == Form::Alter + && whitespace(relation.rest, 0) + .is_some_and(|start| bounded_phrase(relation.rest, start, RENAMING_ALTERS).is_some()); + let mut names: BTreeSet = BTreeSet::from([relation.key]); + if renaming { + names.extend(unambiguous_names( + Form::SecondRelation, + second_relation(relation.rest), + )?); + } + Some(names) +} + +fn named_effect(names: BTreeSet) -> StatementMetadataEffect { + StatementMetadataEffect { + relation_names: names, + ..StatementMetadataEffect::default() + } +} + +fn select_into_effect(statement: &str) -> StatementMetadataEffect { + if !has_into_keyword(statement) { + return StatementMetadataEffect::default(); + } + unambiguous_names(Form::SelectInto, select_into_relation(statement)) + .map_or_else(all_relations, named_effect) +} + +fn relation_ddl_effect(statement: &str) -> StatementMetadataEffect { + [ + ( + Form::Create, + guarded_relation(statement, "CREATE", IF_NOT_EXISTS), + ), + (Form::Drop, guarded_relation(statement, "DROP", IF_EXISTS)), + (Form::Alter, guarded_relation(statement, "ALTER", IF_EXISTS)), + (Form::CopyInto, copy_into_relation(statement)), + ] + .into_iter() + .find_map(|(form, relation)| unambiguous_names(form, relation)) + .map_or_else(all_relations, named_effect) +} + +fn creates_schema_if_missing(statement: &str) -> bool { + let Some(name_start) = phrase(statement, 0, CREATE_SCHEMA_IF_NOT_EXISTS) + .and_then(|end| whitespace(statement, end)) + else { + return false; + }; + let name_length: usize = statement[name_start..] + .chars() + .take_while(|character| *character != STATEMENT_SEPARATOR && !python_isspace(*character)) + .map(char::len_utf8) + .sum(); + name_length > 0 && only_whitespace(statement, name_start + name_length) +} + +fn is_neutral_object(statement: &str) -> bool { + let Some(after_lead) = phrase_then_whitespace(statement, 0, NEUTRAL_OBJECT_LEADS) else { + return false; + }; + let after_replace: usize = + phrase_then_whitespace(statement, after_lead, &[OR_REPLACE]).unwrap_or(after_lead); + let kind_start: usize = repeated_modifiers(statement, after_replace, NEUTRAL_OBJECT_MODIFIERS); + bounded_phrase(statement, kind_start, NEUTRAL_OBJECT_KINDS).is_some() +} + +/// The effect of one reading of the statement's code text; `None` code changes everything. +pub(crate) fn code_effect(code: Option<&str>) -> StatementMetadataEffect { + let Some(code) = code else { + return all_relations(); + }; + let statement: &str = python_strip(python_strip(code).trim_end_matches(STATEMENT_SEPARATOR)) + .trim_start_matches('('); + let first_length: usize = statement + .bytes() + .take_while(u8::is_ascii_alphabetic) + .count(); + if first_length == 0 || statement.contains(STATEMENT_SEPARATOR) { + return all_relations(); + } + let keyword: String = statement[..first_length].to_ascii_uppercase(); + if QUERY_STATEMENTS.contains(&keyword.as_str()) { + return select_into_effect(statement); + } + if READ_OR_DATA_STATEMENTS.contains(&keyword.as_str()) { + return StatementMetadataEffect::default(); + } + if TRANSACTION_END_STATEMENTS.contains(&keyword.as_str()) { + return StatementMetadataEffect { + ends_transaction: true, + ..StatementMetadataEffect::default() + }; + } + if !RELATION_DDL_STATEMENTS.contains(&keyword.as_str()) { + return all_relations(); + } + if creates_schema_if_missing(statement) || is_neutral_object(statement) { + return StatementMetadataEffect::default(); + } + relation_ddl_effect(statement) +} diff --git a/crates/sqlbuild-sqltext/src/ddl_effects/constants.rs b/crates/sqlbuild-sqltext/src/ddl_effects/constants.rs new file mode 100644 index 0000000000..5c5c822933 --- /dev/null +++ b/crates/sqlbuild-sqltext/src/ddl_effects/constants.rs @@ -0,0 +1,135 @@ +//! Statement keywords that decide how executed SQL affects cached relation metadata. + +/// Statements that only read, unless they select into a new table. +pub(crate) const QUERY_STATEMENTS: &[&str] = &["SELECT", "WITH"]; +/// Statements that read, change data or session state, but never relation existence or columns. +pub(crate) const READ_OR_DATA_STATEMENTS: &[&str] = &[ + "VALUES", + "SHOW", + "DESCRIBE", + "DESC", + "EXPLAIN", + "INSERT", + "UPDATE", + "DELETE", + "MERGE", + "TRUNCATE", + "BEGIN", + "START", + "SAVEPOINT", + "RELEASE", + "SET", + "UNSET", + "GRANT", + "REVOKE", + "COMMENT", + "PUT", + "GET", + "LIST", + "LS", + "REMOVE", + "RM", + "ANALYZE", +]; +/// Statements that end the current transaction. +pub(crate) const TRANSACTION_END_STATEMENTS: &[&str] = &["COMMIT", "ROLLBACK", "END", "ABORT"]; +/// Statements whose relation name is read from the statement. +pub(crate) const RELATION_DDL_STATEMENTS: &[&str] = &["CREATE", "DROP", "ALTER", "COPY"]; +/// Modifier phrases allowed between `CREATE`/`DROP`/`ALTER` and `TABLE`/`VIEW`. +pub(crate) const RELATION_KIND_MODIFIERS: &[&[&str]] = &[ + &["OR", "REPLACE"], + &["LOCAL"], + &["GLOBAL"], + &["TEMP"], + &["TEMPORARY"], + &["TRANSIENT"], + &["VOLATILE"], + &["SECURE"], + &["RECURSIVE"], + &["MATERIALIZED"], + &["EXTERNAL"], + &["DYNAMIC"], + &["ICEBERG"], + &["HYBRID"], + &["UNLOGGED"], + &["FOREIGN"], + &["STREAMING"], +]; +/// Relation kinds whose name follows the modifiers. +pub(crate) const RELATION_KINDS: &[&[&str]] = &[&["TABLE"], &["VIEW"]]; +/// The optional existence guard of `CREATE`. +pub(crate) const IF_NOT_EXISTS: &[&str] = &["IF", "NOT", "EXISTS"]; +/// The optional existence guard of `DROP` and `ALTER`. +pub(crate) const IF_EXISTS: &[&str] = &["IF", "EXISTS"]; +/// Words that may follow the created relation's name. +pub(crate) const CREATE_FOLLOWING_WORDS: &[&[&str]] = &[ + &["AS"], + &["CLONE"], + &["SHALLOW"], + &["DEEP"], + &["LIKE"], + &["COPY"], + &["USING"], + &["COMMENT"], + &["CLUSTER"], + &["PARTITION"], + &["OPTIONS"], + &["WITH"], + &["TBLPROPERTIES"], + &["LOCATION"], + &["DATA_RETENTION_TIME_IN_DAYS"], + &["CHANGE_TRACKING"], +]; +/// The optional trailing word of `DROP`. +pub(crate) const DROP_FOLLOWING_WORDS: &[&[&str]] = &[&["CASCADE"], &["RESTRICT"], &["PURGE"]]; +/// Words that may follow the altered relation's name. +pub(crate) const ALTER_FOLLOWING_WORDS: &[&[&str]] = &[ + &["ADD"], + &["DROP"], + &["ALTER"], + &["MODIFY"], + &["RENAME"], + &["SWAP"], + &["SET"], + &["UNSET"], + &["CLUSTER"], + &["RECLUSTER"], + &["OWNER"], + &["SUSPEND"], + &["RESUME"], + &["REFRESH"], +]; +/// Alter clauses that name a second relation. +pub(crate) const RENAMING_ALTERS: &[&[&str]] = &[&["RENAME", "TO"], &["SWAP", "WITH"]]; +/// The word that must follow the target of `COPY INTO` and `SELECT ... INTO`. +pub(crate) const FROM_WORD: &[&str] = &["FROM"]; +/// `COPY INTO` before the loaded relation. +pub(crate) const COPY_INTO: &[&str] = &["COPY", "INTO"]; +/// The `INTO` keyword of `SELECT ... INTO`. +pub(crate) const INTO: &str = "INTO"; +/// Statement leads whose object is never a relation. +pub(crate) const NEUTRAL_OBJECT_LEADS: &[&[&str]] = &[&["CREATE"], &["DROP"], &["ALTER"]]; +/// The optional replacement phrase of a neutral object statement. +pub(crate) const OR_REPLACE: &[&str] = &["OR", "REPLACE"]; +/// Modifiers allowed before a neutral object kind. +pub(crate) const NEUTRAL_OBJECT_MODIFIERS: &[&[&str]] = + &[&["SECURE"], &["TEMP"], &["TEMPORARY"], &["AGGREGATE"]]; +/// Object kinds whose statements never change relation existence or columns. +pub(crate) const NEUTRAL_OBJECT_KINDS: &[&[&str]] = &[ + &["FUNCTION"], + &["PROCEDURE"], + &["SEQUENCE"], + &["STAGE"], + &["FILE", "FORMAT"], + &["INDEX"], + &["UNIQUE", "INDEX"], + &["WAREHOUSE"], + &["MACRO"], +]; +/// `CREATE SCHEMA IF NOT EXISTS` before the schema name. +pub(crate) const CREATE_SCHEMA_IF_NOT_EXISTS: &[&str] = + &["CREATE", "SCHEMA", "IF", "NOT", "EXISTS"]; +/// The statement separator. +pub(crate) const STATEMENT_SEPARATOR: char = ';'; +/// The shortest name that can be wrapped in identifier quotes. +pub(crate) const QUOTED_IDENTIFIER_MIN_LENGTH: usize = 2; diff --git a/crates/sqlbuild-sqltext/src/ddl_effects/main/mod.rs b/crates/sqlbuild-sqltext/src/ddl_effects/main/mod.rs new file mode 100644 index 0000000000..afc03a0b13 --- /dev/null +++ b/crates/sqlbuild-sqltext/src/ddl_effects/main/mod.rs @@ -0,0 +1,3 @@ +pub mod qualified_relation_name_key; +pub mod relation_name_key; +pub mod statement_metadata_effect; diff --git a/crates/sqlbuild-sqltext/src/ddl_effects/main/qualified_relation_name_key.rs b/crates/sqlbuild-sqltext/src/ddl_effects/main/qualified_relation_name_key.rs new file mode 100644 index 0000000000..bd094f0910 --- /dev/null +++ b/crates/sqlbuild-sqltext/src/ddl_effects/main/qualified_relation_name_key.rs @@ -0,0 +1,15 @@ +//! The cache invalidation key of a qualified relation name's last part. + +use crate::ddl_effects::_helpers::matching::{only_whitespace, skip_whitespace}; +use crate::ddl_effects::_helpers::names::{UnquotedPart, part_name_key, qualified_name}; + +/// The key for the last part of a qualified name, or `None` when it cannot be parsed. +#[must_use] +pub fn qualified_relation_name_key(qualified: &str) -> Option { + let name = qualified_name( + qualified, + skip_whitespace(qualified, 0), + UnquotedPart::Loose, + )?; + only_whitespace(qualified, name.end).then(|| part_name_key(name.last_part)) +} diff --git a/crates/sqlbuild-sqltext/src/ddl_effects/main/relation_name_key.rs b/crates/sqlbuild-sqltext/src/ddl_effects/main/relation_name_key.rs new file mode 100644 index 0000000000..621d8fbb05 --- /dev/null +++ b/crates/sqlbuild-sqltext/src/ddl_effects/main/relation_name_key.rs @@ -0,0 +1,9 @@ +//! The cache invalidation key of one relation name. + +use crate::ddl_effects::_helpers::names::name_key; + +/// The invalidation key for one unquoted or quoted unqualified relation name. +#[must_use] +pub fn relation_name_key(name: &str) -> String { + name_key(name) +} diff --git a/crates/sqlbuild-sqltext/src/ddl_effects/main/statement_metadata_effect.rs b/crates/sqlbuild-sqltext/src/ddl_effects/main/statement_metadata_effect.rs new file mode 100644 index 0000000000..0022a087ac --- /dev/null +++ b/crates/sqlbuild-sqltext/src/ddl_effects/main/statement_metadata_effect.rs @@ -0,0 +1,37 @@ +//! Classify executed SQL by its effect on cached relation existence and columns. + +use crate::ddl_effects::_helpers::code_text::{LexicalMode, code_text}; +use crate::ddl_effects::_helpers::relations::{all_relations, code_effect}; +use crate::ddl_effects::models::StatementMetadataEffect; + +/// Which cached relation names `sql` may have changed; readings that disagree change all. +#[must_use] +pub fn statement_metadata_effect(sql: &str) -> StatementMetadataEffect { + let backslash_modes: &[bool] = if sql.contains('\\') { + &[false, true] + } else { + &[false] + }; + let bracket_modes: &[bool] = if sql.contains('[') { + &[false, true] + } else { + &[false] + }; + let mut effects: Vec = Vec::new(); + for backslash_escapes in backslash_modes { + for bracket_identifiers in bracket_modes { + let mode: LexicalMode = LexicalMode { + backslash_escapes: *backslash_escapes, + bracket_identifiers: *bracket_identifiers, + }; + let effect: StatementMetadataEffect = code_effect(code_text(sql, mode).as_deref()); + if !effects.contains(&effect) { + effects.push(effect); + } + } + } + match <[StatementMetadataEffect; 1]>::try_from(effects) { + Ok([effect]) => effect, + Err(_) => all_relations(), + } +} diff --git a/crates/sqlbuild-sqltext/src/ddl_effects/mod.rs b/crates/sqlbuild-sqltext/src/ddl_effects/mod.rs new file mode 100644 index 0000000000..1e64f3d116 --- /dev/null +++ b/crates/sqlbuild-sqltext/src/ddl_effects/mod.rs @@ -0,0 +1,7 @@ +pub(crate) mod _helpers; +pub(crate) mod constants; +pub mod main; +pub mod models; + +#[cfg(test)] +mod tests; diff --git a/crates/sqlbuild-sqltext/src/ddl_effects/models.rs b/crates/sqlbuild-sqltext/src/ddl_effects/models.rs new file mode 100644 index 0000000000..4952635ab0 --- /dev/null +++ b/crates/sqlbuild-sqltext/src/ddl_effects/models.rs @@ -0,0 +1,14 @@ +//! What one executed statement may have changed in cached relation metadata. + +use std::collections::BTreeSet; + +/// Cached relation names a statement may have changed; ambiguity changes all of them. +#[derive(Clone, Debug, Default, PartialEq, Eq)] +pub struct StatementMetadataEffect { + /// Invalidation keys of the relations the statement names. + pub relation_names: BTreeSet, + /// Every cached relation may have changed. + pub invalidates_all: bool, + /// The statement ends the current transaction. + pub ends_transaction: bool, +} diff --git a/crates/sqlbuild-sqltext/src/ddl_effects/tests.rs b/crates/sqlbuild-sqltext/src/ddl_effects/tests.rs new file mode 100644 index 0000000000..b66f762b66 --- /dev/null +++ b/crates/sqlbuild-sqltext/src/ddl_effects/tests.rs @@ -0,0 +1,4 @@ +#[path = "tests/test_ddl_effects.rs"] +mod ddl_effects; +#[path = "tests/test_types.rs"] +mod test_types; diff --git a/crates/sqlbuild-sqltext/src/ddl_effects/tests/test_ddl_effects.rs b/crates/sqlbuild-sqltext/src/ddl_effects/tests/test_ddl_effects.rs new file mode 100644 index 0000000000..761ce6a14a --- /dev/null +++ b/crates/sqlbuild-sqltext/src/ddl_effects/tests/test_ddl_effects.rs @@ -0,0 +1,223 @@ +//! Executed SQL is classified conservatively by its effect on cached relation metadata. + +use crate::ddl_effects::main::qualified_relation_name_key::qualified_relation_name_key; +use crate::ddl_effects::main::relation_name_key::relation_name_key; +use crate::ddl_effects::main::statement_metadata_effect::statement_metadata_effect; +use crate::ddl_effects::models::StatementMetadataEffect; +use crate::ddl_effects::tests::test_types::{NameKeyTestCase, StatementEffectTestCase}; + +#[test] +fn given_executed_statement_when_classifying_then_reports_possible_metadata_changes() { + let test_cases = [ + StatementEffectTestCase { + description: "create or replace transient table", + sql: "CREATE OR REPLACE TRANSIENT TABLE analytics.marts.orders__delta AS SELECT 1 AS id", + expected_relation_names: &["orders__delta"], + expected_invalidates_all: false, + expected_ends_transaction: false, + }, + StatementEffectTestCase { + description: "quoted three-part view", + sql: r#"CREATE OR REPLACE VIEW "ANALYTICS"."MARTS"."ORDERS_V" AS SELECT 1 AS id"#, + expected_relation_names: &["orders_v"], + expected_invalidates_all: false, + expected_ends_transaction: false, + }, + StatementEffectTestCase { + description: "rename names both relations", + sql: "ALTER TABLE marts.orders__stage RENAME TO marts.orders", + expected_relation_names: &["orders", "orders__stage"], + expected_invalidates_all: false, + expected_ends_transaction: false, + }, + StatementEffectTestCase { + description: "backtick qualified name keys its last segment", + sql: "CREATE OR REPLACE TABLE `project.dataset.Orders` AS SELECT 1 AS id", + expected_relation_names: &["orders"], + expected_invalidates_all: false, + expected_ends_transaction: false, + }, + StatementEffectTestCase { + description: "bracket qualified name", + sql: "DROP TABLE IF EXISTS [dbo].[orders__delta]", + expected_relation_names: &["orders__delta"], + expected_invalidates_all: false, + expected_ends_transaction: false, + }, + StatementEffectTestCase { + description: "select into after a common table expression", + sql: "WITH v AS (SELECT 1 AS id) SELECT id INTO dbo.orders_history FROM v", + expected_relation_names: &["orders_history"], + expected_invalidates_all: false, + expected_ends_transaction: false, + }, + StatementEffectTestCase { + description: "copy into a table", + sql: "COPY INTO raw.orders FROM @landing/orders", + expected_relation_names: &["orders"], + expected_invalidates_all: false, + expected_ends_transaction: false, + }, + StatementEffectTestCase { + description: "separator inside a literal is data", + sql: "CREATE OR REPLACE TABLE marts.notes AS SELECT 'a;b -- c' AS note", + expected_relation_names: &["notes"], + expected_invalidates_all: false, + expected_ends_transaction: false, + }, + StatementEffectTestCase { + description: "function body in dollar quotes", + sql: "CREATE OR REPLACE FUNCTION f() RETURNS INT AS $$ SELECT 1; SELECT 2 $$", + expected_relation_names: &[], + expected_invalidates_all: false, + expected_ends_transaction: false, + }, + StatementEffectTestCase { + description: "create schema if missing", + sql: "CREATE SCHEMA IF NOT EXISTS analytics.marts", + expected_relation_names: &[], + expected_invalidates_all: false, + expected_ends_transaction: false, + }, + StatementEffectTestCase { + description: "procedure call", + sql: "CALL marts.rebuild_orders()", + expected_relation_names: &[], + expected_invalidates_all: true, + expected_ends_transaction: false, + }, + StatementEffectTestCase { + description: "multi statement", + sql: "INSERT INTO t VALUES (1); DROP TABLE t", + expected_relation_names: &[], + expected_invalidates_all: true, + expected_ends_transaction: false, + }, + StatementEffectTestCase { + description: "unterminated quoted name", + sql: r#"CREATE TABLE "open AS SELECT 1"#, + expected_relation_names: &[], + expected_invalidates_all: true, + expected_ends_transaction: false, + }, + StatementEffectTestCase { + description: "unquoted non-ascii name", + sql: "CREATE TABLE s.café (id INT)", + expected_relation_names: &[], + expected_invalidates_all: true, + expected_ends_transaction: false, + }, + StatementEffectTestCase { + description: "backslash-escaped quote is ambiguous", + sql: r"SELECT 'a\' AS x FROM t WHERE y = '; DROP TABLE u'", + expected_relation_names: &[], + expected_invalidates_all: true, + expected_ends_transaction: false, + }, + StatementEffectTestCase { + description: "rename target followed by more text", + sql: "ALTER TABLE a RENAME TO b c", + expected_relation_names: &[], + expected_invalidates_all: true, + expected_ends_transaction: false, + }, + StatementEffectTestCase { + description: "commit ends the transaction", + sql: "COMMIT;", + expected_relation_names: &[], + expected_invalidates_all: false, + expected_ends_transaction: true, + }, + ]; + for test_case in test_cases { + let effect: StatementMetadataEffect = statement_metadata_effect(test_case.sql); + assert_eq!( + effect + .relation_names + .iter() + .map(String::as_str) + .collect::>(), + test_case.expected_relation_names, + "{}", + test_case.description + ); + assert_eq!( + effect.invalidates_all, test_case.expected_invalidates_all, + "{}", + test_case.description + ); + assert_eq!( + effect.ends_transaction, test_case.expected_ends_transaction, + "{}", + test_case.description + ); + } +} + +#[test] +fn given_qualified_name_when_keying_then_uses_last_part_or_reports_unparseable() { + let test_cases = [ + NameKeyTestCase { + description: "unquoted part starting with a digit", + name: "my-proj.ds.2024_orders", + expected_key: Some("2024_orders"), + }, + NameKeyTestCase { + description: "backticked path keys its last segment", + name: "`my-proj.ds.Orders`", + expected_key: Some("orders"), + }, + NameKeyTestCase { + description: "quoted dotted part keeps its dot", + name: r#""ANALYTICS"."MARTS"."my.table""#, + expected_key: Some("my.table"), + }, + NameKeyTestCase { + description: "trailing text is unparseable", + name: "ds.orders extra", + expected_key: None, + }, + NameKeyTestCase { + description: "unquoted non-ascii part is unparseable", + name: "ds.café", + expected_key: None, + }, + ]; + for test_case in test_cases { + assert_eq!( + qualified_relation_name_key(test_case.name).as_deref(), + test_case.expected_key, + "{}", + test_case.description + ); + } +} + +#[test] +fn given_relation_name_when_keying_then_strips_quotes_and_casefolds() { + let test_cases = [ + NameKeyTestCase { + description: "doubled quotes unescape", + name: r#""Or""ders""#, + expected_key: Some(r#"or"ders"#), + }, + NameKeyTestCase { + description: "python casefold expands sharp s", + name: "STRAßE", + expected_key: Some("strasse"), + }, + NameKeyTestCase { + description: "bracketed name", + name: "[Orders]", + expected_key: Some("orders"), + }, + ]; + for test_case in test_cases { + assert_eq!( + Some(relation_name_key(test_case.name).as_str()), + test_case.expected_key, + "{}", + test_case.description + ); + } +} diff --git a/crates/sqlbuild-sqltext/src/ddl_effects/tests/test_types.rs b/crates/sqlbuild-sqltext/src/ddl_effects/tests/test_types.rs new file mode 100644 index 0000000000..c027cdfd78 --- /dev/null +++ b/crates/sqlbuild-sqltext/src/ddl_effects/tests/test_types.rs @@ -0,0 +1,13 @@ +pub(super) struct StatementEffectTestCase { + pub(super) description: &'static str, + pub(super) sql: &'static str, + pub(super) expected_relation_names: &'static [&'static str], + pub(super) expected_invalidates_all: bool, + pub(super) expected_ends_transaction: bool, +} + +pub(super) struct NameKeyTestCase { + pub(super) description: &'static str, + pub(super) name: &'static str, + pub(super) expected_key: Option<&'static str>, +} diff --git a/crates/sqlbuild-sqltext/src/lib.rs b/crates/sqlbuild-sqltext/src/lib.rs index 59a15f977b..a1e0563af4 100644 --- a/crates/sqlbuild-sqltext/src/lib.rs +++ b/crates/sqlbuild-sqltext/src/lib.rs @@ -5,6 +5,7 @@ pub mod code_scanner; pub mod compiler; mod constants; +pub mod ddl_effects; pub mod resolution; pub mod sql_references; pub mod sql_scan; diff --git a/python-boundary.toml b/python-boundary.toml index e272b29b1a..a73c16ae83 100644 --- a/python-boundary.toml +++ b/python-boundary.toml @@ -27,7 +27,7 @@ note = "Imports project adapter modules." [[stays]] path = "adapter/relations" reason = "adapter" -note = "Relation inspection wrappers adapters call, with per-connection caches." +note = "Relation inspection wrappers adapters call, with per-connection caches; executed-statement classification and invalidation keys are native." [[stays]] path = "adapter/state_sql" diff --git a/src/sqlbuild/_native.pyi b/src/sqlbuild/_native.pyi index 744e262d19..b171957ba5 100644 --- a/src/sqlbuild/_native.pyi +++ b/src/sqlbuild/_native.pyi @@ -37,7 +37,7 @@ from sqlbuild.adapter.contract.types import ( TablePromotionMode, TypeDialect, ) -from sqlbuild.adapter.relations.models import InspectionQueryRecord +from sqlbuild.adapter.relations.models import InspectionQueryRecord, StatementMetadataEffect from sqlbuild.cli.compile_reuse.models import CompileReuseRequest from sqlbuild.compiler.auditing.types import ( AuditAttachmentKind, @@ -5413,6 +5413,9 @@ def apply_deferred_locations( selected_keys: frozenset[CompiledObjectKey], ) -> tuple[dict[str, CompiledRelationLocation], dict[str, CompiledRelationLocation]]: ... def substitute_placeholder_defaults(sql: str, placeholders: list[tuple[str, str]], /) -> str: ... +def statement_metadata_effect(sql: str, /) -> StatementMetadataEffect: ... +def relation_name_key(name: str, /) -> str: ... +def qualified_relation_name_key(qualified: str, /) -> str | None: ... def fixture_projection_facts( sql: str, dialect: str | None, / ) -> tuple[list[tuple[str, str]], list[str], list[str]]: ... diff --git a/src/sqlbuild/adapter/relations/_helpers/ddl_effects.py b/src/sqlbuild/adapter/relations/_helpers/ddl_effects.py deleted file mode 100644 index f78d68623c..0000000000 --- a/src/sqlbuild/adapter/relations/_helpers/ddl_effects.py +++ /dev/null @@ -1,343 +0,0 @@ -"""Classify executed SQL by its effect on cached relation existence and columns.""" - -from __future__ import annotations - -import re - -from sqlbuild.adapter.relations.constants import ( - BACKTICK, - DOUBLE_QUOTE, - IDENTIFIER_QUOTE_PAIRS, - QUOTED_IDENTIFIER_MIN_LENGTH, - STATEMENT_SEPARATOR, -) -from sqlbuild.adapter.relations.models import StatementMetadataEffect - -_LEXEME_START: re.Pattern[str] = re.compile(r"--|/\*|['\"`$\[]") -_BACKSLASH_LITERAL_STOP: re.Pattern[str] = re.compile(r"[\\']") -_DOLLAR_TAG: re.Pattern[str] = re.compile(r"\$[A-Za-z_]*+\$") -_LINE_COMMENT_START: str = "--" -_BLOCK_COMMENT_START: str = "/*" -_BLOCK_COMMENT_END: str = "*/" -_SINGLE_QUOTE: str = "'" -_BACKSLASH: str = "\\" -_DOLLAR: str = "$" -_OPEN_BRACKET: str = "[" -_CLOSE_BRACKET: str = "]" -_NEWLINE: str = "\n" -_EMPTY_LITERAL: str = "''" -_COMMENT_REPLACEMENT: str = " " -_FIRST_WORD: re.Pattern[str] = re.compile(r"[A-Za-z]+") -_IDENTIFIER_PART: str = r'(?:"(?:[^"]|"")*+"|`[^`]*+`|\[[^\]]*+\]|[A-Za-z0-9_][A-Za-z0-9_$]*+)' -_QUALIFIED_NAME: str = ( - rf"(?P{_IDENTIFIER_PART}(?:\s*+\.\s*+{_IDENTIFIER_PART})*+)" - r"(?P(?=\s|\()(?s:.*)|$)" -) -_LOOSE_IDENTIFIER_PART: str = r'(?:"(?:[^"]|"")*+"|`[^`]*+`|\[[^\]]*+\]|[A-Za-z0-9_$-]++)' -_LOOSE_QUALIFIED_NAME: re.Pattern[str] = re.compile( - rf"^\s*+{_LOOSE_IDENTIFIER_PART}(?:\s*+\.\s*+{_LOOSE_IDENTIFIER_PART})*+\s*+$" -) -_RELATION_KIND: str = ( - r"(?:(?:OR\s++REPLACE|LOCAL|GLOBAL|TEMP|TEMPORARY|TRANSIENT|VOLATILE|SECURE|RECURSIVE|" - r"MATERIALIZED|EXTERNAL|DYNAMIC|ICEBERG|HYBRID|UNLOGGED|FOREIGN|STREAMING)\s++)*+" - r"(?:TABLE|VIEW)\s++" -) -_CREATE_RELATION: re.Pattern[str] = re.compile( - rf"^CREATE\s++{_RELATION_KIND}(?:IF\s++NOT\s++EXISTS\s++)?{_QUALIFIED_NAME}", re.IGNORECASE -) -_DROP_RELATION: re.Pattern[str] = re.compile( - rf"^DROP\s++{_RELATION_KIND}(?:IF\s++EXISTS\s++)?{_QUALIFIED_NAME}", re.IGNORECASE -) -_ALTER_RELATION: re.Pattern[str] = re.compile( - rf"^ALTER\s++{_RELATION_KIND}(?:IF\s++EXISTS\s++)?{_QUALIFIED_NAME}", re.IGNORECASE -) -_COPY_INTO_RELATION: re.Pattern[str] = re.compile( - rf"^COPY\s++INTO\s++{_QUALIFIED_NAME}", re.IGNORECASE -) -_SELECT_INTO_RELATION: re.Pattern[str] = re.compile(rf"\bINTO\s++{_QUALIFIED_NAME}", re.IGNORECASE) -_SELECT_INTO_KEYWORD: re.Pattern[str] = re.compile(r"\bINTO\b", re.IGNORECASE) -_SECOND_RELATION: re.Pattern[str] = re.compile( - rf"^\s++(?:RENAME\s++TO|SWAP\s++WITH)\s++{_QUALIFIED_NAME}", re.IGNORECASE -) -_FOLLOWING_WORDS: dict[re.Pattern[str], re.Pattern[str]] = { - _CREATE_RELATION: re.compile( - r"^\s*+(?:$|\(|(?:AS|CLONE|SHALLOW|DEEP|LIKE|COPY|USING|COMMENT|CLUSTER|PARTITION|OPTIONS|WITH|" - r"TBLPROPERTIES|LOCATION|DATA_RETENTION_TIME_IN_DAYS|CHANGE_TRACKING)\b)", - re.IGNORECASE, - ), - _DROP_RELATION: re.compile(r"^\s*+(?:CASCADE|RESTRICT|PURGE)?+\s*+$", re.IGNORECASE), - _ALTER_RELATION: re.compile( - r"^\s++(?:ADD|DROP|ALTER|MODIFY|RENAME|SWAP|SET|UNSET|CLUSTER|RECLUSTER|OWNER|" - r"SUSPEND|RESUME|REFRESH)\b", - re.IGNORECASE, - ), - _COPY_INTO_RELATION: re.compile(r"^\s++FROM\b", re.IGNORECASE), - _SELECT_INTO_RELATION: re.compile(r"^\s++FROM\b", re.IGNORECASE), - _SECOND_RELATION: re.compile(r"^\s*+$"), -} -_CREATE_SCHEMA_IF_NOT_EXISTS: re.Pattern[str] = re.compile( - r"^CREATE\s++SCHEMA\s++IF\s++NOT\s++EXISTS\s++[^\s;]++\s*+$", re.IGNORECASE -) -_RELATION_NEUTRAL_OBJECT: re.Pattern[str] = re.compile( - r"^(?:CREATE|DROP|ALTER)\s++(?:OR\s++REPLACE\s++)?(?:(?:SECURE|TEMP|TEMPORARY|AGGREGATE)\s++)*+" - r"(?:FUNCTION|PROCEDURE|SEQUENCE|STAGE|FILE\s++FORMAT|INDEX|UNIQUE\s++INDEX|WAREHOUSE|MACRO)\b", - re.IGNORECASE, -) -_RENAMING_ALTER: re.Pattern[str] = re.compile( - r"^\s++(?:RENAME\s++TO|SWAP\s++WITH)\b", re.IGNORECASE -) -_QUERY_STATEMENTS: frozenset[str] = frozenset({"SELECT", "WITH"}) -_READ_OR_DATA_STATEMENTS: frozenset[str] = frozenset( - { - "VALUES", - "SHOW", - "DESCRIBE", - "DESC", - "EXPLAIN", - "INSERT", - "UPDATE", - "DELETE", - "MERGE", - "TRUNCATE", - "BEGIN", - "START", - "SAVEPOINT", - "RELEASE", - "SET", - "UNSET", - "GRANT", - "REVOKE", - "COMMENT", - "PUT", - "GET", - "LIST", - "LS", - "REMOVE", - "RM", - "ANALYZE", - } -) -_TRANSACTION_END_STATEMENTS: frozenset[str] = frozenset({"COMMIT", "ROLLBACK", "END", "ABORT"}) -_RELATION_DDL_STATEMENTS: frozenset[str] = frozenset({"CREATE", "DROP", "ALTER", "COPY"}) - - -_NO_EFFECT: StatementMetadataEffect = StatementMetadataEffect() -_ALL_RELATIONS: StatementMetadataEffect = StatementMetadataEffect(invalidates_all=True) -_TRANSACTION_END: StatementMetadataEffect = StatementMetadataEffect(ends_transaction=True) - - -def statement_metadata_effect(sql: str) -> StatementMetadataEffect: - """Return which cached relation names ``sql`` may have changed; ambiguity changes all.""" - - backslash_modes: tuple[bool, ...] = (False, True) if _BACKSLASH in sql else (False,) - bracket_modes: tuple[bool, ...] = (False, True) if _OPEN_BRACKET in sql else (False,) - effects: set[StatementMetadataEffect] = set() - backslash_escapes: bool - for backslash_escapes in backslash_modes: - effects.update( - _code_effect( - code=_code_text( - sql=sql, backslash_escapes=backslash_escapes, bracket_identifiers=brackets - ) - ) - for brackets in bracket_modes - ) - if len(effects) != 1: - return _ALL_RELATIONS - return effects.pop() - - -def relation_name_key(name: str) -> str: - """Return the invalidation key for one unquoted or quoted unqualified relation name.""" - - return _strip_identifier_quotes(name).casefold() - - -def qualified_relation_name_key(qualified: str) -> str | None: - """Return the key for the last part of a qualified name, or None when it cannot be parsed.""" - - if _LOOSE_QUALIFIED_NAME.match(qualified) is None: - return None - parts: list[str] = re.findall(_LOOSE_IDENTIFIER_PART, qualified) - return _part_name_key(parts[-1]) - - -def _code_text(*, sql: str, backslash_escapes: bool, bracket_identifiers: bool) -> str | None: - pieces: list[str] = [] - position: int = 0 - while True: - start: re.Match[str] | None = _LEXEME_START.search(sql, position) - if start is None: - pieces.append(sql[position:]) - return "".join(pieces) - pieces.append(sql[position : start.start()]) - lexeme: tuple[str, int] | None = _read_lexeme( - sql=sql, - start=start, - backslash_escapes=backslash_escapes, - bracket_identifiers=bracket_identifiers, - ) - if lexeme is None: - return None - pieces.append(lexeme[0]) - position = lexeme[1] - - -def _read_lexeme( - *, sql: str, start: re.Match[str], backslash_escapes: bool, bracket_identifiers: bool -) -> tuple[str, int] | None: - token: str = start.group(0) - begin: int = start.start() - if token == _LINE_COMMENT_START: - line_end: int = sql.find(_NEWLINE, begin) - return _COMMENT_REPLACEMENT, (len(sql) if line_end < 0 else line_end) - if token == _BLOCK_COMMENT_START: - return _closed( - text=_COMMENT_REPLACEMENT, end=sql.find(_BLOCK_COMMENT_END, begin + 2), width=2 - ) - if token == _SINGLE_QUOTE: - return _closed( - text=_EMPTY_LITERAL, - end=_literal_end(sql=sql, begin=begin, backslash_escapes=backslash_escapes), - width=1, - ) - if token == _DOLLAR: - return _dollar_lexeme(sql=sql, begin=begin) - if token == _OPEN_BRACKET and not bracket_identifiers: - return token, begin + 1 - closing: str = _CLOSE_BRACKET if token == _OPEN_BRACKET else token - end: int = _quoted_end(sql=sql, begin=begin, quote=closing) - if end < 0: - return None - return sql[begin : end + 1], end + 1 - - -def _closed(*, text: str, end: int, width: int) -> tuple[str, int] | None: - if end < 0: - return None - return text, end + width - - -def _dollar_lexeme(*, sql: str, begin: int) -> tuple[str, int] | None: - tag: re.Match[str] | None = _DOLLAR_TAG.match(sql, begin) - if tag is None: - return _DOLLAR, begin + 1 - return _closed( - text=_EMPTY_LITERAL, end=sql.find(tag.group(0), tag.end()), width=len(tag.group(0)) - ) - - -def _literal_end(*, sql: str, begin: int, backslash_escapes: bool) -> int: - position: int = begin + 1 - while True: - stop: int = _literal_stop(sql=sql, position=position, backslash_escapes=backslash_escapes) - if stop < 0: - return stop - if sql[stop] == _BACKSLASH: - position = stop + 2 - continue - if sql.startswith(_EMPTY_LITERAL, stop): - position = stop + 2 - continue - return stop - - -def _literal_stop(*, sql: str, position: int, backslash_escapes: bool) -> int: - if not backslash_escapes: - return sql.find(_SINGLE_QUOTE, position) - stop: re.Match[str] | None = _BACKSLASH_LITERAL_STOP.search(sql, position) - return -1 if stop is None else stop.start() - - -def _quoted_end(*, sql: str, begin: int, quote: str) -> int: - position: int = begin + 1 - while True: - end: int = sql.find(quote, position) - if end < 0 or quote == _CLOSE_BRACKET or not sql.startswith(quote * 2, end): - return end - position = end + 2 - - -def _code_effect(*, code: str | None) -> StatementMetadataEffect: - if code is None: - return _ALL_RELATIONS - statement: str = code.strip().rstrip(STATEMENT_SEPARATOR).strip().lstrip("(") - first: re.Match[str] | None = _FIRST_WORD.match(statement) - if first is None or STATEMENT_SEPARATOR in statement: - return _ALL_RELATIONS - keyword: str = first.group(0).upper() - if keyword in _QUERY_STATEMENTS: - return _select_into_effect(statement=statement) - if keyword in _READ_OR_DATA_STATEMENTS: - return _NO_EFFECT - if keyword in _TRANSACTION_END_STATEMENTS: - return _TRANSACTION_END - if keyword not in _RELATION_DDL_STATEMENTS: - return _ALL_RELATIONS - if ( - _CREATE_SCHEMA_IF_NOT_EXISTS.match(statement) is not None - or _RELATION_NEUTRAL_OBJECT.match(statement) is not None - ): - return _NO_EFFECT - return _relation_ddl_effect(statement=statement) - - -def _select_into_effect(*, statement: str) -> StatementMetadataEffect: - if _SELECT_INTO_KEYWORD.search(statement) is None: - return _NO_EFFECT - created: re.Match[str] | None = _SELECT_INTO_RELATION.search(statement) - names: frozenset[str] | None = _unambiguous_names(pattern=_SELECT_INTO_RELATION, match=created) - if names is None: - return _ALL_RELATIONS - return StatementMetadataEffect(relation_names=names) - - -def _relation_ddl_effect(*, statement: str) -> StatementMetadataEffect: - pattern: re.Pattern[str] - for pattern in (_CREATE_RELATION, _DROP_RELATION, _ALTER_RELATION, _COPY_INTO_RELATION): - match: re.Match[str] | None = pattern.match(statement) - names: frozenset[str] | None = _unambiguous_names(pattern=pattern, match=match) - if names is not None: - return StatementMetadataEffect(relation_names=names) - return _ALL_RELATIONS - - -def _unambiguous_names( - *, pattern: re.Pattern[str], match: re.Match[str] | None -) -> frozenset[str] | None: - if match is None: - return None - rest: str = match.group("rest") - if _FOLLOWING_WORDS[pattern].match(rest) is None: - return None - name: str = _unqualified_name_key(match.group("name")) - if pattern is not _ALTER_RELATION or _RENAMING_ALTER.match(rest) is None: - return frozenset({name}) - second: frozenset[str] | None = _unambiguous_names( - pattern=_SECOND_RELATION, match=_SECOND_RELATION.match(rest) - ) - if second is None: - return None - return frozenset({name, *second}) - - -def _unqualified_name_key(qualified: str) -> str: - parts: list[str] = re.findall(_IDENTIFIER_PART, qualified) - return _part_name_key(parts[-1]) - - -def _part_name_key(part: str) -> str: - if part.startswith(BACKTICK): - return _strip_identifier_quotes(part).rsplit(".", 1)[-1].casefold() - return relation_name_key(part) - - -def _strip_identifier_quotes(part: str) -> str: - if len(part) < QUOTED_IDENTIFIER_MIN_LENGTH: - return part - edges: tuple[str, str] = (part[0], part[-1]) - if edges == (DOUBLE_QUOTE, DOUBLE_QUOTE): - return part[1:-1].replace(DOUBLE_QUOTE * 2, DOUBLE_QUOTE) - if edges in IDENTIFIER_QUOTE_PAIRS: - return part[1:-1] - return part diff --git a/src/sqlbuild/adapter/relations/classes/relation_metadata_cache.py b/src/sqlbuild/adapter/relations/classes/relation_metadata_cache.py index 6f745f81f2..588c1ffb73 100644 --- a/src/sqlbuild/adapter/relations/classes/relation_metadata_cache.py +++ b/src/sqlbuild/adapter/relations/classes/relation_metadata_cache.py @@ -6,7 +6,7 @@ from collections.abc import Callable from typing import cast -from sqlbuild.adapter.relations._helpers.ddl_effects import ( +from sqlbuild._native import ( qualified_relation_name_key, relation_name_key, statement_metadata_effect, diff --git a/src/sqlbuild/adapter/relations/constants.py b/src/sqlbuild/adapter/relations/constants.py index e68a5951b2..8fa46168db 100644 --- a/src/sqlbuild/adapter/relations/constants.py +++ b/src/sqlbuild/adapter/relations/constants.py @@ -3,10 +3,5 @@ INSPECTION_DESCRIBED_NAME_LIMIT: int = 3 INSPECTION_IN_LIST_LIMIT: int = 50 METADATA_NAME_FILTER_LIMIT: int = 250 -STATEMENT_SEPARATOR: str = ";" -QUOTED_IDENTIFIER_MIN_LENGTH: int = 2 -DOUBLE_QUOTE: str = '"' -IDENTIFIER_QUOTE_PAIRS: frozenset[tuple[str, str]] = frozenset({("`", "`"), ("[", "]")}) RELATION_COLUMNS_LOOKUP: str = "columns" RELATION_EXISTS_LOOKUP: str = "exists" -BACKTICK: str = "`" diff --git a/tests/unit/src/sqlbuild/adapter/relations/_helpers/ddl_effects/test_adversarial_input_runtime.py b/tests/unit/src/sqlbuild/adapter/relations/_helpers/ddl_effects/test_adversarial_input_runtime.py index aa7e533133..ab35de93a2 100644 --- a/tests/unit/src/sqlbuild/adapter/relations/_helpers/ddl_effects/test_adversarial_input_runtime.py +++ b/tests/unit/src/sqlbuild/adapter/relations/_helpers/ddl_effects/test_adversarial_input_runtime.py @@ -6,7 +6,7 @@ import pytest -from sqlbuild.adapter.relations._helpers.ddl_effects import statement_metadata_effect +from sqlbuild._native import statement_metadata_effect from sqlbuild.adapters.snowflake._helpers.session_context import may_switch_session_database from tests.unit.src.sqlbuild.adapter.relations._helpers.ddl_effects._test_types import ( AdversarialStatementTestCase, diff --git a/tests/unit/src/sqlbuild/adapter/relations/_helpers/ddl_effects/test_statement_metadata_effect.py b/tests/unit/src/sqlbuild/adapter/relations/_helpers/ddl_effects/test_statement_metadata_effect.py index ae3d75af7a..8708427e95 100644 --- a/tests/unit/src/sqlbuild/adapter/relations/_helpers/ddl_effects/test_statement_metadata_effect.py +++ b/tests/unit/src/sqlbuild/adapter/relations/_helpers/ddl_effects/test_statement_metadata_effect.py @@ -4,7 +4,7 @@ import pytest -from sqlbuild.adapter.relations._helpers.ddl_effects import ( +from sqlbuild._native import ( qualified_relation_name_key, statement_metadata_effect, ) From c3f9b48070af01985646d3a4b6e9c5b969e60c5c Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Thu, 15 Oct 2026 18:57:49 +0100 Subject: [PATCH 18/48] refactor: build ingestr destination URIs and command lines natively --- .../ingestr_command/_helpers/connection.rs | 42 ++++ .../ingestr_command/_helpers/destinations.rs | 130 +++++++++++ .../src/ingestr_command/_helpers/mod.rs | 3 + .../src/ingestr_command/_helpers/url_quote.rs | 42 ++++ .../src/ingestr_command/constants.rs | 37 +++ .../src/ingestr_command/errors.rs | 7 + .../main/format_ingestr_command.rs | 13 ++ .../main/ingestr_command_arguments.rs | 43 ++++ .../main/ingestr_destination_table.rs | 10 + .../main/ingestr_destination_uri.rs | 28 +++ .../src/ingestr_command/main/mod.rs | 4 + .../sqlbuild-core/src/ingestr_command/mod.rs | 10 + .../src/ingestr_command/models.rs | 33 +++ .../src/ingestr_command/tests.rs | 4 + .../tests/test_ingestr_command.rs | 124 ++++++++++ .../src/ingestr_command/tests/test_types.rs | 14 ++ crates/sqlbuild-core/src/lib.rs | 2 + .../sqlbuild-core/src/shell_text/main/mod.rs | 1 + .../src/shell_text/main/shell_quote.rs | 16 ++ crates/sqlbuild-core/src/shell_text/mod.rs | 3 + .../bindings/_helpers/load/ingestr_command.rs | 218 ++++++++++++++++++ .../src/bindings/_helpers/load/mod.rs | 1 + .../bindings/_helpers/load/registration.rs | 5 +- .../src/refactor_command/_helpers/quoting.rs | 14 +- .../src/scope_text/_helpers/layout.rs | 15 +- python-boundary.toml | 2 +- src/sqlbuild/_native.pyi | 12 + .../ingestr/_helpers/destination.py | 177 -------------- .../integrations/ingestr/_helpers/output.py | 8 - .../integrations/ingestr/_helpers/runner.py | 2 +- .../integrations/ingestr/constants.py | 2 - .../integrations/ingestr/main/_loaders.py | 13 +- .../integrations/ingestr/_test_types.py | 11 + .../integrations/ingestr/test_command.py | 70 +++++- 34 files changed, 885 insertions(+), 231 deletions(-) create mode 100644 crates/sqlbuild-core/src/ingestr_command/_helpers/connection.rs create mode 100644 crates/sqlbuild-core/src/ingestr_command/_helpers/destinations.rs create mode 100644 crates/sqlbuild-core/src/ingestr_command/_helpers/mod.rs create mode 100644 crates/sqlbuild-core/src/ingestr_command/_helpers/url_quote.rs create mode 100644 crates/sqlbuild-core/src/ingestr_command/constants.rs create mode 100644 crates/sqlbuild-core/src/ingestr_command/errors.rs create mode 100644 crates/sqlbuild-core/src/ingestr_command/main/format_ingestr_command.rs create mode 100644 crates/sqlbuild-core/src/ingestr_command/main/ingestr_command_arguments.rs create mode 100644 crates/sqlbuild-core/src/ingestr_command/main/ingestr_destination_table.rs create mode 100644 crates/sqlbuild-core/src/ingestr_command/main/ingestr_destination_uri.rs create mode 100644 crates/sqlbuild-core/src/ingestr_command/main/mod.rs create mode 100644 crates/sqlbuild-core/src/ingestr_command/mod.rs create mode 100644 crates/sqlbuild-core/src/ingestr_command/models.rs create mode 100644 crates/sqlbuild-core/src/ingestr_command/tests.rs create mode 100644 crates/sqlbuild-core/src/ingestr_command/tests/test_ingestr_command.rs create mode 100644 crates/sqlbuild-core/src/ingestr_command/tests/test_types.rs create mode 100644 crates/sqlbuild-core/src/shell_text/main/mod.rs create mode 100644 crates/sqlbuild-core/src/shell_text/main/shell_quote.rs create mode 100644 crates/sqlbuild-core/src/shell_text/mod.rs create mode 100644 crates/sqlbuild-python/src/bindings/_helpers/load/ingestr_command.rs delete mode 100644 src/sqlbuild/integrations/ingestr/_helpers/destination.py diff --git a/crates/sqlbuild-core/src/ingestr_command/_helpers/connection.rs b/crates/sqlbuild-core/src/ingestr_command/_helpers/connection.rs new file mode 100644 index 0000000000..bd4661ff65 --- /dev/null +++ b/crates/sqlbuild-core/src/ingestr_command/_helpers/connection.rs @@ -0,0 +1,42 @@ +//! Connection settings read the way the ingestr destination expects them. + +use crate::ingestr_command::errors::IngestrCommandError; +use crate::ingestr_command::models::IngestrConnection; +use crate::text::main::python_strip::python_strip; + +/// The stripped string setting `key`, or `None` when unset, not a string or blank. +pub(crate) fn optional<'a>(connection: &'a IngestrConnection, key: &str) -> Option<&'a str> { + connection + .strings + .get(key) + .map(|value| python_strip(value)) + .filter(|value| !value.is_empty()) +} + +/// The stripped string setting `key`, which the adapter's destination requires. +pub(crate) fn required<'a>( + connection: &'a IngestrConnection, + key: &str, + adapter_name: &str, +) -> Result<&'a str, IngestrCommandError> { + optional(connection, key).ok_or_else(|| IngestrCommandError { + message: format!( + "Adapter '{adapter_name}' requires connection.{key} for ingestr integration loaders" + ), + }) +} + +/// `(key, value)` when the setting `key` is set. +fn setting<'a>(connection: &'a IngestrConnection, key: &'a str) -> Option<(&'a str, &'a str)> { + optional(connection, key).map(|value| (key, value)) +} + +/// The settings among `keys` that are set, in `keys` order. +pub(crate) fn present<'a>( + connection: &'a IngestrConnection, + keys: &[&'a str], +) -> Vec<(&'a str, &'a str)> { + keys.iter() + .filter_map(|key| setting(connection, key)) + .collect() +} diff --git a/crates/sqlbuild-core/src/ingestr_command/_helpers/destinations.rs b/crates/sqlbuild-core/src/ingestr_command/_helpers/destinations.rs new file mode 100644 index 0000000000..a8ffad57f9 --- /dev/null +++ b/crates/sqlbuild-core/src/ingestr_command/_helpers/destinations.rs @@ -0,0 +1,130 @@ +//! Destination URIs per warehouse adapter. + +use crate::ingestr_command::_helpers::connection::{optional, present, required}; +use crate::ingestr_command::_helpers::url_quote::{query_suffix, quote, urlencode}; +use crate::ingestr_command::constants::{ + BIGQUERY_PARAMETERS, DATABRICKS_PARAMETERS, MOTHERDUCK_PREFIX, POSTGRES_DEFAULT_PORT, + SNOWFLAKE_PARAMETERS, SQLSERVER_DEFAULT_DRIVER, SQLSERVER_DEFAULT_PORT, + SQLSERVER_DRIVER_PARAMETER, SQLSERVER_PARAMETERS, SQLSERVER_TRUST_SERVER_CERTIFICATE_PARAMETER, +}; +use crate::ingestr_command::errors::IngestrCommandError; +use crate::ingestr_command::models::IngestrConnection; + +/// DuckDB's database file URI. +pub(crate) fn duckdb(connection: &IngestrConnection) -> Result { + Ok(format!( + "duckdb:///{}", + required(connection, "database", "duckdb")? + )) +} + +/// MotherDuck's database URI with its optional token. +pub(crate) fn motherduck(connection: &IngestrConnection) -> String { + let database: &str = optional(connection, "database").unwrap_or_default(); + let database: &str = database.strip_prefix(MOTHERDUCK_PREFIX).unwrap_or(database); + let token: Vec<(&str, &str)> = optional(connection, "token") + .map(|token| ("token", token)) + .into_iter() + .collect(); + format!("motherduck://{}{}", quote(database), query_suffix(&token)) +} + +/// Postgres's URI with its optional SSL mode. +pub(crate) fn postgres(connection: &IngestrConnection) -> Result { + let host: &str = optional(connection, "host").unwrap_or("localhost"); + let port: &str = connection.port.as_deref().unwrap_or(POSTGRES_DEFAULT_PORT); + let user: &str = required(connection, "user", "postgres")?; + let password: &str = optional(connection, "password").unwrap_or_default(); + let database: &str = match optional(connection, "dbname") { + Some(database) => database, + None => required(connection, "database", "postgres")?, + }; + let sslmode: Vec<(&str, &str)> = present(connection, &["sslmode"]); + Ok(format!( + "postgresql://{}:{}@{host}:{port}/{}{}", + quote(user), + quote(password), + quote(database), + query_suffix(&sslmode) + )) +} + +/// BigQuery's project URI with its optional credentials. +pub(crate) fn bigquery(connection: &IngestrConnection) -> Result { + let project: &str = match optional(connection, "project") { + Some(project) => project, + None => required(connection, "database", "bigquery")?, + }; + Ok(format!( + "bigquery://{project}{}", + query_suffix(&present(connection, &BIGQUERY_PARAMETERS)) + )) +} + +/// Snowflake's account URI with its optional session settings. +pub(crate) fn snowflake(connection: &IngestrConnection) -> Result { + let account: &str = required(connection, "account", "snowflake")?; + let user: &str = required(connection, "user", "snowflake")?; + let password: &str = optional(connection, "password").unwrap_or_default(); + let database: &str = required(connection, "database", "snowflake")?; + let schema: &str = required(connection, "schema", "snowflake")?; + Ok(format!( + "snowflake://{}:{}@{account}/{}/{}{}", + quote(user), + quote(password), + quote(database), + quote(schema), + query_suffix(&present(connection, &SNOWFLAKE_PARAMETERS)) + )) +} + +/// Databricks's token URI with its warehouse path. +pub(crate) fn databricks(connection: &IngestrConnection) -> Result { + let host: &str = required(connection, "server_hostname", "databricks")?; + let token: &str = required(connection, "token", "databricks")?; + let mut parameters: Vec<(&str, &str)> = vec![( + "http_path", + required(connection, "http_path", "databricks")?, + )]; + parameters.extend(present(connection, &DATABRICKS_PARAMETERS)); + Ok(format!( + "databricks://token:{}@{host}?{}", + quote(token), + urlencode(¶meters) + )) +} + +/// SQL Server's URI with its ODBC defaults. +pub(crate) fn sqlserver(connection: &IngestrConnection) -> String { + let first = |keys: [&str; 2], default: &'static str| -> String { + keys.iter() + .find_map(|key| optional(connection, key)) + .unwrap_or(default) + .to_owned() + }; + let host: String = first(["host", "server"], "localhost"); + let port: &str = connection.port.as_deref().unwrap_or(SQLSERVER_DEFAULT_PORT); + let user: String = first(["user", "username"], "sa"); + let password: &str = optional(connection, "password").unwrap_or_default(); + let database: String = first(["database", "dbname"], "master"); + let mut parameters: Vec<(&str, &str)> = present(connection, &SQLSERVER_PARAMETERS); + let defaults: [(&str, &str); 2] = [ + (SQLSERVER_DRIVER_PARAMETER, SQLSERVER_DEFAULT_DRIVER), + (SQLSERVER_TRUST_SERVER_CERTIFICATE_PARAMETER, "yes"), + ]; + for (key, value) in defaults { + if !parameters + .iter() + .any(|(present_key, _)| *present_key == key) + { + parameters.push((key, value)); + } + } + format!( + "mssql://{}:{}@{host}:{port}/{}{}", + quote(&user), + quote(password), + quote(&database), + query_suffix(¶meters) + ) +} diff --git a/crates/sqlbuild-core/src/ingestr_command/_helpers/mod.rs b/crates/sqlbuild-core/src/ingestr_command/_helpers/mod.rs new file mode 100644 index 0000000000..0b39ed4d18 --- /dev/null +++ b/crates/sqlbuild-core/src/ingestr_command/_helpers/mod.rs @@ -0,0 +1,3 @@ +pub(crate) mod connection; +pub(crate) mod destinations; +pub(crate) mod url_quote; diff --git a/crates/sqlbuild-core/src/ingestr_command/_helpers/url_quote.rs b/crates/sqlbuild-core/src/ingestr_command/_helpers/url_quote.rs new file mode 100644 index 0000000000..d2566d4c35 --- /dev/null +++ b/crates/sqlbuild-core/src/ingestr_command/_helpers/url_quote.rs @@ -0,0 +1,42 @@ +//! Python's `urllib.parse.quote` and `urlencode` over string pairs. + +use crate::ingestr_command::constants::URL_ALWAYS_SAFE; + +fn quote_with(text: &str, safe: &[u8]) -> String { + let mut quoted: String = String::with_capacity(text.len()); + for byte in text.bytes() { + if byte.is_ascii_alphanumeric() || URL_ALWAYS_SAFE.contains(&byte) || safe.contains(&byte) { + quoted.push(char::from(byte)); + } else { + quoted.push_str(&format!("%{byte:02X}")); + } + } + quoted +} + +/// Python's `urllib.parse.quote(text)`, which keeps `/`. +pub(crate) fn quote(text: &str) -> String { + quote_with(text, b"/") +} + +fn quote_plus(text: &str) -> String { + quote_with(text, b" ").replace(' ', "+") +} + +/// Python's `urlencode` of ordered string pairs. +pub(crate) fn urlencode(pairs: &[(&str, &str)]) -> String { + pairs + .iter() + .map(|(key, value)| format!("{}={}", quote_plus(key), quote_plus(value))) + .collect::>() + .join("&") +} + +/// `?` and the encoded pairs, or nothing when there are none. +pub(crate) fn query_suffix(pairs: &[(&str, &str)]) -> String { + if pairs.is_empty() { + String::new() + } else { + format!("?{}", urlencode(pairs)) + } +} diff --git a/crates/sqlbuild-core/src/ingestr_command/constants.rs b/crates/sqlbuild-core/src/ingestr_command/constants.rs new file mode 100644 index 0000000000..3de722e87b --- /dev/null +++ b/crates/sqlbuild-core/src/ingestr_command/constants.rs @@ -0,0 +1,37 @@ +//! Ingestr command vocabulary and destination defaults. + +/// The fixed leading arguments of every ingestr run. +pub(crate) const COMMAND_PREFIX: [&str; 2] = ["ingestr", "ingest"]; +/// Progress reporting arguments ingestr runs with. +pub(crate) const COMMAND_PROGRESS: [&str; 3] = ["--yes", "--progress", "log"]; +/// Characters Python's `urllib.parse.quote` never escapes. +pub(crate) const URL_ALWAYS_SAFE: &[u8] = b"_.-~"; +/// Postgres's default port. +pub(crate) const POSTGRES_DEFAULT_PORT: &str = "5432"; +/// SQL Server's default port. +pub(crate) const SQLSERVER_DEFAULT_PORT: &str = "1433"; +/// The SQL Server ODBC driver ingestr uses unless the connection names one. +pub(crate) const SQLSERVER_DEFAULT_DRIVER: &str = "ODBC Driver 18 for SQL Server"; +/// Optional BigQuery query parameters, in URI order. +pub(crate) const BIGQUERY_PARAMETERS: [&str; 3] = + ["location", "credentials_path", "credentials_base64"]; +/// Optional Snowflake query parameters, in URI order. +pub(crate) const SNOWFLAKE_PARAMETERS: [&str; 6] = [ + "warehouse", + "role", + "token", + "private_key", + "private_key_passphrase", + "authenticator", +]; +/// Optional Databricks query parameters after `http_path`, in URI order. +pub(crate) const DATABRICKS_PARAMETERS: [&str; 2] = ["catalog", "schema"]; +/// Optional SQL Server query parameters, in URI order. +pub(crate) const SQLSERVER_PARAMETERS: [&str; 3] = + ["driver", "TrustServerCertificate", "Authentication"]; +/// The SQL Server driver query parameter. +pub(crate) const SQLSERVER_DRIVER_PARAMETER: &str = "driver"; +/// The SQL Server certificate trust query parameter. +pub(crate) const SQLSERVER_TRUST_SERVER_CERTIFICATE_PARAMETER: &str = "TrustServerCertificate"; +/// The MotherDuck database prefix the URI omits. +pub(crate) const MOTHERDUCK_PREFIX: &str = "md:"; diff --git a/crates/sqlbuild-core/src/ingestr_command/errors.rs b/crates/sqlbuild-core/src/ingestr_command/errors.rs new file mode 100644 index 0000000000..bb10c5e272 --- /dev/null +++ b/crates/sqlbuild-core/src/ingestr_command/errors.rs @@ -0,0 +1,7 @@ +//! Ingestr command construction errors. + +/// A destination the ingestr integration cannot address, with its user-facing message. +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct IngestrCommandError { + pub message: String, +} diff --git a/crates/sqlbuild-core/src/ingestr_command/main/format_ingestr_command.rs b/crates/sqlbuild-core/src/ingestr_command/main/format_ingestr_command.rs new file mode 100644 index 0000000000..dc8c3ed27d --- /dev/null +++ b/crates/sqlbuild-core/src/ingestr_command/main/format_ingestr_command.rs @@ -0,0 +1,13 @@ +//! The display form of an ingestr command line. + +use crate::shell_text::main::shell_quote::shell_quote; + +/// The arguments shell-quoted and joined with spaces, like Python's `shlex.join`. +#[must_use] +pub fn format_ingestr_command(arguments: &[String]) -> String { + arguments + .iter() + .map(|argument| shell_quote(argument)) + .collect::>() + .join(" ") +} diff --git a/crates/sqlbuild-core/src/ingestr_command/main/ingestr_command_arguments.rs b/crates/sqlbuild-core/src/ingestr_command/main/ingestr_command_arguments.rs new file mode 100644 index 0000000000..34eabc624e --- /dev/null +++ b/crates/sqlbuild-core/src/ingestr_command/main/ingestr_command_arguments.rs @@ -0,0 +1,43 @@ +//! The `ingestr ingest` arguments for one source run. + +use crate::ingestr_command::constants::{COMMAND_PREFIX, COMMAND_PROGRESS}; +use crate::ingestr_command::models::{IngestrDestination, IngestrSource}; + +/// The ingestr command line that loads `source` into `destination`. +#[must_use] +pub fn ingestr_command_arguments( + source: &IngestrSource, + destination: &IngestrDestination, +) -> Vec { + let mut arguments: Vec = COMMAND_PREFIX.map(str::to_owned).to_vec(); + let pairs: [(&str, &str); 4] = [ + ("--source-uri", &source.source_uri), + ("--source-table", &source.source_table), + ("--dest-uri", &destination.uri), + ("--dest-table", &destination.table), + ]; + let optional: [(&str, Option<&String>); 2] = [ + ("--incremental-strategy", source.strategy.as_ref()), + ("--incremental-key", source.incremental_key.as_ref()), + ]; + for (flag, value) in pairs { + arguments.extend([flag.to_owned(), value.to_owned()]); + } + arguments.extend(COMMAND_PROGRESS.map(str::to_owned)); + for (flag, value) in optional { + if let Some(value) = value { + arguments.extend([flag.to_owned(), value.clone()]); + } + } + for primary_key in &source.primary_key { + arguments.extend(["--primary-key".to_owned(), primary_key.clone()]); + } + if let Some(columns) = &source.columns { + arguments.extend(["--columns".to_owned(), columns.clone()]); + } + if destination.is_reload { + arguments.push("--full-refresh".to_owned()); + } + arguments.extend(source.extra_args.iter().cloned()); + arguments +} diff --git a/crates/sqlbuild-core/src/ingestr_command/main/ingestr_destination_table.rs b/crates/sqlbuild-core/src/ingestr_command/main/ingestr_destination_table.rs new file mode 100644 index 0000000000..67fd30a1f4 --- /dev/null +++ b/crates/sqlbuild-core/src/ingestr_command/main/ingestr_destination_table.rs @@ -0,0 +1,10 @@ +//! The relation an ingestr source run writes. + +/// `schema.target_name` when the source declares a schema, otherwise `target_name`. +#[must_use] +pub fn ingestr_destination_table(schema: Option<&str>, target_name: &str) -> String { + match schema { + Some(schema) => format!("{schema}.{target_name}"), + None => target_name.to_owned(), + } +} diff --git a/crates/sqlbuild-core/src/ingestr_command/main/ingestr_destination_uri.rs b/crates/sqlbuild-core/src/ingestr_command/main/ingestr_destination_uri.rs new file mode 100644 index 0000000000..4320263bbb --- /dev/null +++ b/crates/sqlbuild-core/src/ingestr_command/main/ingestr_destination_uri.rs @@ -0,0 +1,28 @@ +//! The ingestr destination URI for a SQLBuild warehouse connection. + +use crate::ingestr_command::_helpers::destinations::{ + bigquery, databricks, duckdb, motherduck, postgres, snowflake, sqlserver, +}; +use crate::ingestr_command::errors::IngestrCommandError; +use crate::ingestr_command::models::IngestrConnection; + +/// The ingestr destination URI for the connection of the adapter named `adapter_name`. +pub fn ingestr_destination_uri( + adapter_name: &str, + connection: &IngestrConnection, +) -> Result { + match adapter_name { + "duckdb" => duckdb(connection), + "motherduck" => Ok(motherduck(connection)), + "postgres" => postgres(connection), + "bigquery" => bigquery(connection), + "snowflake" => snowflake(connection), + "databricks" => databricks(connection), + "sqlserver" => Ok(sqlserver(connection)), + _ => Err(IngestrCommandError { + message: format!( + "Adapter '{adapter_name}' does not support ingestr integration loaders" + ), + }), + } +} diff --git a/crates/sqlbuild-core/src/ingestr_command/main/mod.rs b/crates/sqlbuild-core/src/ingestr_command/main/mod.rs new file mode 100644 index 0000000000..68329553b8 --- /dev/null +++ b/crates/sqlbuild-core/src/ingestr_command/main/mod.rs @@ -0,0 +1,4 @@ +pub mod format_ingestr_command; +pub mod ingestr_command_arguments; +pub mod ingestr_destination_table; +pub mod ingestr_destination_uri; diff --git a/crates/sqlbuild-core/src/ingestr_command/mod.rs b/crates/sqlbuild-core/src/ingestr_command/mod.rs new file mode 100644 index 0000000000..db623d7553 --- /dev/null +++ b/crates/sqlbuild-core/src/ingestr_command/mod.rs @@ -0,0 +1,10 @@ +//! The `ingestr ingest` command line for one declarative ingestr source and its warehouse. + +pub(crate) mod _helpers; +pub(crate) mod constants; +pub mod errors; +pub mod main; +pub mod models; + +#[cfg(test)] +mod tests; diff --git a/crates/sqlbuild-core/src/ingestr_command/models.rs b/crates/sqlbuild-core/src/ingestr_command/models.rs new file mode 100644 index 0000000000..2ae053876d --- /dev/null +++ b/crates/sqlbuild-core/src/ingestr_command/models.rs @@ -0,0 +1,33 @@ +//! Inputs of one ingestr command line. + +use std::collections::BTreeMap; + +/// The adapter connection settings an ingestr destination URI reads. +#[derive(Clone, Debug, Default, PartialEq, Eq)] +pub struct IngestrConnection { + /// Connection settings whose values are Python strings, unstripped. + pub strings: BTreeMap, + /// `str(connection["port"])` when the connection sets a port. + pub port: Option, +} + +/// A resolved ingestr source configuration. +#[derive(Clone, Debug, Default, PartialEq, Eq)] +pub struct IngestrSource { + pub source_uri: String, + pub source_table: String, + pub strategy: Option, + pub incremental_key: Option, + pub primary_key: Vec, + pub columns: Option, + pub extra_args: Vec, +} + +/// Where one ingestr run writes. +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct IngestrDestination { + pub uri: String, + pub table: String, + /// The run rebuilds the destination from scratch. + pub is_reload: bool, +} diff --git a/crates/sqlbuild-core/src/ingestr_command/tests.rs b/crates/sqlbuild-core/src/ingestr_command/tests.rs new file mode 100644 index 0000000000..ed3facc731 --- /dev/null +++ b/crates/sqlbuild-core/src/ingestr_command/tests.rs @@ -0,0 +1,4 @@ +#[path = "tests/test_ingestr_command.rs"] +mod ingestr_command; +#[path = "tests/test_types.rs"] +mod test_types; diff --git a/crates/sqlbuild-core/src/ingestr_command/tests/test_ingestr_command.rs b/crates/sqlbuild-core/src/ingestr_command/tests/test_ingestr_command.rs new file mode 100644 index 0000000000..98287653ae --- /dev/null +++ b/crates/sqlbuild-core/src/ingestr_command/tests/test_ingestr_command.rs @@ -0,0 +1,124 @@ +//! Ingestr destination URIs and command lines match the Python integration's. + +use crate::ingestr_command::main::format_ingestr_command::format_ingestr_command; +use crate::ingestr_command::main::ingestr_command_arguments::ingestr_command_arguments; +use crate::ingestr_command::main::ingestr_destination_table::ingestr_destination_table; +use crate::ingestr_command::main::ingestr_destination_uri::ingestr_destination_uri; +use crate::ingestr_command::models::{IngestrConnection, IngestrDestination, IngestrSource}; +use crate::ingestr_command::tests::test_types::{CommandTestCase, DestinationUriTestCase}; + +#[test] +fn given_connection_when_building_destination_uri_then_matches_python_quoting() { + let test_cases = [ + DestinationUriTestCase { + description: "duckdb path", + adapter_name: "duckdb", + strings: &[("database", " warehouse.duckdb ")], + port: None, + expected_uri: Ok("duckdb:///warehouse.duckdb"), + }, + DestinationUriTestCase { + description: "motherduck prefix and token", + adapter_name: "motherduck", + strings: &[("database", "md:my db/é"), ("token", "t o/k=n&")], + port: None, + expected_uri: Ok("motherduck://my%20db/%C3%A9?token=t+o%2Fk%3Dn%26"), + }, + DestinationUriTestCase { + description: "postgres with sslmode and port", + adapter_name: "postgres", + strings: &[ + ("user", "ana lyst"), + ("password", "p@ss:/w"), + ("dbname", "orders"), + ], + port: Some("6543"), + expected_uri: Ok("postgresql://ana%20lyst:p%40ss%3A/w@localhost:6543/orders"), + }, + DestinationUriTestCase { + description: "sqlserver defaults", + adapter_name: "sqlserver", + strings: &[("password", "Sqlbuild!2026"), ("database", "analytics")], + port: None, + expected_uri: Ok("mssql://sa:Sqlbuild%212026@localhost:1433/analytics\ + ?driver=ODBC+Driver+18+for+SQL+Server&TrustServerCertificate=yes"), + }, + DestinationUriTestCase { + description: "snowflake requires an account", + adapter_name: "snowflake", + strings: &[("account", " ")], + port: None, + expected_uri: Err( + "Adapter 'snowflake' requires connection.account for ingestr integration loaders", + ), + }, + DestinationUriTestCase { + description: "unsupported adapter", + adapter_name: "fabric", + strings: &[], + port: None, + expected_uri: Err("Adapter 'fabric' does not support ingestr integration loaders"), + }, + ]; + for test_case in test_cases { + let connection: IngestrConnection = IngestrConnection { + strings: test_case + .strings + .iter() + .map(|(key, value)| ((*key).to_owned(), (*value).to_owned())) + .collect(), + port: test_case.port.map(str::to_owned), + }; + let uri: Result = + ingestr_destination_uri(test_case.adapter_name, &connection) + .map_err(|error| error.message); + assert_eq!( + uri.as_deref().map_err(String::as_str), + test_case.expected_uri, + "{}", + test_case.description + ); + } +} + +#[test] +fn given_source_when_building_command_then_orders_arguments_like_python() { + let test_cases = [ + CommandTestCase { + description: "reload with schema", + schema: Some("raw"), + is_reload: true, + expected_display: "ingestr ingest --source-uri 'csv://a b' --source-table t \ + --dest-uri duckdb:///w.duckdb --dest-table raw.orders --yes --progress log \ + --primary-key id --full-refresh '--note=it'\"'\"'s'", + }, + CommandTestCase { + description: "incremental without schema", + schema: None, + is_reload: false, + expected_display: "ingestr ingest --source-uri 'csv://a b' --source-table t \ + --dest-uri duckdb:///w.duckdb --dest-table orders --yes --progress log \ + --primary-key id '--note=it'\"'\"'s'", + }, + ]; + let source: IngestrSource = IngestrSource { + source_uri: "csv://a b".to_owned(), + source_table: "t".to_owned(), + primary_key: vec!["id".to_owned()], + extra_args: vec!["--note=it's".to_owned()], + ..IngestrSource::default() + }; + for test_case in test_cases { + let destination: IngestrDestination = IngestrDestination { + uri: "duckdb:///w.duckdb".to_owned(), + table: ingestr_destination_table(test_case.schema, "orders"), + is_reload: test_case.is_reload, + }; + assert_eq!( + format_ingestr_command(&ingestr_command_arguments(&source, &destination)), + test_case.expected_display, + "{}", + test_case.description + ); + } +} diff --git a/crates/sqlbuild-core/src/ingestr_command/tests/test_types.rs b/crates/sqlbuild-core/src/ingestr_command/tests/test_types.rs new file mode 100644 index 0000000000..e406920491 --- /dev/null +++ b/crates/sqlbuild-core/src/ingestr_command/tests/test_types.rs @@ -0,0 +1,14 @@ +pub(super) struct DestinationUriTestCase { + pub(super) description: &'static str, + pub(super) adapter_name: &'static str, + pub(super) strings: &'static [(&'static str, &'static str)], + pub(super) port: Option<&'static str>, + pub(super) expected_uri: Result<&'static str, &'static str>, +} + +pub(super) struct CommandTestCase { + pub(super) description: &'static str, + pub(super) schema: Option<&'static str>, + pub(super) is_reload: bool, + pub(super) expected_display: &'static str, +} diff --git a/crates/sqlbuild-core/src/lib.rs b/crates/sqlbuild-core/src/lib.rs index a77c5ead45..f099845997 100644 --- a/crates/sqlbuild-core/src/lib.rs +++ b/crates/sqlbuild-core/src/lib.rs @@ -27,6 +27,7 @@ pub mod debug_report; pub mod event_export; pub mod format_command; pub mod freshness_text; +pub mod ingestr_command; pub mod init_command; pub mod integration_results; pub mod janitor_text; @@ -43,6 +44,7 @@ pub mod query_command; pub mod redaction; pub mod scenario_command; pub mod seed_command; +pub mod shell_text; pub mod terminal_events; pub mod test_command; pub mod text; diff --git a/crates/sqlbuild-core/src/shell_text/main/mod.rs b/crates/sqlbuild-core/src/shell_text/main/mod.rs new file mode 100644 index 0000000000..3079457ba4 --- /dev/null +++ b/crates/sqlbuild-core/src/shell_text/main/mod.rs @@ -0,0 +1 @@ +pub mod shell_quote; diff --git a/crates/sqlbuild-core/src/shell_text/main/shell_quote.rs b/crates/sqlbuild-core/src/shell_text/main/shell_quote.rs new file mode 100644 index 0000000000..3e7e57a811 --- /dev/null +++ b/crates/sqlbuild-core/src/shell_text/main/shell_quote.rs @@ -0,0 +1,16 @@ +//! Python's `shlex.quote`. + +/// Python's `shlex.quote(text)`: safe ASCII text unchanged, anything else single-quoted. +#[must_use] +pub fn shell_quote(text: &str) -> String { + if text.is_empty() { + return "''".to_owned(); + } + let safe: bool = text + .chars() + .all(|character| character.is_ascii_alphanumeric() || "_@%+=:,./-".contains(character)); + if safe { + return text.to_owned(); + } + format!("'{}'", text.replace('\'', "'\"'\"'")) +} diff --git a/crates/sqlbuild-core/src/shell_text/mod.rs b/crates/sqlbuild-core/src/shell_text/mod.rs new file mode 100644 index 0000000000..66ee6a2194 --- /dev/null +++ b/crates/sqlbuild-core/src/shell_text/mod.rs @@ -0,0 +1,3 @@ +//! Shell-quoted command text. + +pub mod main; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/load/ingestr_command.rs b/crates/sqlbuild-python/src/bindings/_helpers/load/ingestr_command.rs new file mode 100644 index 0000000000..3e64a70e39 --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/load/ingestr_command.rs @@ -0,0 +1,218 @@ +//! The ingestr command line for one declarative ingestr source, built from its runtime values. + +use std::collections::BTreeMap; + +use pyo3::prelude::{ + Bound, PyAny, PyAnyMethods, PyErr, PyModule, PyModuleMethods, PyResult, Python, +}; +use pyo3::types::{PyDict, PyDictMethods, PyList, PyString, PyTuple}; +use pyo3::{pyfunction, wrap_pyfunction}; +use sqlbuild_core::ingestr_command::main::format_ingestr_command::format_ingestr_command as native_format_ingestr_command; +use sqlbuild_core::ingestr_command::main::ingestr_command_arguments::ingestr_command_arguments; +use sqlbuild_core::ingestr_command::main::ingestr_destination_table::ingestr_destination_table; +use sqlbuild_core::ingestr_command::main::ingestr_destination_uri::ingestr_destination_uri; +use sqlbuild_core::ingestr_command::models::{ + IngestrConnection, IngestrDestination, IngestrSource, +}; +use sqlbuild_model_config::templates::models::TemplateOptions; + +use crate::bindings::_helpers::boundary::keyword_request::Request; +use crate::bindings::_helpers::compile_driver::python_calls::imported; +use crate::bindings::_helpers::model_config::template_data::expanded_template_data; + +const INGESTR_ERROR: (&str, &str) = ( + "sqlbuild.integrations.ingestr.exceptions", + "IngestrIntegrationError", +); +const INGESTR_SOURCE_CONFIG: (&str, &str) = ( + "sqlbuild.integrations.ingestr.models", + "IngestrSourceConfig", +); +const INGESTR_KIND: (&str, &str) = ( + "sqlbuild.integrations.ingestr.constants", + "INGESTR_INTEGRATION_KIND", +); +const TEMPLATE_LABEL: &str = "ingestr source config"; +const TEMPLATE_FIELDS: [&str; 7] = [ + "source_uri", + "source_table", + "strategy", + "incremental_key", + "primary_key", + "columns", + "extra_args", +]; +const LIST_FIELDS: [&str; 2] = ["primary_key", "extra_args"]; + +/// An `IngestrIntegrationError`, or the failure to construct one. +fn ingestr_error(py: Python<'_>, message: String) -> PyErr { + imported(py, INGESTR_ERROR.0, INGESTR_ERROR.1) + .and_then(|class| class.call1((message,))) + .map_or_else(|error| error, PyErr::from_value) +} + +/// The source's ingestr config, raising when the source declares no valid one. +fn ingestr_config<'py>(source_entry: &Bound<'py, PyAny>) -> PyResult> { + let py: Python<'py> = source_entry.py(); + let name: String = source_entry.getattr("name")?.str()?.extract()?; + let loader: Bound<'py, PyAny> = source_entry.getattr("integration_loader")?; + if loader.is_none() + || !loader + .getattr("kind")? + .eq(imported(py, INGESTR_KIND.0, INGESTR_KIND.1)?)? + { + return Err(ingestr_error( + py, + format!("Source '{name}' does not define ingestr config"), + )); + } + let config: Bound<'py, PyAny> = loader.getattr("config")?; + if !config.is_instance(&imported( + py, + INGESTR_SOURCE_CONFIG.0, + INGESTR_SOURCE_CONFIG.1, + )?)? { + return Err(ingestr_error( + py, + format!("Source '{name}' defines invalid ingestr config"), + )); + } + Ok(config) +} + +/// `str(value)`, or `None` when the value is `None` or renders empty. +fn optional_text(value: Option>) -> PyResult> { + let Some(value) = value.filter(|value| !value.is_none()) else { + return Ok(None); + }; + let text: String = value.str()?.extract()?; + Ok((!text.is_empty()).then_some(text)) +} + +/// One string per item of a list value, or the value itself as one string. +fn text_list(value: Option>) -> PyResult> { + let Some(value) = value.filter(|value| !value.is_none()) else { + return Ok(Vec::new()); + }; + if value.is_instance_of::() { + return Ok(vec![value.extract()?]); + } + match value.try_iter() { + Ok(items) => items + .map(|item| item?.str()?.extract()) + .collect::>>(), + Err(_) => Ok(vec![value.str()?.extract()?]), + } +} + +/// The source config with `${...}` templates expanded against vars and run context. +fn resolved_source( + config: &Bound<'_, PyAny>, + vars: &Bound<'_, PyDict>, + (environment, run_id): (Option<&str>, &str), +) -> PyResult { + let py: Python<'_> = config.py(); + let value: Bound<'_, PyDict> = PyDict::new(py); + for field in TEMPLATE_FIELDS { + let raw: Bound<'_, PyAny> = config.getattr(field)?; + if LIST_FIELDS.contains(&field) { + value.set_item( + field, + PyList::new(py, raw.try_iter()?.collect::>>()?)?, + )?; + } else { + value.set_item(field, raw)?; + } + } + let context: Bound<'_, PyDict> = PyDict::new(py); + context.set_item("environment", environment)?; + context.set_item("run_id", run_id)?; + let expanded: Bound<'_, PyAny> = expanded_template_data( + value.as_any(), + (vars, &context), + ( + TemplateOptions { + allow_context: true, + preserve_context_tokens: false, + preserve_unknown_context: true, + }, + TEMPLATE_LABEL, + ), + )?; + let expanded: &Bound<'_, PyDict> = expanded.downcast()?; + let get = |key: &str| expanded.get_item(key); + Ok(IngestrSource { + source_uri: expanded.as_any().get_item("source_uri")?.str()?.extract()?, + source_table: expanded + .as_any() + .get_item("source_table")? + .str()? + .extract()?, + strategy: optional_text(get("strategy")?)?, + incremental_key: optional_text(get("incremental_key")?)?, + primary_key: text_list(get("primary_key")?)?, + columns: optional_text(get("columns")?)?, + extra_args: text_list(get("extra_args")?)?, + }) +} + +/// The string settings of a connection and its rendered port. +fn connection(connection_config: &Bound<'_, PyDict>) -> PyResult { + let mut strings: BTreeMap = BTreeMap::new(); + for (key, value) in connection_config.iter() { + if let (Ok(key), Ok(value)) = (key.extract::(), value.downcast::()) { + strings.insert(key, value.extract()?); + } + } + let port: Option = connection_config + .get_item("port")? + .map(|port| port.str()?.extract()) + .transpose()?; + Ok(IngestrConnection { strings, port }) +} + +/// The ingestr CLI command for one source entry. +#[pyfunction] +#[pyo3(signature = (**request))] +fn build_ingestr_command<'py>( + request: Option<&Bound<'py, PyDict>>, +) -> PyResult> { + let request: Request<'_, 'py> = Request::new("build_ingestr_command", request)?; + let py: Python<'py> = request.py(); + let source_entry: Bound<'py, PyAny> = request.get("source_entry")?; + let config: Bound<'py, PyAny> = ingestr_config(&source_entry)?; + let environment: Option = request.get("environment")?.extract()?; + let run_id: String = request.get("run_id")?.extract()?; + let source: IngestrSource = resolved_source( + &config, + request.get("vars")?.downcast()?, + (environment.as_deref(), &run_id), + )?; + let adapter_name: String = request.get("adapter_name")?.extract()?; + let uri: String = ingestr_destination_uri( + &adapter_name, + &connection(request.get("connection_config")?.downcast()?)?, + ) + .map_err(|error| ingestr_error(py, error.message))?; + let schema: Option = source_entry.getattr("schema")?.extract()?; + let destination_name: String = request.get("destination_name")?.extract()?; + let destination: IngestrDestination = IngestrDestination { + uri, + table: ingestr_destination_table(schema.as_deref(), &destination_name), + is_reload: request.get("is_reload")?.is_truthy()?, + }; + PyTuple::new(py, ingestr_command_arguments(&source, &destination)) +} + +/// The display-safe, shell-quoted form of an ingestr command. +#[pyfunction] +#[pyo3(signature = (command, /))] +fn format_ingestr_command(command: Vec) -> String { + native_format_ingestr_command(&command) +} + +pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { + module.add_function(wrap_pyfunction!(build_ingestr_command, module)?)?; + module.add_function(wrap_pyfunction!(format_ingestr_command, module)?)?; + Ok(()) +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/load/mod.rs b/crates/sqlbuild-python/src/bindings/_helpers/load/mod.rs index 68a14122c2..da84870903 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/load/mod.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/load/mod.rs @@ -1,3 +1,4 @@ +pub(crate) mod ingestr_command; pub(crate) mod load_command; pub(crate) mod load_progress; pub(crate) mod load_selection; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/load/registration.rs b/crates/sqlbuild-python/src/bindings/_helpers/load/registration.rs index 198a9ed693..a16bc8dd0e 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/load/registration.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/load/registration.rs @@ -2,9 +2,12 @@ use pyo3::prelude::{Bound, PyModule, PyResult}; -use crate::bindings::_helpers::load::{load_command, load_progress, load_selection}; +use crate::bindings::_helpers::load::{ + ingestr_command, load_command, load_progress, load_selection, +}; pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { + ingestr_command::register(module)?; load_command::register(module)?; load_progress::register(module)?; load_selection::register(module) diff --git a/crates/sqlbuild-refactor/src/refactor_command/_helpers/quoting.rs b/crates/sqlbuild-refactor/src/refactor_command/_helpers/quoting.rs index 6237dd184e..7c7455d18d 100644 --- a/crates/sqlbuild-refactor/src/refactor_command/_helpers/quoting.rs +++ b/crates/sqlbuild-refactor/src/refactor_command/_helpers/quoting.rs @@ -1,18 +1,6 @@ //! Command lines quoted for the shell of the platform they are printed on. -/// Python's `shlex.quote`. -fn shell_quote(text: &str) -> String { - if text.is_empty() { - return "''".to_owned(); - } - let safe: bool = text - .chars() - .all(|character| character.is_ascii_alphanumeric() || "_@%+=:,./-".contains(character)); - if safe { - return text.to_owned(); - } - format!("'{}'", text.replace('\'', "'\"'\"'")) -} +use sqlbuild_core::shell_text::main::shell_quote::shell_quote; /// Python's `subprocess.list2cmdline` for one argument. fn windows_quote(argument: &str) -> String { diff --git a/crates/sqlbuild-scopes/src/scope_text/_helpers/layout.rs b/crates/sqlbuild-scopes/src/scope_text/_helpers/layout.rs index 4541c5dc2c..6d2af0501d 100644 --- a/crates/sqlbuild-scopes/src/scope_text/_helpers/layout.rs +++ b/crates/sqlbuild-scopes/src/scope_text/_helpers/layout.rs @@ -1,6 +1,7 @@ //! Fields, headers, connectors, paths and follow-up commands of scope text output. use sqlbuild_core::cli_text::models::CliStyle; +use sqlbuild_core::shell_text::main::shell_quote::shell_quote; use crate::scope_report::models::{DiagnosticFacts, ReportDiagnostic, ScopeSection}; use crate::scope_text::constants::{ @@ -166,20 +167,6 @@ pub(crate) fn diagnostic_lines( lines } -/// Python's `shlex.quote`. -fn shell_quote(text: &str) -> String { - if text.is_empty() { - return "''".to_owned(); - } - let safe: bool = text - .chars() - .all(|character| character.is_ascii_alphanumeric() || "_@%+=:,./-".contains(character)); - if safe { - return text.to_owned(); - } - format!("'{}'", text.replace('\'', "'\"'\"'")) -} - /// What a follow-up command adds to the request's own options. #[derive(Clone, Copy, Default)] pub(crate) struct FollowUp<'a> { diff --git a/python-boundary.toml b/python-boundary.toml index a73c16ae83..68b27a0b12 100644 --- a/python-boundary.toml +++ b/python-boundary.toml @@ -407,7 +407,7 @@ note = "Loader runtime that runs inside user loaders; synthetic loader record bu [[stays]] path = "integrations/ingestr" reason = "user-code" -note = "Loader runtime that runs inside the executor load phase; synthetic loader record building and naming are native." +note = "Loader runtime that runs the ingestr subprocess inside the executor load phase; synthetic loader record building, naming, destination URIs and command lines are native." [[stays]] path = "integrations/dbt/_helpers/profile/load.py" diff --git a/src/sqlbuild/_native.pyi b/src/sqlbuild/_native.pyi index b171957ba5..5fb843c57b 100644 --- a/src/sqlbuild/_native.pyi +++ b/src/sqlbuild/_native.pyi @@ -4029,6 +4029,18 @@ def format_seed_execution_json( *, results: tuple[SeedExecutionResult, ...], plan: PlanOutput ) -> str: ... def format_load_execution_json(*, results: tuple[LoadExecutionResult, ...]) -> str: ... +def build_ingestr_command( + *, + source_entry: SourceEntry, + adapter_name: str, + connection_config: dict[str, object], + destination_name: str, + vars: dict[str, object], + environment: str | None, + run_id: str, + is_reload: bool, +) -> tuple[str, ...]: ... +def format_ingestr_command(command: Sequence[str], /) -> str: ... def format_clone_execution_json( *, result: CloneExecutionResult, resource_types_by_name: Mapping[str, str] ) -> str: ... diff --git a/src/sqlbuild/integrations/ingestr/_helpers/destination.py b/src/sqlbuild/integrations/ingestr/_helpers/destination.py deleted file mode 100644 index 5ccdb6113c..0000000000 --- a/src/sqlbuild/integrations/ingestr/_helpers/destination.py +++ /dev/null @@ -1,177 +0,0 @@ -"""Build ingestr destination URIs from SQLBuild adapter connection config.""" - -from __future__ import annotations - -from urllib.parse import quote, urlencode - -from sqlbuild.adapter.contract.types import BuiltinAdapter -from sqlbuild.integrations.ingestr.constants import ( - SQLSERVER_DRIVER_PARAMETER, - SQLSERVER_TRUST_SERVER_CERTIFICATE_PARAMETER, -) -from sqlbuild.integrations.ingestr.exceptions import IngestrIntegrationError - - -def build_destination_uri(*, adapter_name: str, connection_config: dict[str, object]) -> str: - """Return the ingestr destination URI for a SQLBuild connection.""" - - if adapter_name == BuiltinAdapter.DUCKDB.value: - return _duckdb_uri(connection_config) - if adapter_name == BuiltinAdapter.MOTHERDUCK.value: - return _motherduck_uri(connection_config) - if adapter_name == BuiltinAdapter.POSTGRES.value: - return _postgres_uri(connection_config) - if adapter_name == BuiltinAdapter.BIGQUERY.value: - return _bigquery_uri(connection_config) - if adapter_name == BuiltinAdapter.SNOWFLAKE.value: - return _snowflake_uri(connection_config) - if adapter_name == BuiltinAdapter.DATABRICKS.value: - return _databricks_uri(connection_config) - if adapter_name == BuiltinAdapter.SQLSERVER.value: - return _sqlserver_uri(connection_config) - raise IngestrIntegrationError( - f"Adapter '{adapter_name}' does not support ingestr integration loaders" - ) - - -def _required_string(*, config: dict[str, object], key: str, adapter_name: str) -> str: - value: object | None = config.get(key) - if not isinstance(value, str) or not value.strip(): - raise IngestrIntegrationError( - f"Adapter '{adapter_name}' requires connection.{key} for ingestr integration loaders" - ) - return value.strip() - - -def _optional_string(*, config: dict[str, object], key: str) -> str | None: - value: object | None = config.get(key) - if not isinstance(value, str) or not value.strip(): - return None - return value.strip() - - -def _duckdb_uri(config: dict[str, object]) -> str: - database: str = _required_string( - config=config, key="database", adapter_name=BuiltinAdapter.DUCKDB.value - ) - return f"duckdb:///{database}" - - -def _motherduck_uri(config: dict[str, object]) -> str: - database: str = _optional_string(config=config, key="database") or "" - if database.startswith("md:"): - database = database.removeprefix("md:") - token: str | None = _optional_string(config=config, key="token") - query: str = f"?{urlencode({'token': token})}" if token else "" - return f"motherduck://{quote(database)}{query}" - - -def _postgres_uri(config: dict[str, object]) -> str: - host: str = _optional_string(config=config, key="host") or "localhost" - port: object = config.get("port", 5432) - user: str = _required_string( - config=config, key="user", adapter_name=BuiltinAdapter.POSTGRES.value - ) - password: str = _optional_string(config=config, key="password") or "" - database: str = _optional_string(config=config, key="dbname") or _required_string( - config=config, key="database", adapter_name=BuiltinAdapter.POSTGRES.value - ) - sslmode: str | None = _optional_string(config=config, key="sslmode") - query: str = f"?{urlencode({'sslmode': sslmode})}" if sslmode else "" - return f"postgresql://{quote(user)}:{quote(password)}@{host}:{port}/{quote(database)}{query}" - - -def _bigquery_uri(config: dict[str, object]) -> str: - project: str = _optional_string(config=config, key="project") or _required_string( - config=config, key="database", adapter_name=BuiltinAdapter.BIGQUERY.value - ) - params: dict[str, str] = {} - for key in ("location", "credentials_path", "credentials_base64"): - value: str | None = _optional_string(config=config, key=key) - if value is not None: - params[key] = value - query: str = f"?{urlencode(params)}" if params else "" - return f"bigquery://{project}{query}" - - -def _snowflake_uri(config: dict[str, object]) -> str: - account: str = _required_string( - config=config, key="account", adapter_name=BuiltinAdapter.SNOWFLAKE.value - ) - user: str = _required_string( - config=config, key="user", adapter_name=BuiltinAdapter.SNOWFLAKE.value - ) - password: str = _optional_string(config=config, key="password") or "" - database: str = _required_string( - config=config, key="database", adapter_name=BuiltinAdapter.SNOWFLAKE.value - ) - schema: str = _required_string( - config=config, key="schema", adapter_name=BuiltinAdapter.SNOWFLAKE.value - ) - params: dict[str, str] = {} - for key in ( - "warehouse", - "role", - "token", - "private_key", - "private_key_passphrase", - "authenticator", - ): - value: str | None = _optional_string(config=config, key=key) - if value is not None: - params[key] = value - query: str = f"?{urlencode(params)}" if params else "" - return ( - f"snowflake://{quote(user)}:{quote(password)}@{account}/" - f"{quote(database)}/{quote(schema)}{query}" - ) - - -def _databricks_uri(config: dict[str, object]) -> str: - host: str = _required_string( - config=config, key="server_hostname", adapter_name=BuiltinAdapter.DATABRICKS.value - ) - token: str = _required_string( - config=config, key="token", adapter_name=BuiltinAdapter.DATABRICKS.value - ) - params: dict[str, str] = { - "http_path": _required_string( - config=config, key="http_path", adapter_name=BuiltinAdapter.DATABRICKS.value - ) - } - for key in ("catalog", "schema"): - value: str | None = _optional_string(config=config, key=key) - if value is not None: - params[key] = value - return f"databricks://token:{quote(token)}@{host}?{urlencode(params)}" - - -def _sqlserver_uri(config: dict[str, object]) -> str: - host: str = ( - _optional_string(config=config, key="host") - or _optional_string(config=config, key="server") - or "localhost" - ) - port: object = config.get("port", 1433) - user: str = ( - _optional_string(config=config, key="user") - or _optional_string(config=config, key="username") - or "sa" - ) - password: str = _optional_string(config=config, key="password") or "" - database: str = ( - _optional_string(config=config, key="database") - or _optional_string(config=config, key="dbname") - or "master" - ) - params: dict[str, str] = {} - for key in ("driver", "TrustServerCertificate", "Authentication"): - value: str | None = _optional_string(config=config, key=key) - if value is not None: - params[key] = value - if SQLSERVER_DRIVER_PARAMETER not in params: - params[SQLSERVER_DRIVER_PARAMETER] = "ODBC Driver 18 for SQL Server" - if SQLSERVER_TRUST_SERVER_CERTIFICATE_PARAMETER not in params: - params[SQLSERVER_TRUST_SERVER_CERTIFICATE_PARAMETER] = "yes" - query: str = f"?{urlencode(params)}" if params else "" - return f"mssql://{quote(user)}:{quote(password)}@{host}:{port}/{quote(database)}{query}" diff --git a/src/sqlbuild/integrations/ingestr/_helpers/output.py b/src/sqlbuild/integrations/ingestr/_helpers/output.py index a9eed714b8..4c7264204a 100644 --- a/src/sqlbuild/integrations/ingestr/_helpers/output.py +++ b/src/sqlbuild/integrations/ingestr/_helpers/output.py @@ -2,18 +2,10 @@ from __future__ import annotations -from shlex import quote - from sqlbuild.executor.load.models import LoaderContext from sqlbuild.integrations.ingestr.models import IngestrCommandResult -def format_ingestr_command(command: tuple[str, ...]) -> str: - """Return a display-safe ingestr command line.""" - - return " ".join(quote(part) for part in command) - - def record_ingestr_output(*, ctx: LoaderContext, result: IngestrCommandResult) -> None: """Attach captured ingestr output to the loader lifecycle.""" diff --git a/src/sqlbuild/integrations/ingestr/_helpers/runner.py b/src/sqlbuild/integrations/ingestr/_helpers/runner.py index 24cd839a74..1f33bc29fa 100644 --- a/src/sqlbuild/integrations/ingestr/_helpers/runner.py +++ b/src/sqlbuild/integrations/ingestr/_helpers/runner.py @@ -8,7 +8,7 @@ from os import environ from pathlib import Path -from sqlbuild.integrations.ingestr._helpers.output import format_ingestr_command +from sqlbuild._native import format_ingestr_command from sqlbuild.integrations.ingestr.exceptions import IngestrIntegrationError from sqlbuild.integrations.ingestr.models import IngestrCommandResult from sqlbuild.runtime.observability.classes.operation_lifecycle import OperationLifecycle diff --git a/src/sqlbuild/integrations/ingestr/constants.py b/src/sqlbuild/integrations/ingestr/constants.py index 80ead410bc..47b3f3f671 100644 --- a/src/sqlbuild/integrations/ingestr/constants.py +++ b/src/sqlbuild/integrations/ingestr/constants.py @@ -3,5 +3,3 @@ from __future__ import annotations INGESTR_INTEGRATION_KIND: str = "ingestr" -SQLSERVER_DRIVER_PARAMETER: str = "driver" -SQLSERVER_TRUST_SERVER_CERTIFICATE_PARAMETER: str = "TrustServerCertificate" diff --git a/src/sqlbuild/integrations/ingestr/main/_loaders.py b/src/sqlbuild/integrations/ingestr/main/_loaders.py index 8516b83633..d5704629c2 100644 --- a/src/sqlbuild/integrations/ingestr/main/_loaders.py +++ b/src/sqlbuild/integrations/ingestr/main/_loaders.py @@ -4,9 +4,8 @@ from collections.abc import Callable -from sqlbuild._native import SourceEntry +from sqlbuild._native import SourceEntry, build_ingestr_command from sqlbuild.executor.load.models import LoaderContext -from sqlbuild.integrations.ingestr._helpers.command import build_ingestr_command from sqlbuild.integrations.ingestr._helpers.output import record_ingestr_output from sqlbuild.integrations.ingestr._helpers.runner import run_ingestr_command from sqlbuild.integrations.ingestr.exceptions import IngestrIntegrationError @@ -23,9 +22,7 @@ def run(ctx: LoaderContext) -> None: source_entry=source_entry, adapter_name=ctx.adapter.adapter_name, connection_config=ctx.connection_config, - destination_table=_destination_table( - source_entry=source_entry, target_name=ctx.destination_name - ), + destination_name=ctx.destination_name, vars=ctx.vars, environment=ctx.target, run_id=ctx.run_id, @@ -42,9 +39,3 @@ def run(ctx: LoaderContext) -> None: run.__name__ = name return run - - -def _destination_table(*, source_entry: SourceEntry, target_name: str) -> str: - if source_entry.schema is not None: - return f"{source_entry.schema}.{target_name}" - return target_name diff --git a/tests/unit/src/sqlbuild/integrations/ingestr/_test_types.py b/tests/unit/src/sqlbuild/integrations/ingestr/_test_types.py index 591cdf8683..685f1fe2a9 100644 --- a/tests/unit/src/sqlbuild/integrations/ingestr/_test_types.py +++ b/tests/unit/src/sqlbuild/integrations/ingestr/_test_types.py @@ -4,6 +4,8 @@ from dataclasses import dataclass +from sqlbuild.spec.contracts.models import SourceEntry + @dataclass(frozen=True) class IngestrCommandTestCase: @@ -13,6 +15,15 @@ class IngestrCommandTestCase: expected_command: tuple[str, ...] +@dataclass(frozen=True) +class IngestrCommandErrorTestCase: + description: str + source_entry: SourceEntry + adapter_name: str + connection_config: dict[str, object] + expected_message: str + + @dataclass(frozen=True) class IngestrLoaderDiscoveryTestCase: description: str diff --git a/tests/unit/src/sqlbuild/integrations/ingestr/test_command.py b/tests/unit/src/sqlbuild/integrations/ingestr/test_command.py index edc33a8eb8..1f7a664fde 100644 --- a/tests/unit/src/sqlbuild/integrations/ingestr/test_command.py +++ b/tests/unit/src/sqlbuild/integrations/ingestr/test_command.py @@ -4,10 +4,15 @@ import pytest -from sqlbuild._native import IntegrationLoaderConfig, SourceEntry +from sqlbuild._native import IntegrationLoaderConfig, SourceEntry, build_ingestr_command from sqlbuild.integrations.ingestr._helpers.command import build_ingestr_command +from sqlbuild.integrations.ingestr.exceptions import IngestrIntegrationError from sqlbuild.integrations.ingestr.models import IngestrSourceConfig -from tests.unit.src.sqlbuild.integrations.ingestr._test_types import IngestrCommandTestCase +from sqlbuild.spec.contracts.models import IntegrationLoaderConfig, SourceEntry +from tests.unit.src.sqlbuild.integrations.ingestr._test_types import ( + IngestrCommandErrorTestCase, + IngestrCommandTestCase, +) @pytest.mark.parametrize( @@ -108,7 +113,7 @@ def test_given_ingestr_source_when_building_command_then_it_uses_sqlbuild_destin source_entry=source_entry, adapter_name=test_case.adapter_name, connection_config=test_case.connection_config, - destination_table="raw.orders", + destination_name="orders", vars={"token": "secret"}, environment="dev", run_id="run-1", @@ -116,3 +121,62 @@ def test_given_ingestr_source_when_building_command_then_it_uses_sqlbuild_destin ) assert command == test_case.expected_command + + +@pytest.mark.parametrize( + "test_case", + [ + IngestrCommandErrorTestCase( + description="unsupported adapter", + source_entry=SourceEntry( + name="raw_orders", + integration_loader=IntegrationLoaderConfig( + kind="ingestr", + config=IngestrSourceConfig(source_uri="csv://x", source_table="orders"), + ), + ), + adapter_name="fabric", + connection_config={}, + expected_message="Adapter 'fabric' does not support ingestr integration loaders", + ), + IngestrCommandErrorTestCase( + description="blank required connection setting", + source_entry=SourceEntry( + name="raw_orders", + integration_loader=IntegrationLoaderConfig( + kind="ingestr", + config=IngestrSourceConfig(source_uri="csv://x", source_table="orders"), + ), + ), + adapter_name="postgres", + connection_config={"user": " "}, + expected_message=( + "Adapter 'postgres' requires connection.user for ingestr integration loaders" + ), + ), + IngestrCommandErrorTestCase( + description="source without ingestr config", + source_entry=SourceEntry(name="raw_orders"), + adapter_name="duckdb", + connection_config={"database": "warehouse.duckdb"}, + expected_message="Source 'raw_orders' does not define ingestr config", + ), + ], + ids=lambda case: case.description, +) +def test_given_unbuildable_ingestr_source_when_building_command_then_raises_integration_error( + test_case: IngestrCommandErrorTestCase, +) -> None: + with pytest.raises(IngestrIntegrationError) as raised: + _ = build_ingestr_command( + source_entry=test_case.source_entry, + adapter_name=test_case.adapter_name, + connection_config=test_case.connection_config, + destination_name="orders", + vars={}, + environment=None, + run_id="run-1", + is_reload=False, + ) + + assert str(raised.value) == test_case.expected_message From b0eea5b31b3e16d98df9fbf00e59d1cb14884845 Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Thu, 15 Oct 2026 21:03:59 +0100 Subject: [PATCH 19/48] chore: drop ingestr command module kept alive by integration --- .../integrations/ingestr/_helpers/command.py | 124 ------------------ .../integrations/ingestr/test_command.py | 2 - 2 files changed, 126 deletions(-) delete mode 100644 src/sqlbuild/integrations/ingestr/_helpers/command.py diff --git a/src/sqlbuild/integrations/ingestr/_helpers/command.py b/src/sqlbuild/integrations/ingestr/_helpers/command.py deleted file mode 100644 index 2c2a6f5531..0000000000 --- a/src/sqlbuild/integrations/ingestr/_helpers/command.py +++ /dev/null @@ -1,124 +0,0 @@ -"""Command construction helpers for ingestr integration loaders.""" - -from __future__ import annotations - -from collections.abc import Iterable, Mapping -from dataclasses import replace -from typing import cast - -from sqlbuild._native import IntegrationLoaderConfig, SourceEntry, expand_template_data -from sqlbuild.integrations.ingestr._helpers.destination import build_destination_uri -from sqlbuild.integrations.ingestr.constants import INGESTR_INTEGRATION_KIND -from sqlbuild.integrations.ingestr.exceptions import IngestrIntegrationError -from sqlbuild.integrations.ingestr.models import IngestrSourceConfig - - -def resolve_ingestr_config( - *, config: IngestrSourceConfig, vars: dict[str, object], environment: str | None, run_id: str -) -> IngestrSourceConfig: - """Resolve supported templates in ingestr source config values.""" - - values: object = expand_template_data( - value={ - "source_uri": config.source_uri, - "source_table": config.source_table, - "strategy": config.strategy, - "incremental_key": config.incremental_key, - "primary_key": list(config.primary_key), - "columns": config.columns, - "extra_args": list(config.extra_args), - }, - variables=vars, - context_values={"environment": environment, "run_id": run_id}, - context_label="ingestr source config", - allow_context=True, - preserve_context_tokens=False, - preserve_unknown_context=True, - ) - if not isinstance(values, Mapping): - return config - resolved_values: Mapping[str, object] = cast(Mapping[str, object], values) - return replace( - config, - source_uri=str(resolved_values["source_uri"]), - source_table=str(resolved_values["source_table"]), - strategy=_optional_str(resolved_values.get("strategy")), - incremental_key=_optional_str(resolved_values.get("incremental_key")), - primary_key=_string_tuple(resolved_values.get("primary_key")), - columns=_optional_str(resolved_values.get("columns")), - extra_args=_string_tuple(resolved_values.get("extra_args")), - ) - - -def build_ingestr_command( - *, - source_entry: SourceEntry, - adapter_name: str, - connection_config: dict[str, object], - destination_table: str, - vars: dict[str, object], - environment: str | None, - run_id: str, - is_reload: bool, -) -> tuple[str, ...]: - """Build the ingestr CLI command for one source entry.""" - - integration_loader: IntegrationLoaderConfig | None = source_entry.integration_loader - if integration_loader is None or integration_loader.kind != INGESTR_INTEGRATION_KIND: - raise IngestrIntegrationError( - f"Source '{source_entry.name}' does not define ingestr config" - ) - if not isinstance(integration_loader.config, IngestrSourceConfig): - raise IngestrIntegrationError( - f"Source '{source_entry.name}' defines invalid ingestr config" - ) - config: IngestrSourceConfig = resolve_ingestr_config( - config=integration_loader.config, - vars=vars, - environment=environment, - run_id=run_id, - ) - args: list[str] = [ - "ingestr", - "ingest", - "--source-uri", - config.source_uri, - "--source-table", - config.source_table, - "--dest-uri", - build_destination_uri(adapter_name=adapter_name, connection_config=connection_config), - "--dest-table", - destination_table, - "--yes", - "--progress", - "log", - ] - if config.strategy is not None: - args.extend(("--incremental-strategy", config.strategy)) - if config.incremental_key is not None: - args.extend(("--incremental-key", config.incremental_key)) - for primary_key in config.primary_key: - args.extend(("--primary-key", primary_key)) - if config.columns is not None: - args.extend(("--columns", config.columns)) - if is_reload: - args.append("--full-refresh") - args.extend(config.extra_args) - return tuple(args) - - -def _optional_str(value: object | None) -> str | None: - if value is None: - return None - text: str = str(value) - return text if text else None - - -def _string_tuple(value: object | None) -> tuple[str, ...]: - if value is None: - return () - if isinstance(value, str): - return (value,) - if isinstance(value, Iterable): - return tuple(str(item) for item in value) - return (str(value),) diff --git a/tests/unit/src/sqlbuild/integrations/ingestr/test_command.py b/tests/unit/src/sqlbuild/integrations/ingestr/test_command.py index 1f7a664fde..bced98e655 100644 --- a/tests/unit/src/sqlbuild/integrations/ingestr/test_command.py +++ b/tests/unit/src/sqlbuild/integrations/ingestr/test_command.py @@ -5,10 +5,8 @@ import pytest from sqlbuild._native import IntegrationLoaderConfig, SourceEntry, build_ingestr_command -from sqlbuild.integrations.ingestr._helpers.command import build_ingestr_command from sqlbuild.integrations.ingestr.exceptions import IngestrIntegrationError from sqlbuild.integrations.ingestr.models import IngestrSourceConfig -from sqlbuild.spec.contracts.models import IntegrationLoaderConfig, SourceEntry from tests.unit.src.sqlbuild.integrations.ingestr._test_types import ( IngestrCommandErrorTestCase, IngestrCommandTestCase, From 4f18a58344ccc236e2329b3af2aa8464b0d6c10d Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Thu, 15 Oct 2026 21:04:12 +0100 Subject: [PATCH 20/48] test: import native SourceEntry in ingestr test types --- tests/unit/src/sqlbuild/integrations/ingestr/_test_types.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/unit/src/sqlbuild/integrations/ingestr/_test_types.py b/tests/unit/src/sqlbuild/integrations/ingestr/_test_types.py index 685f1fe2a9..ac2c01efab 100644 --- a/tests/unit/src/sqlbuild/integrations/ingestr/_test_types.py +++ b/tests/unit/src/sqlbuild/integrations/ingestr/_test_types.py @@ -4,7 +4,7 @@ from dataclasses import dataclass -from sqlbuild.spec.contracts.models import SourceEntry +from sqlbuild._native import SourceEntry @dataclass(frozen=True) From ff4fbc4d480d6d4bd35d367d6499283c3ef5c794 Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Thu, 15 Oct 2026 21:34:24 +0100 Subject: [PATCH 21/48] style: sort query artifact imports --- src/sqlbuild/executor/diff/_helpers/query_artifacts.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/sqlbuild/executor/diff/_helpers/query_artifacts.py b/src/sqlbuild/executor/diff/_helpers/query_artifacts.py index 8eca81bac8..2d88fa29dc 100644 --- a/src/sqlbuild/executor/diff/_helpers/query_artifacts.py +++ b/src/sqlbuild/executor/diff/_helpers/query_artifacts.py @@ -8,10 +8,10 @@ from typing import Any from sqlbuild._native import ( - build_query_diff_artifact_name, - compute_query_hash, Fingerprint, FingerprintSet, + build_query_diff_artifact_name, + compute_query_hash, parse_query_diff_artifact_name, read_latest_fingerprints, write_fingerprint, From e716a1e69f2e744ff9394e1adeb61286e9c9805c Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Thu, 15 Oct 2026 21:59:35 +0100 Subject: [PATCH 22/48] refactor: project Dagster and Rivers DAG assets and checks natively --- .../src/dag_artifact/_helpers/mod.rs | 1 + .../src/dag_artifact/_helpers/projection.rs | 27 ++ .../src/dag_artifact/constants.rs | 2 + .../src/dag_artifact/main/dag_asset_plans.rs | 37 +++ .../dag_artifact/main/dag_asset_spec_kinds.rs | 27 ++ .../src/dag_artifact/main/dag_check_plans.rs | 42 +++ .../main/dag_selected_asset_nodes.rs | 21 ++ .../src/dag_artifact/main/mod.rs | 4 + .../src/dag_artifact/models.rs | 65 ++++ .../src/dag_artifact/tests.rs | 2 + .../src/dag_artifact/tests/helpers.rs | 39 +++ .../src/dag_artifact/tests/test_projection.rs | 115 +++++++ .../src/dag_artifact/tests/test_types.rs | 19 ++ .../src/bindings/_helpers/dag_artifact/mod.rs | 1 + .../_helpers/dag_artifact/projection.rs | 300 ++++++++++++++++++ .../_helpers/dag_artifact/registration.rs | 3 +- .../_helpers/dag_artifact/selection.rs | 23 +- python-boundary.toml | 4 +- src/sqlbuild/_native.pyi | 15 +- .../integrations/dagster/_helpers/assets.py | 83 +---- .../dagster/_helpers/selection.py | 26 +- .../integrations/rivers/_helpers/assets.py | 35 +- .../integrations/dagster/_test_types.py | 8 + .../sqlbuild/integrations/dagster/helpers.py | 19 ++ .../integrations/dagster/test_assets.py | 39 +++ .../integrations/rivers/_test_types.py | 8 + .../sqlbuild/integrations/rivers/helpers.py | 17 + .../integrations/rivers/test_assets.py | 38 ++- 28 files changed, 882 insertions(+), 138 deletions(-) create mode 100644 crates/sqlbuild-analysis/src/dag_artifact/_helpers/projection.rs create mode 100644 crates/sqlbuild-analysis/src/dag_artifact/main/dag_asset_plans.rs create mode 100644 crates/sqlbuild-analysis/src/dag_artifact/main/dag_asset_spec_kinds.rs create mode 100644 crates/sqlbuild-analysis/src/dag_artifact/main/dag_check_plans.rs create mode 100644 crates/sqlbuild-analysis/src/dag_artifact/main/dag_selected_asset_nodes.rs create mode 100644 crates/sqlbuild-analysis/src/dag_artifact/tests/test_projection.rs create mode 100644 crates/sqlbuild-python/src/bindings/_helpers/dag_artifact/projection.rs diff --git a/crates/sqlbuild-analysis/src/dag_artifact/_helpers/mod.rs b/crates/sqlbuild-analysis/src/dag_artifact/_helpers/mod.rs index 8d9c11ec79..6d4d2d0a6c 100644 --- a/crates/sqlbuild-analysis/src/dag_artifact/_helpers/mod.rs +++ b/crates/sqlbuild-analysis/src/dag_artifact/_helpers/mod.rs @@ -1,5 +1,6 @@ pub(crate) mod checks; pub(crate) mod edges; pub(crate) mod nodes; +pub(crate) mod projection; pub(crate) mod selection; pub(crate) mod values; diff --git a/crates/sqlbuild-analysis/src/dag_artifact/_helpers/projection.rs b/crates/sqlbuild-analysis/src/dag_artifact/_helpers/projection.rs new file mode 100644 index 0000000000..8860bcf778 --- /dev/null +++ b/crates/sqlbuild-analysis/src/dag_artifact/_helpers/projection.rs @@ -0,0 +1,27 @@ +//! Node lookups the orchestrator projections share. + +use std::collections::HashMap; + +use crate::dag_artifact::models::DagProjectionGraph; + +/// Node index by id; a later node with a repeated id replaces the earlier one. +pub(crate) fn nodes_by_id(graph: &DagProjectionGraph) -> HashMap<&str, usize> { + graph + .nodes + .iter() + .enumerate() + .map(|(index, node)| (node.id.as_str(), index)) + .collect() +} + +/// Upstream node ids by downstream node id, in edge order. +pub(crate) fn upstream_by_id(graph: &DagProjectionGraph) -> HashMap<&str, Vec<&str>> { + let mut upstream: HashMap<&str, Vec<&str>> = HashMap::new(); + for (from_id, to_id) in &graph.edges { + upstream + .entry(to_id.as_str()) + .or_default() + .push(from_id.as_str()); + } + upstream +} diff --git a/crates/sqlbuild-analysis/src/dag_artifact/constants.rs b/crates/sqlbuild-analysis/src/dag_artifact/constants.rs index 77a41ef817..5de360110e 100644 --- a/crates/sqlbuild-analysis/src/dag_artifact/constants.rs +++ b/crates/sqlbuild-analysis/src/dag_artifact/constants.rs @@ -34,3 +34,5 @@ pub(crate) const SELECTOR_KIND_CHECK: &str = "check"; pub(crate) const SELECTOR_KIND_TEST: &str = "test"; pub(crate) const SELECTOR_KIND_SEPARATOR: &str = ":"; pub(crate) const SELECTOR_MODELS_PREFIX: &str = "models/"; +/// The orchestrator kind every SQLBuild asset carries. +pub(crate) const SQLBUILD_ASSET_KIND: &str = "sqlbuild"; diff --git a/crates/sqlbuild-analysis/src/dag_artifact/main/dag_asset_plans.rs b/crates/sqlbuild-analysis/src/dag_artifact/main/dag_asset_plans.rs new file mode 100644 index 0000000000..d187feb77d --- /dev/null +++ b/crates/sqlbuild-analysis/src/dag_artifact/main/dag_asset_plans.rs @@ -0,0 +1,37 @@ +//! The asset nodes an orchestrator definition owns and their dependencies. + +use std::collections::HashMap; + +use crate::dag_artifact::_helpers::projection::{nodes_by_id, upstream_by_id}; +use crate::dag_artifact::models::{AssetNodePlan, AssetProjectionPolicy, DagProjectionGraph}; + +/// Each node of an asset kind, in artifact order, with its upstream nodes in edge order. +#[must_use] +pub fn dag_asset_plans( + graph: &DagProjectionGraph, + policy: &AssetProjectionPolicy, +) -> Vec { + let nodes: HashMap<&str, usize> = nodes_by_id(graph); + let upstream: HashMap<&str, Vec<&str>> = upstream_by_id(graph); + let is_asset = |index: usize| policy.asset_kinds.contains(&graph.nodes[index].kind); + let mut plans: Vec = Vec::new(); + for (index, node) in graph.nodes.iter().enumerate() { + if !is_asset(index) { + continue; + } + let mut dependencies: Vec = Vec::new(); + for upstream_id in upstream.get(node.id.as_str()).into_iter().flatten() { + match nodes.get(upstream_id) { + Some(dependency) if !policy.upstream_asset_kinds_only || is_asset(*dependency) => { + dependencies.push(*dependency); + } + _ => {} + } + } + plans.push(AssetNodePlan { + node: index, + upstream: dependencies, + }); + } + plans +} diff --git a/crates/sqlbuild-analysis/src/dag_artifact/main/dag_asset_spec_kinds.rs b/crates/sqlbuild-analysis/src/dag_artifact/main/dag_asset_spec_kinds.rs new file mode 100644 index 0000000000..e7bf3c9116 --- /dev/null +++ b/crates/sqlbuild-analysis/src/dag_artifact/main/dag_asset_spec_kinds.rs @@ -0,0 +1,27 @@ +//! The orchestrator kinds of one asset. + +use crate::dag_artifact::constants::SQLBUILD_ASSET_KIND; +use crate::dag_artifact::models::AssetSpecKindPolicy; + +/// `sqlbuild` plus `view`/`table` for models, the node kind for direct kinds, else nothing more. +#[must_use] +pub fn dag_asset_spec_kinds( + (kind, materialization_type): (&str, &str), + policy: &AssetSpecKindPolicy, +) -> Vec { + let detail: Option<&str> = if kind == policy.model_kind { + Some(if materialization_type == policy.view_materialization { + "view" + } else { + "table" + }) + } else if policy.direct_kinds.contains(kind) { + Some(kind) + } else { + None + }; + std::iter::once(SQLBUILD_ASSET_KIND) + .chain(detail) + .map(str::to_owned) + .collect() +} diff --git a/crates/sqlbuild-analysis/src/dag_artifact/main/dag_check_plans.rs b/crates/sqlbuild-analysis/src/dag_artifact/main/dag_check_plans.rs new file mode 100644 index 0000000000..540a8b0201 --- /dev/null +++ b/crates/sqlbuild-analysis/src/dag_artifact/main/dag_check_plans.rs @@ -0,0 +1,42 @@ +//! The checks an orchestrator check definition exposes and the nodes they check. + +use std::collections::HashMap; + +use crate::dag_artifact::_helpers::projection::nodes_by_id; +use crate::dag_artifact::models::{ + CheckKindFilter, CheckPlan, DagProjectionCheck, DagProjectionGraph, +}; + +/// Whether `filter` exposes checks of `kind`. +fn exposed(filter: &CheckKindFilter, kind: &str) -> bool { + !filter.excluded.contains(kind) + && filter + .required + .as_ref() + .is_none_or(|required| required.contains(kind)) +} + +/// The indexes of the known nodes `check` names, in its order. +fn checked_nodes(nodes: &HashMap<&str, usize>, check: &DagProjectionCheck) -> Vec { + check + .checked_asset_ids + .iter() + .filter_map(|id| nodes.get(id.as_str()).copied()) + .collect() +} + +/// Every check in artifact order, whether `filter` exposes its kind, and its known checked nodes. +#[must_use] +pub fn dag_check_plans(graph: &DagProjectionGraph, filter: &CheckKindFilter) -> Vec { + let nodes: HashMap<&str, usize> = nodes_by_id(graph); + graph + .checks + .iter() + .enumerate() + .map(|(index, check)| CheckPlan { + check: index, + included: exposed(filter, &check.kind), + nodes: checked_nodes(&nodes, check), + }) + .collect() +} diff --git a/crates/sqlbuild-analysis/src/dag_artifact/main/dag_selected_asset_nodes.rs b/crates/sqlbuild-analysis/src/dag_artifact/main/dag_selected_asset_nodes.rs new file mode 100644 index 0000000000..db335097e7 --- /dev/null +++ b/crates/sqlbuild-analysis/src/dag_artifact/main/dag_selected_asset_nodes.rs @@ -0,0 +1,21 @@ +//! The selected nodes an orchestrator owns as assets. + +use std::collections::BTreeSet; + +use crate::dag_artifact::models::DagProjectionGraph; + +/// Indexes of nodes whose id is selected and whose kind is an asset kind, in artifact order. +#[must_use] +pub fn dag_selected_asset_nodes( + graph: &DagProjectionGraph, + selected_ids: &BTreeSet, + asset_kinds: &BTreeSet, +) -> Vec { + graph + .nodes + .iter() + .enumerate() + .filter(|(_, node)| selected_ids.contains(&node.id) && asset_kinds.contains(&node.kind)) + .map(|(index, _)| index) + .collect() +} diff --git a/crates/sqlbuild-analysis/src/dag_artifact/main/mod.rs b/crates/sqlbuild-analysis/src/dag_artifact/main/mod.rs index 80514c0a8a..2d2ed30464 100644 --- a/crates/sqlbuild-analysis/src/dag_artifact/main/mod.rs +++ b/crates/sqlbuild-analysis/src/dag_artifact/main/mod.rs @@ -1,2 +1,6 @@ pub mod build_dag_json; +pub mod dag_asset_plans; +pub mod dag_asset_spec_kinds; +pub mod dag_check_plans; +pub mod dag_selected_asset_nodes; pub mod resolve_dag_selection; diff --git a/crates/sqlbuild-analysis/src/dag_artifact/models.rs b/crates/sqlbuild-analysis/src/dag_artifact/models.rs index 7ec5cf89a3..5e4fdee3a9 100644 --- a/crates/sqlbuild-analysis/src/dag_artifact/models.rs +++ b/crates/sqlbuild-analysis/src/dag_artifact/models.rs @@ -1,5 +1,7 @@ //! The compiled project, Python node graph and authored values the DAG artifact is built from. +use std::collections::BTreeSet; + use crate::assembly::project::types::ObjectKey; use crate::dag_artifact::types::{DagColumnLineage, DagValue}; @@ -243,6 +245,69 @@ pub struct DagSelectionGraph { pub checks: Vec, } +/// One DAG artifact node as orchestrator asset projection reads it. +#[derive(Clone, Debug, Default, PartialEq, Eq)] +pub struct DagProjectionNode { + pub id: String, + pub kind: String, +} + +/// One DAG artifact check as orchestrator check projection reads it. +#[derive(Clone, Debug, Default, PartialEq, Eq)] +pub struct DagProjectionCheck { + pub kind: String, + pub checked_asset_ids: Vec, +} + +/// The DAG artifact's nodes, `(from, to)` edges and checks, in artifact order. +#[derive(Clone, Debug, Default, PartialEq, Eq)] +pub struct DagProjectionGraph { + pub nodes: Vec, + pub edges: Vec<(String, String)>, + pub checks: Vec, +} + +/// Which nodes become orchestrator assets and which upstream nodes they depend on. +#[derive(Clone, Debug, Default, PartialEq, Eq)] +pub struct AssetProjectionPolicy { + /// Node kinds the orchestrator owns as assets. + pub asset_kinds: BTreeSet, + /// Upstream nodes of other kinds are left out of an asset's dependencies. + pub upstream_asset_kinds_only: bool, +} + +/// One asset node and the nodes it depends on, as indexes into the graph's nodes. +#[derive(Clone, Debug, Default, PartialEq, Eq)] +pub struct AssetNodePlan { + pub node: usize, + pub upstream: Vec, +} + +/// Which check kinds a check definition exposes. +#[derive(Clone, Debug, Default, PartialEq, Eq)] +pub struct CheckKindFilter { + pub excluded: BTreeSet, + /// When set, only these kinds are exposed. + pub required: Option>, +} + +/// One check, whether its kind is exposed, and the nodes it checks as graph indexes. +#[derive(Clone, Debug, Default, PartialEq, Eq)] +pub struct CheckPlan { + pub check: usize, + pub included: bool, + pub nodes: Vec, +} + +/// How an asset's orchestrator kinds follow from its node kind and materialization. +#[derive(Clone, Debug, Default, PartialEq, Eq)] +pub struct AssetSpecKindPolicy { + pub model_kind: String, + pub view_materialization: String, + /// Node kinds reported as their own orchestrator kind. + pub direct_kinds: BTreeSet, +} + /// The node lookup and edge indexes every selector token resolves against. #[derive(Debug)] pub(crate) struct DagSelectionIndex<'a> { diff --git a/crates/sqlbuild-analysis/src/dag_artifact/tests.rs b/crates/sqlbuild-analysis/src/dag_artifact/tests.rs index e7bae12d29..ff81f5efd0 100644 --- a/crates/sqlbuild-analysis/src/dag_artifact/tests.rs +++ b/crates/sqlbuild-analysis/src/dag_artifact/tests.rs @@ -2,5 +2,7 @@ mod build_dag_json; #[path = "tests/helpers.rs"] mod helpers; +#[path = "tests/test_projection.rs"] +mod projection; #[path = "tests/test_types.rs"] mod test_types; diff --git a/crates/sqlbuild-analysis/src/dag_artifact/tests/helpers.rs b/crates/sqlbuild-analysis/src/dag_artifact/tests/helpers.rs index c41a2a4f7f..a6ab1a2e78 100644 --- a/crates/sqlbuild-analysis/src/dag_artifact/tests/helpers.rs +++ b/crates/sqlbuild-analysis/src/dag_artifact/tests/helpers.rs @@ -96,3 +96,42 @@ pub(crate) fn team_meta(value: DagValue) -> DagValue { (text_key("team"), value), ]) } + +/// Nodes `seed.a`, `model.b`, `check.c`, `model.b` again (the later one wins) and edges into them. +pub(crate) fn projection_graph() -> crate::dag_artifact::models::DagProjectionGraph { + use crate::dag_artifact::models::{DagProjectionCheck, DagProjectionGraph, DagProjectionNode}; + let node = |id: &str, kind: &str| DagProjectionNode { + id: id.to_owned(), + kind: kind.to_owned(), + }; + let edge = |from: &str, to: &str| (from.to_owned(), to.to_owned()); + DagProjectionGraph { + nodes: vec![ + node("seed.a", "seed"), + node("model.b", "model"), + node("check.c", "check"), + node("model.b", "model"), + ], + edges: vec![ + edge("missing", "model.b"), + edge("check.c", "model.b"), + edge("seed.a", "model.b"), + edge("model.b", "check.c"), + ], + checks: vec![ + DagProjectionCheck { + kind: "scenario".to_owned(), + checked_asset_ids: vec!["model.b".to_owned(), "missing".to_owned()], + }, + DagProjectionCheck { + kind: "audit".to_owned(), + checked_asset_ids: vec!["seed.a".to_owned()], + }, + ], + } +} + +/// The Dagster and Rivers asset node kinds of the projection graph. +pub(crate) fn asset_kinds() -> std::collections::BTreeSet { + ["seed", "model"].map(str::to_owned).into() +} diff --git a/crates/sqlbuild-analysis/src/dag_artifact/tests/test_projection.rs b/crates/sqlbuild-analysis/src/dag_artifact/tests/test_projection.rs new file mode 100644 index 0000000000..749a854954 --- /dev/null +++ b/crates/sqlbuild-analysis/src/dag_artifact/tests/test_projection.rs @@ -0,0 +1,115 @@ +//! Orchestrator asset and check projections follow the integrations' artifact indexing. + +use std::collections::BTreeSet; + +use crate::dag_artifact::main::dag_asset_plans::dag_asset_plans; +use crate::dag_artifact::main::dag_asset_spec_kinds::dag_asset_spec_kinds; +use crate::dag_artifact::main::dag_check_plans::dag_check_plans; +use crate::dag_artifact::models::{ + AssetNodePlan, AssetProjectionPolicy, AssetSpecKindPolicy, CheckKindFilter, CheckPlan, +}; +use crate::dag_artifact::tests::helpers::{asset_kinds, projection_graph}; +use crate::dag_artifact::tests::test_types::{ + AssetPlansTestCase, CheckPlansTestCase, SpecKindsTestCase, +}; + +#[test] +fn given_artifact_graph_when_planning_assets_then_keeps_edge_order_and_last_duplicate() { + let test_cases = [ + AssetPlansTestCase { + description: "any known upstream node", + upstream_asset_kinds_only: false, + expected_plans: vec![(0, vec![]), (1, vec![2, 0]), (3, vec![2, 0])], + }, + AssetPlansTestCase { + description: "upstream asset nodes only", + upstream_asset_kinds_only: true, + expected_plans: vec![(0, vec![]), (1, vec![0]), (3, vec![0])], + }, + ]; + for test_case in test_cases { + let policy: AssetProjectionPolicy = AssetProjectionPolicy { + asset_kinds: asset_kinds(), + upstream_asset_kinds_only: test_case.upstream_asset_kinds_only, + }; + let plans: Vec<(usize, Vec)> = dag_asset_plans(&projection_graph(), &policy) + .into_iter() + .map(|plan: AssetNodePlan| (plan.node, plan.upstream)) + .collect(); + assert_eq!(plans, test_case.expected_plans, "{}", test_case.description); + } +} + +#[test] +fn given_check_filter_when_planning_checks_then_marks_exposed_kinds() { + let scenario: BTreeSet = BTreeSet::from(["scenario".to_owned()]); + let test_cases = [ + CheckPlansTestCase { + description: "scenarios excluded", + filter: CheckKindFilter { + excluded: scenario.clone(), + required: None, + }, + expected_plans: vec![(0, false, vec![3]), (1, true, vec![0])], + }, + CheckPlansTestCase { + description: "scenarios only", + filter: CheckKindFilter { + excluded: BTreeSet::new(), + required: Some(scenario.clone()), + }, + expected_plans: vec![(0, true, vec![3]), (1, false, vec![0])], + }, + ]; + for test_case in test_cases { + let plans: Vec<(usize, bool, Vec)> = + dag_check_plans(&projection_graph(), &test_case.filter) + .into_iter() + .map(|plan: CheckPlan| (plan.check, plan.included, plan.nodes)) + .collect(); + assert_eq!(plans, test_case.expected_plans, "{}", test_case.description); + } +} + +#[test] +fn given_asset_node_when_choosing_spec_kinds_then_reports_materialization_or_kind() { + let test_cases = [ + SpecKindsTestCase { + description: "view model", + kind: "model", + materialization_type: "view", + expected_kinds: vec!["sqlbuild", "view"], + }, + SpecKindsTestCase { + description: "incremental model is a table", + kind: "model", + materialization_type: "incremental", + expected_kinds: vec!["sqlbuild", "table"], + }, + SpecKindsTestCase { + description: "direct kind", + kind: "seed", + materialization_type: "table", + expected_kinds: vec!["sqlbuild", "seed"], + }, + SpecKindsTestCase { + description: "other kind", + kind: "check", + materialization_type: "table", + expected_kinds: vec!["sqlbuild"], + }, + ]; + let policy: AssetSpecKindPolicy = AssetSpecKindPolicy { + model_kind: "model".to_owned(), + view_materialization: "view".to_owned(), + direct_kinds: BTreeSet::from(["seed".to_owned()]), + }; + for test_case in test_cases { + assert_eq!( + dag_asset_spec_kinds((test_case.kind, test_case.materialization_type), &policy), + test_case.expected_kinds, + "{}", + test_case.description + ); + } +} diff --git a/crates/sqlbuild-analysis/src/dag_artifact/tests/test_types.rs b/crates/sqlbuild-analysis/src/dag_artifact/tests/test_types.rs index 529ebbda57..5b05ed6281 100644 --- a/crates/sqlbuild-analysis/src/dag_artifact/tests/test_types.rs +++ b/crates/sqlbuild-analysis/src/dag_artifact/tests/test_types.rs @@ -9,3 +9,22 @@ pub(crate) struct BuildDagJsonTestCase { pub(crate) upstream: Vec<(ObjectKey, Vec)>, pub(crate) expected_json: Result<&'static str, DagArtifactError>, } + +pub(crate) struct AssetPlansTestCase { + pub(crate) description: &'static str, + pub(crate) upstream_asset_kinds_only: bool, + pub(crate) expected_plans: Vec<(usize, Vec)>, +} + +pub(crate) struct CheckPlansTestCase { + pub(crate) description: &'static str, + pub(crate) filter: crate::dag_artifact::models::CheckKindFilter, + pub(crate) expected_plans: Vec<(usize, bool, Vec)>, +} + +pub(crate) struct SpecKindsTestCase { + pub(crate) description: &'static str, + pub(crate) kind: &'static str, + pub(crate) materialization_type: &'static str, + pub(crate) expected_kinds: Vec<&'static str>, +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/dag_artifact/mod.rs b/crates/sqlbuild-python/src/bindings/_helpers/dag_artifact/mod.rs index b90720c229..c86bdc51d8 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/dag_artifact/mod.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/dag_artifact/mod.rs @@ -1,4 +1,5 @@ pub(crate) mod artifact; +pub(crate) mod projection; pub(crate) mod registration; pub(crate) mod rows; pub(crate) mod selection; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/dag_artifact/projection.rs b/crates/sqlbuild-python/src/bindings/_helpers/dag_artifact/projection.rs new file mode 100644 index 0000000000..d697b98371 --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/dag_artifact/projection.rs @@ -0,0 +1,300 @@ +//! Dagster and Rivers asset and check plans over a DAG artifact, holding the exact translator inputs. + +use std::collections::BTreeSet; + +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyModule, PyModuleMethods, PyResult, Python}; +use pyo3::types::{PyDict, PyDictMethods, PyList, PyMapping, PySet, PyTuple}; +use pyo3::{IntoPyObject, pyfunction, wrap_pyfunction}; +use sqlbuild_analysis::dag_artifact::main::dag_asset_plans::dag_asset_plans; +use sqlbuild_analysis::dag_artifact::main::dag_asset_spec_kinds::dag_asset_spec_kinds; +use sqlbuild_analysis::dag_artifact::main::dag_check_plans::dag_check_plans; +use sqlbuild_analysis::dag_artifact::main::dag_selected_asset_nodes::dag_selected_asset_nodes; +use sqlbuild_analysis::dag_artifact::models::{ + AssetNodePlan, AssetProjectionPolicy, AssetSpecKindPolicy, CheckKindFilter, CheckPlan, + DagProjectionCheck, DagProjectionGraph, DagProjectionNode, +}; + +use crate::bindings::_helpers::compile_driver::python_calls::imported; +use crate::bindings::_helpers::dag_artifact::selection::selected_node_ids; + +const DAGSTER_CONSTANTS: &str = "sqlbuild.integrations.dagster.constants"; +const RIVERS_CONSTANTS: &str = "sqlbuild.integrations.rivers.constants"; +const PROJECT_NAME_KEY: &str = "project_name"; +const DEFAULT_MATERIALIZATION: &str = "table"; + +/// The artifact's nodes, edges and checks as Python objects alongside their projection facts. +struct ArtifactGraph<'py> { + project_name: Bound<'py, PyAny>, + nodes: Vec>, + checks: Vec>, + graph: DagProjectionGraph, +} + +fn text(value: &Bound<'_, PyAny>) -> PyResult { + value.str()?.extract() +} + +fn listed<'py>(value: &Bound<'py, PyAny>) -> PyResult>> { + value.try_iter()?.collect() +} + +fn projection_node(node: &Bound<'_, PyAny>) -> PyResult { + Ok(DagProjectionNode { + id: text(&node.get_item("id")?)?, + kind: text(&node.call_method1("get", ("kind",))?)?, + }) +} + +fn projection_check(check: &Bound<'_, PyAny>) -> PyResult { + let empty: Bound<'_, PyTuple> = PyTuple::empty(check.py()); + Ok(DagProjectionCheck { + kind: text(&check.call_method1("get", ("kind",))?)?, + checked_asset_ids: listed(&check.call_method1("get", ("checked_asset_ids", empty))?)? + .iter() + .map(text) + .collect::>()?, + }) +} + +/// The artifact read the way the integrations index it: `nodes`, `edges` and `checks` by key. +fn artifact_graph<'py>(dag: &Bound<'py, PyAny>, with_checks: bool) -> PyResult> { + let nodes: Vec> = listed(&dag.get_item("nodes")?)?; + let checks: Vec> = if with_checks { + listed(&dag.get_item("checks")?)? + } else { + Vec::new() + }; + let edges: Vec<(String, String)> = if with_checks { + Vec::new() + } else { + listed(&dag.get_item("edges")?)? + .iter() + .map(|edge| { + Ok(( + text(&edge.get_item("from_id")?)?, + text(&edge.get_item("to_id")?)?, + )) + }) + .collect::>()? + }; + Ok(ArtifactGraph { + project_name: dag.call_method1("get", (PROJECT_NAME_KEY,))?, + graph: DagProjectionGraph { + nodes: nodes.iter().map(projection_node).collect::>()?, + edges, + checks: checks + .iter() + .map(projection_check) + .collect::>()?, + }, + nodes, + checks, + }) +} + +/// `{**node, "project_name": project_name}`. +fn with_project_name<'py>( + node: &Bound<'py, PyAny>, + project_name: &Bound<'py, PyAny>, +) -> PyResult> { + let translated: Bound<'py, PyDict> = PyDict::new(node.py()); + translated.update(node.downcast::()?)?; + translated.set_item(PROJECT_NAME_KEY, project_name)?; + Ok(translated.into_any()) +} + +/// One `{**node, "project_name": ...}` per artifact node, in artifact order. +fn translated_nodes<'py>(artifact: &ArtifactGraph<'py>) -> PyResult>> { + artifact + .nodes + .iter() + .map(|node| with_project_name(node, &artifact.project_name)) + .collect() +} + +fn kind_set(py: Python<'_>, (module, name): (&str, &str)) -> PyResult> { + listed(&imported(py, module, name)?)? + .iter() + .map(text) + .collect() +} + +fn constant_text(py: Python<'_>, (module, name): (&str, &str)) -> PyResult { + text(&imported(py, module, name)?) +} + +fn picked<'py>(objects: &[Bound<'py, PyAny>], indexes: &[usize]) -> Vec> { + indexes + .iter() + .map(|index| objects[*index].clone()) + .collect() +} + +/// `str(node.get("materialization_type") or "table")`. +fn materialization_type(node: &Bound<'_, PyAny>) -> PyResult { + let value: Bound<'_, PyAny> = node.call_method1("get", ("materialization_type",))?; + if value.is_truthy()? { + text(&value) + } else { + Ok(DEFAULT_MATERIALIZATION.to_owned()) + } +} + +/// Dagster asset specs to build: each translated asset node, its upstream nodes and its kinds. +#[pyfunction] +#[pyo3(signature = (dag, /))] +fn dagster_asset_plans<'py>(dag: &Bound<'py, PyAny>) -> PyResult> { + let py: Python<'py> = dag.py(); + let artifact: ArtifactGraph<'py> = artifact_graph(dag, false)?; + let upstream_nodes: Vec> = translated_nodes(&artifact)?; + let policy: AssetProjectionPolicy = AssetProjectionPolicy { + asset_kinds: kind_set(py, (DAGSTER_CONSTANTS, "DAGSTER_ASSET_NODE_KINDS"))?, + upstream_asset_kinds_only: false, + }; + let kinds: AssetSpecKindPolicy = AssetSpecKindPolicy { + model_kind: constant_text(py, (DAGSTER_CONSTANTS, "MODEL_NODE_KIND"))?, + view_materialization: constant_text(py, (DAGSTER_CONSTANTS, "VIEW_MATERIALIZATION_TYPE"))?, + direct_kinds: kind_set(py, (DAGSTER_CONSTANTS, "DAGSTER_DIRECT_KIND_NODE_KINDS"))?, + }; + let plans: Vec> = dag_asset_plans(&artifact.graph, &policy) + .iter() + .map(|plan: &AssetNodePlan| { + let node: &Bound<'py, PyAny> = &artifact.nodes[plan.node]; + let spec_kinds: Vec = dag_asset_spec_kinds( + ( + &artifact.graph.nodes[plan.node].kind, + &materialization_type(node)?, + ), + &kinds, + ); + PyTuple::new( + py, + [ + with_project_name(node, &artifact.project_name)?, + PyList::new(py, picked(&upstream_nodes, &plan.upstream))?.into_any(), + PySet::new(py, spec_kinds)?.into_any(), + ], + ) + }) + .collect::>()?; + PyList::new(py, plans) +} + +/// Rivers asset definitions to build: each translated asset node and its upstream asset nodes. +#[pyfunction] +#[pyo3(signature = (dag, /))] +fn rivers_asset_plans<'py>(dag: &Bound<'py, PyAny>) -> PyResult> { + let py: Python<'py> = dag.py(); + let artifact: ArtifactGraph<'py> = artifact_graph(dag, false)?; + let policy: AssetProjectionPolicy = AssetProjectionPolicy { + asset_kinds: kind_set(py, (RIVERS_CONSTANTS, "RIVERS_ASSET_NODE_KINDS"))?, + upstream_asset_kinds_only: true, + }; + let plans: Vec> = dag_asset_plans(&artifact.graph, &policy) + .iter() + .map(|plan: &AssetNodePlan| { + PyTuple::new( + py, + [ + with_project_name(&artifact.nodes[plan.node], &artifact.project_name)?, + PyList::new(py, picked(&artifact.nodes, &plan.upstream))?.into_any(), + ], + ) + }) + .collect::>()?; + PyList::new(py, plans) +} + +/// `str(check.get("path"))`, or `None` without a path. +fn check_description<'py>(check: &Bound<'py, PyAny>) -> PyResult> { + let path: Bound<'py, PyAny> = check.call_method1("get", ("path",))?; + if path.is_none() { + Ok(path) + } else { + Ok(path.str()?.into_any()) + } +} + +/// Dagster check specs to build: check, kind exposed, translated checked nodes and description. +#[pyfunction] +#[pyo3(signature = (dag, /, *, include_scenarios, only_scenarios))] +fn dagster_check_plans<'py>( + dag: &Bound<'py, PyAny>, + include_scenarios: bool, + only_scenarios: bool, +) -> PyResult> { + let py: Python<'py> = dag.py(); + let artifact: ArtifactGraph<'py> = artifact_graph(dag, true)?; + let checked_nodes: Vec> = translated_nodes(&artifact)?; + let scenario: BTreeSet = BTreeSet::from([constant_text( + py, + (DAGSTER_CONSTANTS, "SCENARIO_CHECK_KIND"), + )?]); + let filter: CheckKindFilter = CheckKindFilter { + excluded: if include_scenarios || only_scenarios { + BTreeSet::new() + } else { + scenario.clone() + }, + required: only_scenarios.then_some(scenario), + }; + let plans: Vec> = dag_check_plans(&artifact.graph, &filter) + .iter() + .map(|plan: &CheckPlan| { + let check: &Bound<'py, PyAny> = &artifact.checks[plan.check]; + PyTuple::new( + py, + [ + check.clone(), + plan.included.into_pyobject(py)?.to_owned().into_any(), + PyList::new(py, picked(&checked_nodes, &plan.nodes))?.into_any(), + check_description(check)?, + ], + ) + }) + .collect::>()?; + PyList::new(py, plans) +} + +/// The translated nodes Dagster owns as assets among those `select` minus `exclude` matches. +#[pyfunction] +#[pyo3(signature = (dag, /, *, select, exclude = None))] +fn dagster_selected_asset_nodes<'py>( + dag: &Bound<'py, PyAny>, + select: &str, + exclude: Option<&str>, +) -> PyResult> { + let py: Python<'py> = dag.py(); + let selected: BTreeSet = selected_node_ids(dag, (select, exclude))?; + let nodes: Vec> = + listed(&dag.call_method1("get", ("nodes", PyTuple::empty(py)))?)?; + let project_name: Bound<'py, PyAny> = dag.call_method1("get", (PROJECT_NAME_KEY,))?; + let graph: DagProjectionGraph = DagProjectionGraph { + nodes: nodes.iter().map(selection_node).collect::>()?, + ..DagProjectionGraph::default() + }; + let asset_kinds: BTreeSet = + kind_set(py, (DAGSTER_CONSTANTS, "DAGSTER_ASSET_NODE_KINDS"))?; + let translated: Vec> = + dag_selected_asset_nodes(&graph, &selected, &asset_kinds) + .iter() + .map(|index| with_project_name(&nodes[*index], &project_name)) + .collect::>()?; + PyList::new(py, translated) +} + +/// A node read the way selection reads it: `str(node.get("id"))` and `str(node.get("kind"))`. +fn selection_node(node: &Bound<'_, PyAny>) -> PyResult { + Ok(DagProjectionNode { + id: text(&node.call_method1("get", ("id",))?)?, + kind: text(&node.call_method1("get", ("kind",))?)?, + }) +} + +pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { + module.add_function(wrap_pyfunction!(dagster_asset_plans, module)?)?; + module.add_function(wrap_pyfunction!(dagster_check_plans, module)?)?; + module.add_function(wrap_pyfunction!(dagster_selected_asset_nodes, module)?)?; + module.add_function(wrap_pyfunction!(rivers_asset_plans, module)?)?; + Ok(()) +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/dag_artifact/registration.rs b/crates/sqlbuild-python/src/bindings/_helpers/dag_artifact/registration.rs index 55b6276cfe..ff1a3af1a3 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/dag_artifact/registration.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/dag_artifact/registration.rs @@ -2,11 +2,12 @@ use pyo3::prelude::{Bound, PyModule, PyResult}; -use crate::bindings::_helpers::dag_artifact::{artifact, selection}; +use crate::bindings::_helpers::dag_artifact::{artifact, projection, selection}; use crate::bindings::_helpers::dagster_invocation; pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { artifact::register(module)?; + projection::register(module)?; selection::register(module)?; dagster_invocation::registration::register(module) } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/dag_artifact/selection.rs b/crates/sqlbuild-python/src/bindings/_helpers/dag_artifact/selection.rs index 70099991bc..8fcf88a7ab 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/dag_artifact/selection.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/dag_artifact/selection.rs @@ -1,10 +1,12 @@ -//! Dagster asset selection: SQLBuild selectors resolved over a DAG artifact mapping. +//! DAG artifact loading and SQLBuild selectors resolved over a DAG artifact mapping. + +use std::collections::BTreeSet; use pyo3::exceptions::PyOSError; use pyo3::prelude::{ Bound, PyAny, PyAnyMethods, PyErr, PyModule, PyModuleMethods, PyResult, Python, }; -use pyo3::types::{PyDict, PyFrozenSet, PyList, PyString, PyTuple}; +use pyo3::types::{PyDict, PyList, PyString, PyTuple}; use pyo3::{pyfunction, wrap_pyfunction}; use sqlbuild_analysis::dag_artifact::errors::DagSelectionError; use sqlbuild_analysis::dag_artifact::main::resolve_dag_selection::resolve_dag_selection as resolve; @@ -106,18 +108,12 @@ fn selection_error(py: Python<'_>, error: DagSelectionError, raw: &str) -> PyErr } /// Ids of DAG artifact nodes `select` matches minus those `exclude` matches. -#[pyfunction] -#[pyo3(signature = (*, dag, select, exclude = None))] -fn resolve_dag_selection<'py>( - dag: &Bound<'py, PyAny>, - select: &str, - exclude: Option<&str>, -) -> PyResult> { - let py: Python<'py> = dag.py(); +pub(crate) fn selected_node_ids( + dag: &Bound<'_, PyAny>, + (select, exclude): (&str, Option<&str>), +) -> PyResult> { let graph: DagSelectionGraph = selection_graph(dag)?; - let selected = - resolve(&graph, (select, exclude)).map_err(|error| selection_error(py, error, select))?; - PyFrozenSet::new(py, selected) + resolve(&graph, (select, exclude)).map_err(|error| selection_error(dag.py(), error, select)) } fn raised(error_class: &Bound<'_, PyAny>, message: String, cause: Option) -> PyErr { @@ -212,7 +208,6 @@ fn load_sqlbuild_dag<'py>( } pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { - module.add_function(wrap_pyfunction!(resolve_dag_selection, module)?)?; module.add_function(wrap_pyfunction!(load_sqlbuild_dag, module)?)?; Ok(()) } diff --git a/python-boundary.toml b/python-boundary.toml index 68b27a0b12..d79f6e5b03 100644 --- a/python-boundary.toml +++ b/python-boundary.toml @@ -392,12 +392,12 @@ note = "Rule 3 shapes: the invocation-context variable and wire constants used b [[stays]] path = "integrations/dagster" reason = "public-api" -note = "Documented orchestrator integration and rule 4 Dagster glue: context reads, subprocess streaming and Dagster event construction; CLI arguments, selector logs, payload and failure parsing are native." +note = "Documented orchestrator integration and rule 4 Dagster glue: translator invocation, context reads, subprocess streaming and Dagster event construction; CLI arguments, selector logs, payload and failure parsing, asset/check/selection projection of the DAG artifact are native." [[stays]] path = "integrations/rivers" reason = "public-api" -note = "Documented orchestrator integration." +note = "Documented orchestrator integration and rule 4 Rivers glue: translator invocation and AssetDef construction; the asset projection of the DAG artifact is native." [[stays]] path = "integrations/dlt" diff --git a/src/sqlbuild/_native.pyi b/src/sqlbuild/_native.pyi index 5fb843c57b..be0c342019 100644 --- a/src/sqlbuild/_native.pyi +++ b/src/sqlbuild/_native.pyi @@ -5298,9 +5298,18 @@ def aggregate_build_result( source_audit_results: tuple[AuditExecutionResult, ...], end_audit_results: tuple[AuditExecutionResult, ...], ) -> BuildExecutionResult: ... -def resolve_dag_selection( - *, dag: Mapping[str, Any], select: str, exclude: str | None = None -) -> frozenset[str]: ... +def dagster_asset_plans( + dag: Mapping[str, Any], / +) -> list[tuple[dict[str, Any], list[dict[str, Any]], set[str]]]: ... +def dagster_check_plans( + dag: Mapping[str, Any], /, *, include_scenarios: bool, only_scenarios: bool +) -> list[tuple[Mapping[str, Any], bool, list[Mapping[str, Any]], str | None]]: ... +def dagster_selected_asset_nodes( + dag: Mapping[str, Any], /, *, select: str, exclude: str | None = None +) -> list[dict[str, Any]]: ... +def rivers_asset_plans( + dag: Mapping[str, Any], / +) -> list[tuple[dict[str, Any], list[Mapping[str, Any]]]]: ... def load_sqlbuild_dag( dag: Mapping[str, Any] | str | Path, *, error_class: type[Exception] ) -> Mapping[str, Any]: ... diff --git a/src/sqlbuild/integrations/dagster/_helpers/assets.py b/src/sqlbuild/integrations/dagster/_helpers/assets.py index 555137015e..c7841a94c6 100644 --- a/src/sqlbuild/integrations/dagster/_helpers/assets.py +++ b/src/sqlbuild/integrations/dagster/_helpers/assets.py @@ -5,16 +5,11 @@ from collections.abc import Mapping from typing import Any +from sqlbuild._native import dagster_asset_plans, dagster_check_plans from sqlbuild.integrations.dagster._helpers.imports import load_dagster from sqlbuild.integrations.dagster.classes.sqlbuild_dagster_translator import ( SqlBuildDagsterTranslator, ) -from sqlbuild.integrations.dagster.constants import ( - DAGSTER_ASSET_NODE_KINDS, - DAGSTER_DIRECT_KIND_NODE_KINDS, - MODEL_NODE_KIND, - VIEW_MATERIALIZATION_TYPE, -) def build_asset_specs( @@ -23,22 +18,13 @@ def build_asset_specs( """Build materializable Dagster asset specs from a SQLBuild DAG artifact.""" dg: Any = load_dagster() - nodes_by_id: dict[str, Mapping[str, Any]] = _nodes_by_id(dag) - upstream_by_id: dict[str, list[str]] = _upstream_by_id(dag) - project_name: object = dag.get("project_name") specs: list[Any] = [] - for node in dag["nodes"]: - if str(node.get("kind")) not in DAGSTER_ASSET_NODE_KINDS: - continue - translated_node: Mapping[str, Any] = {**node, "project_name": project_name} + for translated_node, upstream_nodes, kinds in dagster_asset_plans(dag): if not translator.is_asset_node(translated_node): continue - deps: list[Any] = [] - for upstream_id in upstream_by_id.get(str(node["id"]), []): - upstream_node: Mapping[str, Any] | None = nodes_by_id.get(upstream_id) - if upstream_node is None: - continue - deps.append(dg.AssetDep(translator.get_asset_key(upstream_node))) + deps: list[Any] = [ + dg.AssetDep(translator.get_asset_key(upstream_node)) for upstream_node in upstream_nodes + ] specs.append( dg.AssetSpec( key=translator.get_asset_key(translated_node), @@ -47,7 +33,7 @@ def build_asset_specs( group_name=translator.get_group_name(translated_node), metadata=translator.get_metadata(translated_node), tags=translator.get_tags(translated_node), - kinds=_asset_spec_kinds(translated_node), + kinds=kinds, ) ) return tuple(specs) @@ -61,12 +47,9 @@ def build_check_specs( ) -> tuple[Any, ...]: """Build Dagster check specs from SQLBuild test/audit/scenario checks.""" - excluded_kinds: set[str] = set() if include_scenarios else {"scenario"} return _build_check_specs( - dag=dag, + plans=dagster_check_plans(dag, include_scenarios=include_scenarios, only_scenarios=False), translator=translator, - kinds=None, - excluded_kinds=excluded_kinds, ) @@ -76,70 +59,30 @@ def build_scenario_check_specs( """Build Dagster check specs for SQLBuild scenario checks only.""" return _build_check_specs( - dag=dag, + plans=dagster_check_plans(dag, include_scenarios=True, only_scenarios=True), translator=translator, - kinds={"scenario"}, - excluded_kinds=set(), ) def _build_check_specs( *, - dag: Mapping[str, Any], + plans: list[tuple[Mapping[str, Any], bool, list[Mapping[str, Any]], str | None]], translator: SqlBuildDagsterTranslator, - kinds: set[str] | None, - excluded_kinds: set[str], ) -> tuple[Any, ...]: - """Build Dagster check specs from SQLBuild checks matching optional kinds.""" - dg: Any = load_dagster() - nodes_by_id: dict[str, Mapping[str, Any]] = _nodes_by_id(dag) specs: list[Any] = [] - for check in dag["checks"]: - check_kind: str = str(check.get("kind")) - if not translator.is_asset_check(check): - continue - if check_kind in excluded_kinds: - continue - if kinds is not None and check_kind not in kinds: + for check, included, checked_nodes, description in plans: + if not translator.is_asset_check(check) or not included: continue - for asset_id in check.get("checked_asset_ids", ()): - node: Mapping[str, Any] | None = nodes_by_id.get(str(asset_id)) - if node is None: - continue + for node in checked_nodes: if not translator.is_asset_node(node): continue specs.append( dg.AssetCheckSpec( name=translator.get_check_name(check), asset=translator.get_asset_key(node), - description=str(check.get("path")) if check.get("path") is not None else None, + description=description, metadata=translator.get_check_metadata(check), ) ) return tuple(specs) - - -def _nodes_by_id(dag: Mapping[str, Any]) -> dict[str, Mapping[str, Any]]: - project_name: object = dag.get("project_name") - return {str(node["id"]): {**node, "project_name": project_name} for node in dag["nodes"]} - - -def _upstream_by_id(dag: Mapping[str, Any]) -> dict[str, list[str]]: - upstream: dict[str, list[str]] = {} - edge: Mapping[str, Any] - for edge in dag["edges"]: - upstream.setdefault(str(edge["to_id"]), []).append(str(edge["from_id"])) - return upstream - - -def _asset_spec_kinds(node: Mapping[str, Any]) -> set[str]: - kind: str = str(node.get("kind")) - if kind == MODEL_NODE_KIND: - materialization_type: str = str(node.get("materialization_type") or "table") - if materialization_type == VIEW_MATERIALIZATION_TYPE: - return {"sqlbuild", "view"} - return {"sqlbuild", "table"} - if kind in DAGSTER_DIRECT_KIND_NODE_KINDS: - return {"sqlbuild", kind} - return {"sqlbuild"} diff --git a/src/sqlbuild/integrations/dagster/_helpers/selection.py b/src/sqlbuild/integrations/dagster/_helpers/selection.py index 9f935ab8ec..a3926d2ed3 100644 --- a/src/sqlbuild/integrations/dagster/_helpers/selection.py +++ b/src/sqlbuild/integrations/dagster/_helpers/selection.py @@ -5,12 +5,11 @@ from collections.abc import Mapping, Sequence from typing import Any -from sqlbuild._native import load_sqlbuild_dag, resolve_dag_selection +from sqlbuild._native import dagster_selected_asset_nodes, load_sqlbuild_dag from sqlbuild.integrations.dagster._helpers.imports import load_dagster from sqlbuild.integrations.dagster.classes.sqlbuild_dagster_translator import ( SqlBuildDagsterTranslator, ) -from sqlbuild.integrations.dagster.constants import DAGSTER_ASSET_NODE_KINDS from sqlbuild.integrations.dagster.exceptions import DagsterDagInputError from sqlbuild.integrations.dagster.types import SqlBuildDagInput @@ -29,20 +28,11 @@ def build_sqlbuild_asset_selection_impl( raise DagsterDagInputError("SQLBuild asset selection requires an asset definition") resolved_dag: Mapping[str, Any] = load_sqlbuild_dag(dag, error_class=DagsterDagInputError) resolved_translator: SqlBuildDagsterTranslator = translator or SqlBuildDagsterTranslator() - selected_ids: frozenset[str] = resolve_dag_selection( - dag=resolved_dag, - select=sqlbuild_select, - exclude=sqlbuild_exclude, - ) - asset_keys: list[Any] = [] - project_name: object = resolved_dag.get("project_name") - for node in resolved_dag.get("nodes", ()): - if str(node.get("id")) not in selected_ids: - continue - if str(node.get("kind")) not in DAGSTER_ASSET_NODE_KINDS: - continue - translated_node: Mapping[str, Any] = {**node, "project_name": project_name} - if not resolved_translator.is_asset_node(translated_node): - continue - asset_keys.append(resolved_translator.get_asset_key(translated_node)) + asset_keys: list[Any] = [ + resolved_translator.get_asset_key(node) + for node in dagster_selected_asset_nodes( + resolved_dag, select=sqlbuild_select, exclude=sqlbuild_exclude + ) + if resolved_translator.is_asset_node(node) + ] return load_dagster().AssetSelection.assets(*asset_keys) diff --git a/src/sqlbuild/integrations/rivers/_helpers/assets.py b/src/sqlbuild/integrations/rivers/_helpers/assets.py index 29087fd7e8..c3fbdad830 100644 --- a/src/sqlbuild/integrations/rivers/_helpers/assets.py +++ b/src/sqlbuild/integrations/rivers/_helpers/assets.py @@ -5,11 +5,11 @@ from collections.abc import Mapping from typing import Any +from sqlbuild._native import rivers_asset_plans from sqlbuild.integrations.rivers._helpers.imports import load_rivers from sqlbuild.integrations.rivers.classes.sqlbuild_rivers_translator import ( SqlBuildRiversTranslator, ) -from sqlbuild.integrations.rivers.constants import RIVERS_ASSET_NODE_KINDS def build_asset_defs( @@ -18,23 +18,12 @@ def build_asset_defs( """Build materializable Rivers asset definitions from a SQLBuild DAG artifact.""" rs: Any = load_rivers() - nodes_by_id: dict[str, Mapping[str, Any]] = _nodes_by_id(dag) - upstream_by_id: dict[str, list[str]] = _upstream_by_id(dag) - project_name: object = dag.get("project_name") asset_defs: list[Any] = [] - for node in dag["nodes"]: - if str(node.get("kind")) not in RIVERS_ASSET_NODE_KINDS: - continue - translated_node: Mapping[str, Any] = {**node, "project_name": project_name} - deps: list[Any] = [] - for upstream_id in upstream_by_id.get(str(node["id"]), []): - upstream_node: Mapping[str, Any] | None = nodes_by_id.get(upstream_id) - if ( - upstream_node is None - or str(upstream_node.get("kind")) not in RIVERS_ASSET_NODE_KINDS - ): - continue - deps.append(rs.AssetDef.dep(translator.get_asset_name(upstream_node))) + for translated_node, upstream_nodes in rivers_asset_plans(dag): + deps: list[Any] = [ + rs.AssetDef.dep(translator.get_asset_name(upstream_node)) + for upstream_node in upstream_nodes + ] asset_defs.append( rs.AssetDef( name=translator.get_asset_name(translated_node), @@ -46,15 +35,3 @@ def build_asset_defs( ) ) return tuple(asset_defs) - - -def _nodes_by_id(dag: Mapping[str, Any]) -> dict[str, Mapping[str, Any]]: - return {str(node["id"]): node for node in dag["nodes"]} - - -def _upstream_by_id(dag: Mapping[str, Any]) -> dict[str, list[str]]: - upstream: dict[str, list[str]] = {} - edge: Mapping[str, Any] - for edge in dag["edges"]: - upstream.setdefault(str(edge["to_id"]), []).append(str(edge["from_id"])) - return upstream diff --git a/tests/unit/src/sqlbuild/integrations/dagster/_test_types.py b/tests/unit/src/sqlbuild/integrations/dagster/_test_types.py index 620bbf8231..1d5876813d 100644 --- a/tests/unit/src/sqlbuild/integrations/dagster/_test_types.py +++ b/tests/unit/src/sqlbuild/integrations/dagster/_test_types.py @@ -318,3 +318,11 @@ class DagsterUnitTestSelectorTestCase: select: str expected_code: str expected_message: str + + +@dataclass(frozen=True) +class DagsterCheckTranslatorCallTestCase: + description: str + include_scenarios: bool + expected_is_asset_check_ids: tuple[str, ...] + expected_checked_node_last_key: str diff --git a/tests/unit/src/sqlbuild/integrations/dagster/helpers.py b/tests/unit/src/sqlbuild/integrations/dagster/helpers.py index 4da70ff215..d97ed5225c 100644 --- a/tests/unit/src/sqlbuild/integrations/dagster/helpers.py +++ b/tests/unit/src/sqlbuild/integrations/dagster/helpers.py @@ -7,6 +7,9 @@ from typing import Any from sqlbuild._native import IntegrationResultEnvelope +from sqlbuild.integrations.dagster.classes.sqlbuild_dagster_translator import ( + SqlBuildDagsterTranslator, +) def assert_exhaustive_enum_partition[EnumT: StrEnum]( @@ -621,3 +624,19 @@ def build_dagster_selection_graph_dag() -> Mapping[str, Any]: ], "checks": [], } + + +class RecordingDagsterTranslator(SqlBuildDagsterTranslator): + """Default Dagster translator that records the checks and nodes it is asked about.""" + + def __init__(self) -> None: + self.asset_checks: list[Mapping[str, Any]] = [] + self.asset_nodes: list[Mapping[str, Any]] = [] + + def is_asset_node(self, node: Mapping[str, Any]) -> bool: + self.asset_nodes.append(node) + return super().is_asset_node(node) + + def is_asset_check(self, check: Mapping[str, Any]) -> bool: + self.asset_checks.append(check) + return super().is_asset_check(check) diff --git a/tests/unit/src/sqlbuild/integrations/dagster/test_assets.py b/tests/unit/src/sqlbuild/integrations/dagster/test_assets.py index 01b3f07982..31739b05c8 100644 --- a/tests/unit/src/sqlbuild/integrations/dagster/test_assets.py +++ b/tests/unit/src/sqlbuild/integrations/dagster/test_assets.py @@ -25,6 +25,7 @@ DagsterAssetSelectionErrorTestCase, DagsterAssetSelectionTestCase, DagsterAssetSpecTestCase, + DagsterCheckTranslatorCallTestCase, DagsterConflictingInputTestCase, DagsterDecoratorTestCase, DagsterDescriptionTestCase, @@ -35,6 +36,7 @@ DagsterUnitTestSelectorTestCase, ) from tests.unit.src.sqlbuild.integrations.dagster.helpers import ( + RecordingDagsterTranslator, build_dagster_selection_graph_dag, build_dagster_test_dag, build_python_augmented_dagster_test_dag, @@ -689,3 +691,40 @@ def assets_def() -> dg.MaterializeResult: assert raised.value.code == test_case.expected_code assert raised.value.message == test_case.expected_message + + +@pytest.mark.parametrize( + "test_case", + [ + DagsterCheckTranslatorCallTestCase( + description="excluded scenario checks are still offered to the translator", + include_scenarios=False, + expected_is_asset_check_ids=( + "audit:not_null:model:orders:order_id", + "audit:freshness:source:raw_orders:loaded_at", + "sql_scenario:orders_minimal", + "sql_scenario:customers_minimal", + ), + expected_checked_node_last_key="project_name", + ) + ], + ids=lambda case: case.description, +) +def test_given_check_filter_when_building_check_specs_then_translator_sees_every_check( + test_case: DagsterCheckTranslatorCallTestCase, +) -> None: + translator: RecordingDagsterTranslator = RecordingDagsterTranslator() + + _ = build_check_specs( + dag=build_dagster_test_dag(), + translator=translator, + include_scenarios=test_case.include_scenarios, + ) + checks: list[Mapping[str, Any]] = translator.asset_checks + checked_nodes: list[Mapping[str, Any]] = translator.asset_nodes + + assert tuple(str(check["id"]) for check in checks) == test_case.expected_is_asset_check_ids + assert checked_nodes + assert all( + tuple(node)[-1] == test_case.expected_checked_node_last_key for node in checked_nodes + ) diff --git a/tests/unit/src/sqlbuild/integrations/rivers/_test_types.py b/tests/unit/src/sqlbuild/integrations/rivers/_test_types.py index 32e5fc67ba..7fce8bdc2b 100644 --- a/tests/unit/src/sqlbuild/integrations/rivers/_test_types.py +++ b/tests/unit/src/sqlbuild/integrations/rivers/_test_types.py @@ -21,3 +21,11 @@ class RiversPythonArtifactCompatibilityTestCase: expected_task_group: str expected_asset_group: str expected_asset_metadata_keys: tuple[str, ...] + + +@dataclass(frozen=True) +class RiversTranslatorArgumentTestCase: + description: str + node_id: str + expected_dependency_name_count: int + expected_owned_node_keys_end: tuple[str, ...] diff --git a/tests/unit/src/sqlbuild/integrations/rivers/helpers.py b/tests/unit/src/sqlbuild/integrations/rivers/helpers.py index 18d6ff44ba..4f520eb34e 100644 --- a/tests/unit/src/sqlbuild/integrations/rivers/helpers.py +++ b/tests/unit/src/sqlbuild/integrations/rivers/helpers.py @@ -1,6 +1,12 @@ from __future__ import annotations +from collections.abc import Mapping from dataclasses import dataclass +from typing import Any + +from sqlbuild.integrations.rivers.classes.sqlbuild_rivers_translator import ( + SqlBuildRiversTranslator, +) @dataclass(frozen=True) @@ -25,3 +31,14 @@ def dep(name: str) -> FakeRiversAssetDep: @dataclass(frozen=True) class FakeRiversModule: AssetDef: type[FakeRiversAssetDef] = FakeRiversAssetDef + + +class RecordingRiversTranslator(SqlBuildRiversTranslator): + """Default Rivers translator that records named nodes by id and project-name presence.""" + + def __init__(self) -> None: + self.named_nodes: dict[tuple[str, bool], list[Mapping[str, Any]]] = {} + + def get_asset_name(self, node: Mapping[str, Any]) -> str: + self.named_nodes.setdefault((str(node["id"]), "project_name" in node), []).append(node) + return super().get_asset_name(node) diff --git a/tests/unit/src/sqlbuild/integrations/rivers/test_assets.py b/tests/unit/src/sqlbuild/integrations/rivers/test_assets.py index 5850aa6b39..755d55f294 100644 --- a/tests/unit/src/sqlbuild/integrations/rivers/test_assets.py +++ b/tests/unit/src/sqlbuild/integrations/rivers/test_assets.py @@ -15,8 +15,12 @@ ) from tests.unit.src.sqlbuild.integrations.rivers._test_types import ( RiversPythonArtifactCompatibilityTestCase, + RiversTranslatorArgumentTestCase, +) +from tests.unit.src.sqlbuild.integrations.rivers.helpers import ( + FakeRiversModule, + RecordingRiversTranslator, ) -from tests.unit.src.sqlbuild.integrations.rivers.helpers import FakeRiversModule @pytest.mark.parametrize( @@ -77,3 +81,35 @@ def test_given_python_augmented_dag_when_building_asset_defs_then_maps_python_no assert task_def.group == test_case.expected_task_group assert python_asset_def.group == test_case.expected_asset_group assert all(key in python_asset_def.metadata for key in test_case.expected_asset_metadata_keys) + + +@pytest.mark.parametrize( + "test_case", + [ + RiversTranslatorArgumentTestCase( + description="dependencies are the artifact's own nodes without the project name", + node_id="model:orders", + expected_dependency_name_count=1, + expected_owned_node_keys_end=("materialization_type", "project_name"), + ) + ], + ids=lambda case: case.description, +) +def test_given_python_augmented_dag_when_building_asset_defs_then_translator_sees_artifact_nodes( + test_case: RiversTranslatorArgumentTestCase, + monkeypatch: pytest.MonkeyPatch, +) -> None: + dag: Mapping[str, Any] = build_python_augmented_dagster_test_dag() + monkeypatch.setattr(rivers_assets, "load_rivers", lambda: FakeRiversModule()) + translator: RecordingRiversTranslator = RecordingRiversTranslator() + + _ = build_asset_defs(dag=dag, translator=translator) + artifact_node: Mapping[str, Any] = {str(node["id"]): node for node in dag["nodes"]}[ + test_case.node_id + ] + dependencies: list[Mapping[str, Any]] = translator.named_nodes[(test_case.node_id, False)] + owned_nodes: list[Mapping[str, Any]] = translator.named_nodes[(test_case.node_id, True)] + + assert len(dependencies) == test_case.expected_dependency_name_count + assert all(dependency is artifact_node for dependency in dependencies) + assert [tuple(node)[-2:] for node in owned_nodes] == [test_case.expected_owned_node_keys_end] From 9a0706b5e0bc38e972debf7853b32c93bad19073 Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Thu, 15 Oct 2026 22:28:11 +0100 Subject: [PATCH 23/48] refactor: host planner value records on the shared native record mechanism --- .../fixture_inferences.rs | 0 .../bindings/_helpers/analysis_session/mod.rs | 1 + .../_helpers/analysis_session/registration.rs | 7 +- .../_helpers/compile_records/constants.rs | 17 + .../_helpers/compiler_records/constants.rs | 4 + .../_helpers/contract_records/constants.rs | 1 + .../src/bindings/_helpers/mod.rs | 1 - .../_helpers/planner_models/constants.rs | 51 -- .../bindings/_helpers/planner_models/enums.rs | 19 - .../bindings/_helpers/planner_models/mod.rs | 8 - .../_helpers/planner_models/record.rs | 214 ----- .../_helpers/planner_models/records.rs | 770 ------------------ .../_helpers/planner_models/registration.rs | 83 -- .../_helpers/planner_records/constants.rs | 715 +++++++++++++++- .../planner_registration/registration.rs | 7 +- .../_helpers/record_behaviours/constants.rs | 12 +- .../_helpers/record_behaviours/planner.rs | 42 + .../bindings/_helpers/records/constants.rs | 6 +- .../src/bindings/_helpers/records/fields.rs | 6 +- .../src/bindings/_helpers/records/models.rs | 24 +- .../bindings/_helpers/records/registration.rs | 2 +- .../_helpers/scope_records/constants.rs | 1 + src/sqlbuild/_native.pyi | 220 ++--- 23 files changed, 882 insertions(+), 1329 deletions(-) rename crates/sqlbuild-python/src/bindings/_helpers/{planner_models => analysis_session}/fixture_inferences.rs (100%) delete mode 100644 crates/sqlbuild-python/src/bindings/_helpers/planner_models/constants.rs delete mode 100644 crates/sqlbuild-python/src/bindings/_helpers/planner_models/enums.rs delete mode 100644 crates/sqlbuild-python/src/bindings/_helpers/planner_models/mod.rs delete mode 100644 crates/sqlbuild-python/src/bindings/_helpers/planner_models/record.rs delete mode 100644 crates/sqlbuild-python/src/bindings/_helpers/planner_models/records.rs delete mode 100644 crates/sqlbuild-python/src/bindings/_helpers/planner_models/registration.rs diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_models/fixture_inferences.rs b/crates/sqlbuild-python/src/bindings/_helpers/analysis_session/fixture_inferences.rs similarity index 100% rename from crates/sqlbuild-python/src/bindings/_helpers/planner_models/fixture_inferences.rs rename to crates/sqlbuild-python/src/bindings/_helpers/analysis_session/fixture_inferences.rs diff --git a/crates/sqlbuild-python/src/bindings/_helpers/analysis_session/mod.rs b/crates/sqlbuild-python/src/bindings/_helpers/analysis_session/mod.rs index baca587c27..5773febd9c 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/analysis_session/mod.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/analysis_session/mod.rs @@ -1,4 +1,5 @@ pub(crate) mod fixture_columns; +pub(crate) mod fixture_inferences; pub(crate) mod model_analyses; pub(crate) mod nullability_rules; pub(crate) mod oracle; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/analysis_session/registration.rs b/crates/sqlbuild-python/src/bindings/_helpers/analysis_session/registration.rs index 04b7a39550..0c43aea023 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/analysis_session/registration.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/analysis_session/registration.rs @@ -1,15 +1,16 @@ //! Register the native model analysis session functions. -use pyo3::prelude::{Bound, PyModule, PyResult}; +use pyo3::prelude::{Bound, PyModule, PyModuleMethods, PyResult}; use crate::bindings::_helpers::analysis_session::{ - fixture_columns, model_analyses, oracle, outcome_objects, query_columns, session, - stage_functions, + fixture_columns, fixture_inferences, model_analyses, oracle, outcome_objects, query_columns, + session, stage_functions, }; pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { session::register(module)?; fixture_columns::register(module)?; + module.add_class::()?; model_analyses::register(module)?; outcome_objects::register(module)?; query_columns::register(module)?; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compile_records/constants.rs b/crates/sqlbuild-python/src/bindings/_helpers/compile_records/constants.rs index b868c74650..f47fb226b8 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compile_records/constants.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compile_records/constants.rs @@ -84,12 +84,14 @@ const R_DECLARATION_SCOPE_RESOLVER: RecordSpec = RecordSpec { name: "native_contexts", default: FieldDefault::Required, compared: false, + shown: false, }, FieldSpec::new("resource_specific", FieldDefault::None), FieldSpec { name: "contexts_by_directory", default: FieldDefault::EmptyDict, compared: false, + shown: false, }, ], }; @@ -221,6 +223,7 @@ const R_COMPILE_ANALYSIS_SELECTION: RecordSpec = RecordSpec { name: "on_inputs_ready", default: FieldDefault::None, compared: false, + shown: false, }, ], }; @@ -304,6 +307,7 @@ const R_COMPILE_PROJECT_INPUTS: RecordSpec = RecordSpec { name: "macro_context", default: FieldDefault::None, compared: false, + shown: false, }, FieldSpec::new("loaded_macros", FieldDefault::EmptyDict), FieldSpec::new("public_enums", FieldDefault::EmptyDict), @@ -325,11 +329,13 @@ const R_COMPILE_PROJECT_INPUTS: RecordSpec = RecordSpec { name: "declaration_scope", default: FieldDefault::None, compared: false, + shown: false, }, FieldSpec { name: "native_project", default: FieldDefault::None, compared: false, + shown: false, }, ], }; @@ -1154,6 +1160,7 @@ const R_DISCOVERED_PROJECT_INPUTS: RecordSpec = RecordSpec { name: "native_session", default: FieldDefault::None, compared: false, + shown: false, }, ], }; @@ -1258,21 +1265,25 @@ const R_MACRO_CONTEXT: RecordSpec = RecordSpec { name: "_value_renderer", default: FieldDefault::None, compared: false, + shown: false, }, FieldSpec { name: "_collection_rendering", default: FieldDefault::Member("sqlbuild._native", "CollectionRendering", "VALUE_LIST"), compared: false, + shown: false, }, FieldSpec { name: "_constant_declarations", default: FieldDefault::EmptyDict, compared: false, + shown: false, }, FieldSpec { name: "_enforce_explicit_references", default: FieldDefault::True, compared: false, + shown: false, }, ], }; @@ -1376,11 +1387,13 @@ const R_COMPILE_MODEL_INPUT: RecordSpec = RecordSpec { name: "sql_expansion", default: FieldDefault::None, compared: false, + shown: false, }, FieldSpec { name: "lint_expansion", default: FieldDefault::None, compared: false, + shown: false, }, ], }; @@ -1498,6 +1511,7 @@ const R_COMPILED_PROJECT: RecordSpec = RecordSpec { name: "binding_catalog", default: FieldDefault::None, compared: false, + shown: false, }, FieldSpec::new("effective_target_database", FieldDefault::None), FieldSpec::new("effective_target_schema", FieldDefault::None), @@ -1541,16 +1555,19 @@ const R_COMPILED_PROJECT: RecordSpec = RecordSpec { name: "sql_expansions", default: FieldDefault::EmptyDict, compared: false, + shown: false, }, FieldSpec { name: "native_session", default: FieldDefault::None, compared: false, + shown: false, }, FieldSpec { name: "native_project", default: FieldDefault::None, compared: false, + shown: false, }, ], }; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compiler_records/constants.rs b/crates/sqlbuild-python/src/bindings/_helpers/compiler_records/constants.rs index 0e2151ad76..b34dea553e 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compiler_records/constants.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compiler_records/constants.rs @@ -419,6 +419,7 @@ const R_LINT_CONFIG: RecordSpec = RecordSpec { name: "relation_keys", default: FieldDefault::EmptyDict, compared: false, + shown: false, }, ], }; @@ -433,6 +434,7 @@ const R_LINT_RUN_RESULT: RecordSpec = RecordSpec { name: "source_texts", default: FieldDefault::EmptyDict, compared: false, + shown: false, }, FieldSpec::new("unexpandable", FieldDefault::EmptyDict), ], @@ -674,6 +676,7 @@ const R_SQL_SCHEMA_VALIDATION_REQUEST: RecordSpec = RecordSpec { name: "catalog", default: FieldDefault::None, compared: false, + shown: false, }, ], }; @@ -962,6 +965,7 @@ const R_PROJECT_GRAPH: RecordSpec = RecordSpec { name: "native", default: FieldDefault::Required, compared: false, + shown: false, }, ], }; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/contract_records/constants.rs b/crates/sqlbuild-python/src/bindings/_helpers/contract_records/constants.rs index afd0ce4986..4b9a90b52a 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/contract_records/constants.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/contract_records/constants.rs @@ -590,6 +590,7 @@ const R_SCHEMA_AUDIT_INSTANCE: RecordSpec = RecordSpec { name: "location", default: FieldDefault::None, compared: false, + shown: false, }, ], }; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/mod.rs b/crates/sqlbuild-python/src/bindings/_helpers/mod.rs index d5320e8f37..85851a32d6 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/mod.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/mod.rs @@ -61,7 +61,6 @@ pub(crate) mod planner_changes; pub(crate) mod planner_entries; pub(crate) mod planner_identity; pub(crate) mod planner_migrations; -pub(crate) mod planner_models; pub(crate) mod planner_phases; pub(crate) mod planner_records; pub(crate) mod planner_registration; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_models/constants.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_models/constants.rs deleted file mode 100644 index 7680642569..0000000000 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_models/constants.rs +++ /dev/null @@ -1,51 +0,0 @@ -//! Planner enumerations declared natively. - -/// Each planner `StrEnum`: its name and its members with their values, in declaration order. -pub(crate) const PLANNER_STR_ENUMS: &[(&str, &[(&str, &str)])] = &[ - ( - "GraphResourceKind", - &[ - ("MODEL", "model"), - ("SEED", "seed"), - ("SOURCE", "source"), - ("FUNCTION", "function"), - ], - ), - ( - "ChangeKind", - &[ - ("FIRST_RUN", "first_run"), - ("RENAMED", "renamed"), - ("QUERY_CHANGED", "query_changed"), - ("FUNCTION_CHANGED", "function_changed"), - ("CONFIG_CHANGED", "config_changed"), - ("SCHEMA_CHANGED", "schema_changed"), - ("NO_CHANGE", "no_change"), - ], - ), - ( - "SchemaChangeKind", - &[ - ("COLUMN_ADDED", "column_added"), - ("COLUMN_REMOVED", "column_removed"), - ("COLUMN_TYPE_CHANGED", "column_type_changed"), - ], - ), - ( - "SchemaColumnSource", - &[("YML", "yml"), ("SQL_ANALYSIS", "sql_analysis")], - ), - ("CursorWatermarkMode", &[("ALL", "all"), ("ANY", "any")]), - ( - "SchemaActionKind", - &[ - ("ADD_COLUMN", "add_column"), - ("DROP_COLUMN", "drop_column"), - ("ALTER_COLUMN_TYPE", "alter_column_type"), - ], - ), - ( - "WarningSeverity", - &[("ERROR", "error"), ("WARNING", "warning"), ("INFO", "info")], - ), -]; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_models/enums.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_models/enums.rs deleted file mode 100644 index cca4508a64..0000000000 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_models/enums.rs +++ /dev/null @@ -1,19 +0,0 @@ -//! Creates the planner `StrEnum` classes in the native module. - -use crate::bindings::_helpers::compile_driver::python_calls::imported; -use crate::bindings::_helpers::planner_models::constants::PLANNER_STR_ENUMS; -use pyo3::prelude::{Bound, PyAnyMethods, PyModule, PyModuleMethods, PyResult}; -use pyo3::types::{PyDict, PyDictMethods, PyList}; - -pub(crate) fn register_enums(module: &Bound<'_, PyModule>) -> PyResult<()> { - let py = module.py(); - let str_enum = imported(py, "enum", "StrEnum")?; - for (name, members) in PLANNER_STR_ENUMS { - let options = PyDict::new(py); - options.set_item("module", "sqlbuild._native")?; - options.set_item("qualname", *name)?; - let class = str_enum.call((*name, PyList::new(py, members.iter())?), Some(&options))?; - module.add(*name, class)?; - } - Ok(()) -} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_models/mod.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_models/mod.rs deleted file mode 100644 index 0be40906ca..0000000000 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_models/mod.rs +++ /dev/null @@ -1,8 +0,0 @@ -//! Planner value records, enumerations and caches hosted natively. - -pub(crate) mod constants; -pub(crate) mod enums; -pub(crate) mod fixture_inferences; -pub(crate) mod record; -pub(crate) mod records; -pub(crate) mod registration; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_models/record.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_models/record.rs deleted file mode 100644 index f9cac51993..0000000000 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_models/record.rs +++ /dev/null @@ -1,214 +0,0 @@ -//! Frozen value records with dataclass construction, equality, hashing and repr. - -use crate::bindings::_helpers::compile_driver::python_calls::imported; -use pyo3::IntoPyObject; -use pyo3::exceptions::PyTypeError; -use pyo3::prelude::{Bound, Py, PyAny, PyAnyMethods, PyResult, Python}; -use pyo3::types::{PyDict, PyDictMethods, PyFrozenSet, PyString, PyTuple, PyTupleMethods}; - -/// How a record field is filled when the caller leaves it out. -pub(crate) enum FieldDefault { - /// The caller must pass the field. - Required, - /// `None`. - NoneValue, - /// A boolean. - Bool(bool), - /// A string. - Text(&'static str), - /// A new empty tuple. - EmptyTuple, - /// A new empty frozenset. - EmptyFrozenSet, - /// A new empty dict. - EmptyDict, - /// An enum member, by class and member name. - Member((&'static str, &'static str), &'static str), - /// A new instance of a class called without arguments. - Class((&'static str, &'static str)), - /// A value built by a function. - Factory(for<'py> fn(Python<'py>) -> PyResult>), -} - -/// One record field. -pub(crate) struct FieldSpec { - pub(crate) name: &'static str, - pub(crate) default: FieldDefault, - pub(crate) compare: bool, - pub(crate) repr: bool, -} - -/// The fields of one record class. -pub(crate) struct RecordSpec { - pub(crate) name: &'static str, - pub(crate) kw_only: bool, - pub(crate) fields: &'static [FieldSpec], -} - -fn default_value<'py>(py: Python<'py>, default: &FieldDefault) -> PyResult> { - match default { - FieldDefault::Required => Ok(py.None().into_bound(py)), - FieldDefault::NoneValue => Ok(py.None().into_bound(py)), - FieldDefault::Bool(value) => Ok(value.into_pyobject(py)?.to_owned().into_any()), - FieldDefault::Text(value) => Ok(PyString::new(py, value).into_any()), - FieldDefault::EmptyTuple => Ok(PyTuple::empty(py).into_any()), - FieldDefault::EmptyFrozenSet => Ok(PyFrozenSet::empty(py)?.into_any()), - FieldDefault::EmptyDict => Ok(PyDict::new(py).into_any()), - FieldDefault::Member((module, class), member) => { - imported(py, module, class)?.getattr(*member) - } - FieldDefault::Class((module, class)) => imported(py, module, class)?.call0(), - FieldDefault::Factory(build) => build(py), - } -} - -/// Bind constructor arguments to the record's fields in order, as a dataclass `__init__` does. -pub(crate) fn bind_fields( - spec: &RecordSpec, - args: &Bound<'_, PyTuple>, - kwargs: Option<&Bound<'_, PyDict>>, -) -> PyResult>> { - let py = args.py(); - if args.len() > 0 && spec.kw_only { - return Err(PyTypeError::new_err(format!( - "{}.__init__() takes 1 positional argument but {} were given", - spec.name, - args.len() + 1 - ))); - } - if args.len() > spec.fields.len() { - return Err(PyTypeError::new_err(format!( - "{}.__init__() takes {} positional arguments but {} were given", - spec.name, - spec.fields.len() + 1, - args.len() + 1 - ))); - } - let mut values: Vec>> = args.iter().map(Some).collect(); - values.resize_with(spec.fields.len(), || None); - if let Some(kwargs) = kwargs { - for (key, value) in kwargs.iter() { - let key: String = key.extract()?; - let Some(index) = spec.fields.iter().position(|field| field.name == key) else { - return Err(PyTypeError::new_err(format!( - "{}.__init__() got an unexpected keyword argument '{key}'", - spec.name - ))); - }; - if values[index].is_some() { - return Err(PyTypeError::new_err(format!( - "{}.__init__() got multiple values for argument '{key}'", - spec.name - ))); - } - values[index] = Some(value); - } - } - spec.fields - .iter() - .zip(values) - .map(|(field, value)| match value { - Some(value) => Ok(value.unbind()), - None if matches!(field.default, FieldDefault::Required) => { - Err(PyTypeError::new_err(format!( - "{}.__init__() missing 1 required argument: '{}'", - spec.name, field.name - ))) - } - None => Ok(default_value(py, &field.default)?.unbind()), - }) - .collect() -} - -/// The current field values with `changes` applied, as `dataclasses.replace` binds them. -pub(crate) fn replaced_fields( - spec: &RecordSpec, - current: Vec>, - changes: Option<&Bound<'_, PyDict>>, -) -> PyResult>> { - let mut values: Vec> = current.into_iter().map(Bound::unbind).collect(); - let Some(changes) = changes else { - return Ok(values); - }; - for (key, value) in changes.iter() { - let key: String = key.extract()?; - let Some(index) = spec.fields.iter().position(|field| field.name == key) else { - return Err(PyTypeError::new_err(format!( - "{}.__init__() got an unexpected keyword argument '{key}'", - spec.name - ))); - }; - values[index] = value.unbind(); - } - Ok(values) -} - -fn compared<'py>( - py: Python<'py>, - spec: &RecordSpec, - values: &[Bound<'py, PyAny>], -) -> PyResult> { - let kept: Vec> = spec - .fields - .iter() - .zip(values) - .filter(|(field, _)| field.compare) - .map(|(_, value)| value.clone()) - .collect(); - PyTuple::new(py, kept) -} - -/// Dataclass equality: same class and equal compared fields. -pub(crate) fn record_eq<'py>( - spec: &RecordSpec, - (left, right): (&[Bound<'py, PyAny>], Option<&[Bound<'py, PyAny>]>), - py: Python<'py>, -) -> PyResult> { - let Some(right) = right else { - return Ok(py.NotImplemented()); - }; - let equal: bool = compared(py, spec, left)?.eq(compared(py, spec, right)?)?; - Ok(equal.into_pyobject(py)?.to_owned().into_any().unbind()) -} - -/// Dataclass hash: the hash of the compared fields. -pub(crate) fn record_hash<'py>( - spec: &RecordSpec, - values: &[Bound<'py, PyAny>], - py: Python<'py>, -) -> PyResult { - compared(py, spec, values)?.hash() -} - -/// Dataclass repr: `Name(field=value, ...)` over the represented fields. -pub(crate) fn record_repr(spec: &RecordSpec, values: &[Bound<'_, PyAny>]) -> PyResult { - let mut parts: Vec = Vec::new(); - for (field, value) in spec.fields.iter().zip(values) { - if field.repr { - parts.push(format!("{}={}", field.name, value.repr()?)); - } - } - Ok(format!("{}({})", spec.name, parts.join(", "))) -} - -/// Give a record class the dataclass field table `dataclasses.replace`, `fields` and `asdict` read. -pub(crate) fn install_dataclass_fields( - class: &Bound<'_, PyAny>, - spec: &RecordSpec, -) -> PyResult<()> { - let py = class.py(); - let dataclasses = py.import("dataclasses")?; - let field_kind = dataclasses.getattr("_FIELD")?; - let table = PyDict::new(py); - for field in spec.fields { - let options = PyDict::new(py); - options.set_item("compare", field.compare)?; - options.set_item("repr", field.repr)?; - let described = dataclasses.call_method("field", (), Some(&options))?; - described.setattr("name", field.name)?; - described.setattr("type", "object")?; - described.setattr("_field_type", &field_kind)?; - table.set_item(field.name, described)?; - } - class.setattr("__dataclass_fields__", table) -} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_models/records.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_models/records.rs deleted file mode 100644 index 2dc22e5db5..0000000000 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_models/records.rs +++ /dev/null @@ -1,770 +0,0 @@ -//! Planner value records the native planner builds and reads. - -use crate::bindings::_helpers::compile_driver::python_calls::imported; -use crate::bindings::_helpers::planner_models::record::FieldDefault; -use pyo3::prelude::{Bound, Py, PyAny, PyAnyMethods, PyResult, Python}; -use pyo3::types::PyDict; - -/// Declares a frozen record class over `Py` fields. -macro_rules! planner_record { - (@convert $py:ident, $values:expr) => {{ - let _ = $py; - $values - }}; - (@convert $py:ident, $values:expr, $convert:path) => { $convert($py, $values)? }; - ( - $(#[$meta:meta])* - $name:ident, $kw_only:literal, [ - $(($field:ident = $field_name:literal, $default:expr, $compare:literal, $repr:literal)),* $(,)? - ] - $(, convert = $convert:path)? - $(, methods { $($methods:tt)* })? - ) => { - $(#[$meta])* - #[pyo3::pyclass(module = "sqlbuild._native", frozen)] - pub(crate) struct $name { - $( - #[pyo3(get)] - $field: pyo3::Py, - )* - } - - impl $name { - const SPEC: $crate::bindings::_helpers::planner_models::record::RecordSpec = - $crate::bindings::_helpers::planner_models::record::RecordSpec { - name: stringify!($name), - kw_only: $kw_only, - fields: &[$( - $crate::bindings::_helpers::planner_models::record::FieldSpec { - name: $field_name, - default: $default, - compare: $compare, - repr: $repr, - }, - )*], - }; - - /// Add the class to `module` with its dataclass field table. - pub(crate) fn register_class( - module: &pyo3::Bound<'_, pyo3::types::PyModule>, - ) -> pyo3::PyResult<()> { - use pyo3::types::{PyAnyMethods, PyModuleMethods}; - module.add_class::()?; - $crate::bindings::_helpers::planner_models::record::install_dataclass_fields( - &module.getattr(stringify!($name))?, - &Self::SPEC, - ) - } - - fn values<'py>(&self, py: pyo3::Python<'py>) -> Vec> { - vec![$(self.$field.bind(py).clone()),*] - } - - fn from_values(values: Vec>) -> pyo3::PyResult { - let mut values = values.into_iter(); - Ok(Self { - $( - $field: values.next().ok_or_else(|| { - pyo3::exceptions::PyTypeError::new_err(concat!("missing ", $field_name)) - })?, - )* - }) - } - } - - #[pyo3::pymethods] - impl $name { - #[new] - #[pyo3(signature = (*args, **kwargs))] - fn __new__( - args: &pyo3::Bound<'_, pyo3::types::PyTuple>, - kwargs: Option<&pyo3::Bound<'_, pyo3::types::PyDict>>, - ) -> pyo3::PyResult { - let py = args.py(); - let values = $crate::bindings::_helpers::planner_models::record::bind_fields( - &Self::SPEC, - args, - kwargs, - )?; - Self::from_values(planner_record!(@convert py, values $(, $convert)?)) - } - - #[pyo3(signature = (**changes))] - fn __replace__( - &self, - py: pyo3::Python<'_>, - changes: Option<&pyo3::Bound<'_, pyo3::types::PyDict>>, - ) -> pyo3::PyResult { - let values = $crate::bindings::_helpers::planner_models::record::replaced_fields( - &Self::SPEC, - self.values(py), - changes, - )?; - Self::from_values(planner_record!(@convert py, values $(, $convert)?)) - } - - fn __eq__( - &self, - other: &pyo3::Bound<'_, pyo3::PyAny>, - ) -> pyo3::PyResult> { - let py = other.py(); - let right = match other.cast::() { - Ok(right) => Some(right.get().values(py)), - Err(_) => None, - }; - $crate::bindings::_helpers::planner_models::record::record_eq( - &Self::SPEC, - (&self.values(py), right.as_deref()), - py, - ) - } - - fn __hash__(&self, py: pyo3::Python<'_>) -> pyo3::PyResult { - $crate::bindings::_helpers::planner_models::record::record_hash( - &Self::SPEC, - &self.values(py), - py, - ) - } - - fn __repr__(&self, py: pyo3::Python<'_>) -> pyo3::PyResult { - $crate::bindings::_helpers::planner_models::record::record_repr( - &Self::SPEC, - &self.values(py), - ) - } - - $($($methods)*)? - } - }; -} - -fn forward_only_backfill(py: Python<'_>) -> PyResult> { - let kwargs = PyDict::new(py); - kwargs.set_item( - "action", - imported(py, "sqlbuild._native", "BackfillAction")?.getattr("FORWARD_ONLY")?, - )?; - imported(py, "sqlbuild._native", "BackfillResult")?.call((), Some(&kwargs)) -} - -fn utc_now(py: Python<'_>) -> PyResult> { - imported(py, "datetime", "datetime")?.call_method1("now", (imported(py, "datetime", "UTC")?,)) -} - -fn scenario_artifact_kind(py: Python<'_>, mut values: Vec>) -> PyResult>> { - if let Some(kind) = values.first_mut() { - *kind = imported(py, "sqlbuild._native", "ScenarioArtifactKind")? - .call1((kind.bind(py),))? - .unbind(); - } - Ok(values) -} - -fn rename_hint_message( - identical: &Bound<'_, PyAny>, - candidates: &Bound<'_, PyAny>, - added: &Bound<'_, PyAny>, -) -> PyResult { - let relation: &str = if identical.is_truthy()? { - "same expression as" - } else { - "similar to" - }; - let candidates: Vec = candidates - .try_iter()? - .map(|candidate| Ok(candidate?.str()?.to_string())) - .collect::>()?; - let origin: &str = match candidates.as_slice() { - [only] => only, - _ => "", - }; - Ok(format!( - "{relation} {}; if this is a rename, add {} (migrate_from {origin})", - candidates.join(", "), - added.str()? - )) -} - -planner_record! { - /// One native-planned SQL-test artifact and its distinct planning error messages. - NativeSqlTestArtifact, false, [ - (sql = "sql", FieldDefault::Required, true, true), - (model_names = "model_names", FieldDefault::Required, true, true), - (error_messages = "error_messages", FieldDefault::EmptyTuple, true, true), - ] -} - -planner_record! { - /// One statically provable invalid fixture condition. - RelationFixtureDiagnostic, false, [ - (key = "key", FieldDefault::Required, true, true), - (message = "message", FieldDefault::Required, true, true), - ] -} - -planner_record! { - /// Completed fixture SQL plus metadata reused by SQL-test validation. - RelationFixtureCompletion, false, [ - (fixture_sql_by_key = "fixture_sql_by_key", FieldDefault::Required, true, true), - (inferred_by_fixture = "inferred_by_fixture", FieldDefault::Required, true, true), - (expected_types = "expected_types", FieldDefault::Required, true, true), - (diagnostics = "diagnostics", FieldDefault::Required, true, true), - ] -} - -planner_record! { - /// Known type and nullability for one relation fixture column. - FixtureColumnMetadata, false, [ - (name = "name", FieldDefault::Required, true, true), - (r#type = "type", FieldDefault::Required, true, true), - (nullable = "nullable", FieldDefault::Required, true, true), - ] -} - -planner_record! { - /// Known columns and whether they form an authoritative relation shape. - FixtureRelationMetadata, false, [ - (columns = "columns", FieldDefault::Required, true, true), - (authoritative_names = "authoritative_names", FieldDefault::Required, true, true), - ] -} - -planner_record! { - /// Project-wide fixture metadata shared across test and scenario planning. - RelationFixturePlanningContext, false, [ - (models_by_name = "models_by_name", FieldDefault::Required, true, true), - (relations = "relations", FieldDefault::Required, true, true), - (authoritative_columns = "authoritative_columns", FieldDefault::Required, true, true), - (expected_types = "expected_types", FieldDefault::Required, true, true), - (fixture_inferences = "fixture_inferences", FieldDefault::Class(("sqlbuild._native", "FixtureColumnInferences")), false, true), - ] -} - -planner_record! { - /// One selector split into graph expansion flags and core text. - SelectorExpansion, false, [ - (core = "core", FieldDefault::Required, true, true), - (upstream = "upstream", FieldDefault::Bool(false), true, true), - (downstream = "downstream", FieldDefault::Bool(false), true, true), - ] -} - -planner_record! { - /// Parsed physical name for one planner-owned scenario artifact. - ParsedScenarioArtifactName, false, [ - (hash_prefix = "hash_prefix", FieldDefault::Required, true, true), - (kind = "kind", FieldDefault::Required, true, true), - (logical_name = "logical_name", FieldDefault::Required, true, true), - ] -} - -planner_record! { - /// One upstream dependency missing from both scope and warehouse. - MissingUpstream, false, [ - (key = "key", FieldDefault::Required, true, true), - (required_by = "required_by", FieldDefault::Required, true, true), - ] -} - -planner_record! { - /// Latest direct fingerprints grouped by node type. - WarehouseFingerprints, false, [ - (models = "models", FieldDefault::EmptyDict, true, true), - (functions = "functions", FieldDefault::EmptyDict, true, true), - (seeds = "seeds", FieldDefault::EmptyDict, true, true), - (python_nodes = "python_nodes", FieldDefault::EmptyDict, true, true), - (unfiltered_schemas = "unfiltered_schemas", FieldDefault::EmptyDict, false, false), - (unfiltered_database = "unfiltered_database", FieldDefault::NoneValue, false, false), - ] -} - -planner_record! { - /// Execution selection used for cursor validation and runtime producer resolution. - CursorSnapshotScope, false, [ - (model_keys = "model_keys", FieldDefault::Required, true, true), - (runtime_producer_keys = "runtime_producer_keys", FieldDefault::Required, true, true), - (invocation_time = "invocation_time", FieldDefault::NoneValue, true, true), - (start_cursor_config = "start_cursor_config", FieldDefault::NoneValue, true, true), - (cursor_overrides = "cursor_overrides", FieldDefault::NoneValue, true, true), - ] -} - -planner_record! { - /// One model rename that downstream references may have been rewritten for. - ReferenceRename, false, [ - (new_name = "new_name", FieldDefault::Required, true, true), - (origin_name = "origin_name", FieldDefault::Required, true, true), - (recorded_at = "recorded_at", FieldDefault::NoneValue, true, true), - ] -} - -planner_record! { - /// Frozen point-in-time picture of warehouse state for planning. - WarehouseSnapshot, false, [ - (existing_relations = "existing_relations", FieldDefault::EmptyDict, true, true), - (existing_columns = "existing_columns", FieldDefault::EmptyDict, true, true), - (fingerprints = "fingerprints", FieldDefault::Class(("sqlbuild._native", "WarehouseFingerprints")), true, true), - (cursor_snapshots = "cursor_snapshots", FieldDefault::EmptyDict, true, true), - (source_freshness_state_schemas = "source_freshness_state_schemas", FieldDefault::EmptyFrozenSet, true, true), - (column_dialect = "column_dialect", FieldDefault::NoneValue, true, true), - (renamed_models = "renamed_models", FieldDefault::EmptyFrozenSet, true, true), - (reference_renames = "reference_renames", FieldDefault::EmptyTuple, true, true), - (migration_state_schemas = "migration_state_schemas", FieldDefault::EmptyFrozenSet, true, true), - (old_name_view_state_schemas = "old_name_view_state_schemas", FieldDefault::EmptyFrozenSet, true, true), - (listed_state_schemas = "listed_state_schemas", FieldDefault::EmptyFrozenSet, true, true), - ] -} - -planner_record! { - /// Neutral node identity for shared selection-aware staleness classification. - SelectionStalenessNodeKey, false, [ - (resource_type = "resource_type", FieldDefault::Required, true, true), - (name = "name", FieldDefault::Required, true, true), - ] -} - -planner_record! { - /// Neutral graph inputs for selected-model stale warning classification. - SelectionStalenessGraph, false, [ - (upstream_deps = "upstream_deps", FieldDefault::Required, true, true), - (selected_model_names = "selected_model_names", FieldDefault::Required, true, true), - (run_model_names = "run_model_names", FieldDefault::Required, true, true), - (run_seed_names = "run_seed_names", FieldDefault::Required, true, true), - (run_source_names = "run_source_names", FieldDefault::Required, true, true), - (changed_model_names = "changed_model_names", FieldDefault::Required, true, true), - (changed_seed_names = "changed_seed_names", FieldDefault::Required, true, true), - (changed_source_names = "changed_source_names", FieldDefault::Required, true, true), - ] -} - -planner_record! { - /// Neutral stale warning classification for one selected model. - SelectionStalenessWarning, false, [ - (model_name = "model_name", FieldDefault::Required, true, true), - (trigger_names = "trigger_names", FieldDefault::Required, true, true), - ] -} - -planner_record! { - /// Resolved backfill action from a change detection policy. - BackfillResult, false, [ - (action = "action", FieldDefault::Required, true, true), - (duration = "duration", FieldDefault::NoneValue, true, true), - ] -} - -planner_record! { - /// Per-model output from change detection and policy resolution. - ChangeDetectionResult, false, [ - (model_name = "model_name", FieldDefault::Required, true, true), - (change_kind = "change_kind", FieldDefault::Required, true, true), - (query_changed = "query_changed", FieldDefault::Bool(false), true, true), - (config_changed = "config_changed", FieldDefault::Bool(false), true, true), - (fingerprint_metadata_json = "fingerprint_metadata_json", FieldDefault::NoneValue, true, true), - (previous_metadata_json = "previous_metadata_json", FieldDefault::NoneValue, true, true), - (fingerprint_version_hash = "fingerprint_version_hash", FieldDefault::NoneValue, true, true), - (previous_version_hash = "previous_version_hash", FieldDefault::NoneValue, true, true), - (schema_findings = "schema_findings", FieldDefault::EmptyTuple, true, true), - (backfill = "backfill", FieldDefault::Factory(forward_only_backfill), true, true), - (recorded_build_relation_missing = "recorded_build_relation_missing", FieldDefault::Bool(false), true, true), - (changed_functions = "changed_functions", FieldDefault::EmptyTuple, true, true), - ] -} - -planner_record! { - /// Resource selectors left after unit-test selectors moved into a test selection. - UnitTestSelectorSplit, false, [ - (select = "select", FieldDefault::Required, true, true), - (exclude = "exclude", FieldDefault::Required, true, true), - (sql_test_selection = "sql_test_selection", FieldDefault::Required, true, true), - ] -} - -planner_record! { - /// Relation lookups and source columns for building one model plan entry. - ModelPlanContext, false, [ - (model_locations = "model_locations", FieldDefault::Required, true, true), - (models_by_name = "models_by_name", FieldDefault::Required, true, true), - (seed_locations = "seed_locations", FieldDefault::Required, true, true), - (function_locations = "function_locations", FieldDefault::Required, true, true), - (source_map = "source_map", FieldDefault::Required, true, true), - (source_warehouse_columns = "source_warehouse_columns", FieldDefault::Required, true, true), - (star_exclude_keyword = "star_exclude_keyword", FieldDefault::Required, true, true), - (functions_by_name = "functions_by_name", FieldDefault::EmptyDict, true, true), - (runtime_cursor_producer_names = "runtime_cursor_producer_names", FieldDefault::EmptyFrozenSet, true, true), - (future_cursor_config = "future_cursor_config", FieldDefault::NoneValue, true, true), - (start_cursor_config = "start_cursor_config", FieldDefault::NoneValue, true, true), - (invocation_time = "invocation_time", FieldDefault::NoneValue, true, true), - ] -} - -planner_record! { - /// Resolved per-model cursor start/end overrides for plan entry construction. - CursorOverridePair, false, [ - (start_cursor_override = "start_cursor_override", FieldDefault::NoneValue, true, true), - (end_cursor_override = "end_cursor_override", FieldDefault::NoneValue, true, true), - ] -} - -planner_record! { - /// Blocked models and cursor overrides for plan entry building. - PlanEntryBuildInputs, false, [ - (source_freshness_blocked_model_names = "source_freshness_blocked_model_names", FieldDefault::EmptyFrozenSet, true, true), - (external_blocked_model_names = "external_blocked_model_names", FieldDefault::EmptyFrozenSet, true, true), - (start_cursor_override = "start_cursor_override", FieldDefault::NoneValue, true, true), - (end_cursor_override = "end_cursor_override", FieldDefault::NoneValue, true, true), - (future_cursor_config = "future_cursor_config", FieldDefault::NoneValue, true, true), - (start_cursor_config = "start_cursor_config", FieldDefault::NoneValue, true, true), - (invocation_time = "invocation_time", FieldDefault::NoneValue, true, true), - (max_microbatches = "max_microbatches", FieldDefault::NoneValue, true, true), - (max_microbatches_is_override = "max_microbatches_is_override", FieldDefault::Bool(false), true, true), - (microbatch_limit_action = "microbatch_limit_action", FieldDefault::Member(("sqlbuild._native", "MicrobatchLimitAction"), "ERROR"), true, true), - ] -} - -planner_record! { - /// Per-function output from change detection. - FunctionChangeResult, false, [ - (fingerprint_sql = "fingerprint_sql", FieldDefault::Required, true, true), - (reason = "reason", FieldDefault::Member(("sqlbuild._native", "PlanReason"), "NO_CHANGE"), true, true), - ] -} - -planner_record! { - /// Change detection output for selected planner resources. - PlannerChangeResults, false, [ - (models = "models", FieldDefault::Required, true, true), - (functions = "functions", FieldDefault::Required, true, true), - ] -} - -planner_record! { - /// Effective model change and backfill decided from the model's own change. - ResolvedModelAction, false, [ - (change = "change", FieldDefault::Required, true, true), - (backfill = "backfill", FieldDefault::Required, true, true), - ] -} - -planner_record! { - /// Per-model planning decisions keyed by model name. - PlannerResolvedActions, false, [ - (models = "models", FieldDefault::Required, true, true), - ] -} - -planner_record! { - /// Current direct model version identity values by model name. - DirectModelVersionIdentities, false, [ - (function_local_hashes = "function_local_hashes", FieldDefault::Required, true, true), - (seed_version_hashes = "seed_version_hashes", FieldDefault::Required, true, true), - (seed_metadata_jsons = "seed_metadata_jsons", FieldDefault::Required, true, true), - (model_metadata_jsons = "model_metadata_jsons", FieldDefault::Required, true, true), - (model_local_hashes = "model_local_hashes", FieldDefault::Required, true, true), - (model_version_hashes = "model_version_hashes", FieldDefault::Required, true, true), - ] -} - -planner_record! { - /// Model plan-entry phase output. - PlannerModelEntryResults, false, [ - (entries = "entries", FieldDefault::Required, true, true), - (warnings = "warnings", FieldDefault::EmptyTuple, true, true), - ] -} - -planner_record! { - /// One concrete schema change action to apply to a target table. - SchemaAction, false, [ - (kind = "kind", FieldDefault::Required, true, true), - (column_name = "column_name", FieldDefault::Required, true, true), - (column_type = "column_type", FieldDefault::NoneValue, true, true), - ] -} - -planner_record! { - /// One warning or error produced during plan resolution. - PlanWarning, false, [ - (model_name = "model_name", FieldDefault::Required, true, true), - (severity = "severity", FieldDefault::Required, true, true), - (message = "message", FieldDefault::Required, true, true), - (code = "code", FieldDefault::NoneValue, true, true), - ] -} - -planner_record! { - /// A removed column that an added column may have been renamed from. - ColumnRenameHint, false, [ - (model_name = "model_name", FieldDefault::Required, true, true), - (added_column = "added_column", FieldDefault::Required, true, true), - (candidate_columns = "candidate_columns", FieldDefault::Required, true, true), - (identical = "identical", FieldDefault::Bool(false), true, true), - ], - methods { - /// The one-line hint shown beside the added column. - #[getter] - fn message(&self, py: Python<'_>) -> PyResult { - rename_hint_message( - self.identical.bind(py), - self.candidate_columns.bind(py), - self.added_column.bind(py), - ) - } - } -} - -planner_record! { - /// One named top-level output column of a query and its canonical expression. - QueryProjection, false, [ - (name = "name", FieldDefault::Required, true, true), - (expression = "expression", FieldDefault::Required, true, true), - (references = "references", FieldDefault::Required, true, true), - (passthrough = "passthrough", FieldDefault::Bool(false), true, true), - ], - methods { - /// The case-insensitive output name. - #[getter] - fn key<'py>(&self, py: Python<'py>) -> PyResult> { - self.name.bind(py).call_method0("lower") - } - } -} - -planner_record! { - /// Canonical top-level projections and the rest of one query, ignoring formatting. - QueryShape, false, [ - (projections = "projections", FieldDefault::Required, true, true), - (body = "body", FieldDefault::Required, true, true), - (alias_clauses = "alias_clauses", FieldDefault::Text("{}"), true, true), - (positional = "positional", FieldDefault::Bool(false), true, true), - (input_relations = "input_relations", FieldDefault::NoneValue, true, true), - (local_input_columns = "local_input_columns", FieldDefault::EmptyFrozenSet, true, true), - (clause_references = "clause_references", FieldDefault::EmptyFrozenSet, true, true), - ] -} - -planner_record! { - /// Optional supplemental seed fingerprints and precomputed SQL tests for plan assembly. - PlanOutputExtras, false, [ - (seed_version_hashes = "seed_version_hashes", FieldDefault::NoneValue, true, true), - (seed_metadata_jsons = "seed_metadata_jsons", FieldDefault::NoneValue, true, true), - (seed_plan_reasons = "seed_plan_reasons", FieldDefault::NoneValue, true, true), - (planned_sql_tests = "planned_sql_tests", FieldDefault::NoneValue, true, true), - ] -} - -planner_record! { - /// SQL test entries planned for one selection; reusable only for the same selection. - PlannedSqlTests, false, [ - (selected_keys = "selected_keys", FieldDefault::Required, true, true), - (sql_test_selection = "sql_test_selection", FieldDefault::Class(("sqlbuild._native", "SqlTestSelection")), true, true), - (entries = "entries", FieldDefault::EmptyTuple, true, true), - (warnings = "warnings", FieldDefault::EmptyTuple, true, true), - ] -} - -planner_record! { - /// Native chain and assertion steps for one SQL test, with SQL when rendered. - NativeSqlTestPlan, false, [ - (chain = "chain", FieldDefault::Required, true, true), - (assertions = "assertions", FieldDefault::Required, true, true), - (model_names = "model_names", FieldDefault::Required, true, true), - (warnings = "warnings", FieldDefault::Required, true, true), - (error_messages = "error_messages", FieldDefault::Required, true, true), - (sql = "sql", FieldDefault::NoneValue, true, true), - ] -} - -planner_record! { - /// One SQL test's plan entry and warnings, or the fixture diagnostics preventing it. - SqlTestPlanResult, false, [ - (entry = "entry", FieldDefault::Required, true, true), - (warnings = "warnings", FieldDefault::EmptyTuple, true, true), - (fixture_diagnostics = "fixture_diagnostics", FieldDefault::EmptyTuple, true, true), - ] -} - -planner_record! { - /// Inferred graph slice and fixture boundaries for one SQL scenario. - ScenarioGraphPlan, false, [ - (key = "key", FieldDefault::Required, true, true), - (name = "name", FieldDefault::Required, true, true), - (target_model_names = "target_model_names", FieldDefault::EmptyTuple, true, true), - (assertion_target_model_names = "assertion_target_model_names", FieldDefault::EmptyTuple, true, true), - (model_names = "model_names", FieldDefault::EmptyTuple, true, true), - (source_fixture_names = "source_fixture_names", FieldDefault::EmptyTuple, true, true), - (ref_fixture_names = "ref_fixture_names", FieldDefault::EmptyTuple, true, true), - (seed_names = "seed_names", FieldDefault::EmptyTuple, true, true), - (seed_fixture_names = "seed_fixture_names", FieldDefault::EmptyTuple, true, true), - (dbt_ref_fixture_names = "dbt_ref_fixture_names", FieldDefault::EmptyTuple, true, true), - (function_deps = "function_deps", FieldDefault::EmptyTuple, true, true), - ] -} - -planner_record! { - /// Logical identity for one scenario-owned physical artifact. - ScenarioArtifactIdentity, false, [ - (kind = "kind", FieldDefault::Required, true, true), - (logical_name = "logical_name", FieldDefault::Required, true, true), - ], - convert = scenario_artifact_kind -} - -planner_record! { - /// Resolved physical relation name for one scenario artifact. - ScenarioArtifactName, false, [ - (identity = "identity", FieldDefault::Required, true, true), - (physical_name = "physical_name", FieldDefault::Required, true, true), - ] -} - -planner_record! { - /// Provider usage metadata scoped to planned work. - PlanProviderUsage, false, [ - (provider_name = "provider_name", FieldDefault::Required, true, true), - (consumer_kind = "consumer_kind", FieldDefault::Required, true, true), - (consumer_name = "consumer_name", FieldDefault::Required, true, true), - (parameter_name = "parameter_name", FieldDefault::Required, true, true), - (annotation_class_name = "annotation_class_name", FieldDefault::NoneValue, true, true), - (annotation_module = "annotation_module", FieldDefault::NoneValue, true, true), - ] -} - -planner_record! { - /// Resolved execution environment shared by planner phases. - PlannerRuntime, false, [ - (project = "project", FieldDefault::Required, true, true), - (adapter = "adapter", FieldDefault::Required, true, true), - (connection = "connection", FieldDefault::Required, true, true), - (project_config = "project_config", FieldDefault::NoneValue, true, true), - (local_config = "local_config", FieldDefault::NoneValue, true, true), - (on_progress = "on_progress", FieldDefault::NoneValue, true, true), - (invocation_time = "invocation_time", FieldDefault::Factory(utc_now), true, true), - ] -} - -planner_record! { - /// Resolved planner scopes for one plan build. - PlannerScopeResolution, false, [ - (selected_scope = "selected_scope", FieldDefault::Required, true, true), - (stale_warning_scope = "stale_warning_scope", FieldDefault::Required, true, true), - (inspection_scope = "inspection_scope", FieldDefault::Required, true, true), - ] -} - -planner_record! { - /// Warehouse snapshot and inspection relations gathered once per plan. - PlannerWarehouseState, false, [ - (snapshot = "snapshot", FieldDefault::Required, true, true), - (inspection_relations = "inspection_relations", FieldDefault::Required, true, true), - (migration_entries = "migration_entries", FieldDefault::EmptyTuple, true, true), - (migration_warnings = "migration_warnings", FieldDefault::EmptyTuple, true, true), - (column_migration_entries = "column_migration_entries", FieldDefault::EmptyTuple, true, true), - (column_rename_hints = "column_rename_hints", FieldDefault::EmptyTuple, true, true), - (old_name_view_entries = "old_name_view_entries", FieldDefault::EmptyTuple, true, true), - (old_name_views = "old_name_views", FieldDefault::EmptyTuple, true, true), - (migration_fingerprints = "migration_fingerprints", FieldDefault::Class(("sqlbuild._native", "MigrationFingerprintCache")), true, true), - ] -} - -planner_record! { - /// One destination model whose data should come from an earlier relation. - ModelMigrationRequest, false, [ - (model = "model", FieldDefault::Required, true, true), - (discovery = "discovery", FieldDefault::Required, true, true), - (raw_origin = "raw_origin", FieldDefault::NoneValue, true, true), - (origin_location = "origin_location", FieldDefault::NoneValue, true, true), - (origin_model = "origin_model", FieldDefault::NoneValue, true, true), - (force = "force", FieldDefault::Bool(false), true, true), - (identity_only = "identity_only", FieldDefault::Bool(false), true, true), - ] -} - -planner_record! { - /// One project model's explicit migrate_from, resolved to a relation. - ModelMigrationDeclaration, false, [ - (model_name = "model_name", FieldDefault::Required, true, true), - (origin_location = "origin_location", FieldDefault::Required, true, true), - (origin_model = "origin_model", FieldDefault::Required, true, true), - ] -} - -planner_record! { - /// Manual and automatically discovered migration requests for one plan. - ModelMigrationDiscovery, false, [ - (requests = "requests", FieldDefault::EmptyTuple, true, true), - (warnings = "warnings", FieldDefault::EmptyTuple, true, true), - (destination_fingerprints = "destination_fingerprints", FieldDefault::EmptyDict, true, true), - ] -} - -planner_record! { - /// Compatibility status plus human-readable blocking findings. - MigrationCompatibilityResult, false, [ - (status = "status", FieldDefault::Required, true, true), - (findings = "findings", FieldDefault::EmptyTuple, true, true), - ] -} - -planner_record! { - /// Migration decisions plus the warehouse snapshot as it will look after moves. - ModelMigrationPlanning, false, [ - (snapshot = "snapshot", FieldDefault::Required, true, true), - (entries = "entries", FieldDefault::EmptyTuple, true, true), - (warnings = "warnings", FieldDefault::EmptyTuple, true, true), - ] -} - -planner_record! { - /// Old-name steps for this build plus every compatibility view still recorded. - OldNameViewPlanning, false, [ - (entries = "entries", FieldDefault::EmptyTuple, true, true), - (views = "views", FieldDefault::EmptyTuple, true, true), - (warnings = "warnings", FieldDefault::EmptyTuple, true, true), - ] -} - -planner_record! { - /// Column rename decisions plus the warehouse snapshot as it will look after renames. - ColumnMigrationPlanning, false, [ - (snapshot = "snapshot", FieldDefault::Required, true, true), - (entries = "entries", FieldDefault::EmptyTuple, true, true), - (hints = "hints", FieldDefault::EmptyTuple, true, true), - (warnings = "warnings", FieldDefault::EmptyTuple, true, true), - ] -} - -planner_record! { - /// Expected version identities for inspection and stale-warning scopes. - PlannerIdentityContext, false, [ - (version_identities = "version_identities", FieldDefault::Required, true, true), - (stale_warning_identities = "stale_warning_identities", FieldDefault::Required, true, true), - ] -} - -planner_record! { - /// Scope pruning outcome with pruned scopes, actions, and staleness state. - PlannerScopePruningResult, false, [ - (inspection_scope = "inspection_scope", FieldDefault::Required, true, true), - (execution_scope = "execution_scope", FieldDefault::Required, true, true), - (resolved_actions = "resolved_actions", FieldDefault::Required, true, true), - (pruned_direct_model_names = "pruned_direct_model_names", FieldDefault::Required, true, true), - (direct_identity_stale_model_names = "direct_identity_stale_model_names", FieldDefault::Required, true, true), - ] -} - -planner_record! { - /// Write-hash-honest changes merged back into resolved actions. - PlannerChangeReconciliation, false, [ - (changes = "changes", FieldDefault::Required, true, true), - (resolved_actions = "resolved_actions", FieldDefault::Required, true, true), - ] -} - -planner_record! { - /// Model plan entries for one plan build. - PlannerEntryResults, false, [ - (model_entry_results = "model_entry_results", FieldDefault::Required, true, true), - ] -} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_models/registration.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_models/registration.rs deleted file mode 100644 index 242379387b..0000000000 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_models/registration.rs +++ /dev/null @@ -1,83 +0,0 @@ -//! Registers the planner value records. - -use crate::bindings::_helpers::planner_models::enums::register_enums; -use crate::bindings::_helpers::planner_models::fixture_inferences::FixtureColumnInferences; -use crate::bindings::_helpers::planner_models::records::{ - BackfillResult, ChangeDetectionResult, ColumnMigrationPlanning, ColumnRenameHint, - CursorOverridePair, CursorSnapshotScope, DirectModelVersionIdentities, FixtureColumnMetadata, - FixtureRelationMetadata, FunctionChangeResult, MigrationCompatibilityResult, MissingUpstream, - ModelMigrationDeclaration, ModelMigrationDiscovery, ModelMigrationPlanning, - ModelMigrationRequest, ModelPlanContext, NativeSqlTestArtifact, NativeSqlTestPlan, - OldNameViewPlanning, ParsedScenarioArtifactName, PlanEntryBuildInputs, PlanOutputExtras, - PlanProviderUsage, PlanWarning, PlannedSqlTests, PlannerChangeReconciliation, - PlannerChangeResults, PlannerEntryResults, PlannerIdentityContext, PlannerModelEntryResults, - PlannerResolvedActions, PlannerRuntime, PlannerScopePruningResult, PlannerScopeResolution, - PlannerWarehouseState, QueryProjection, QueryShape, ReferenceRename, RelationFixtureCompletion, - RelationFixtureDiagnostic, RelationFixturePlanningContext, ResolvedModelAction, - ScenarioArtifactIdentity, ScenarioArtifactName, ScenarioGraphPlan, SchemaAction, - SelectionStalenessGraph, SelectionStalenessNodeKey, SelectionStalenessWarning, - SelectorExpansion, SqlTestPlanResult, UnitTestSelectorSplit, WarehouseFingerprints, - WarehouseSnapshot, -}; -use pyo3::prelude::{Bound, PyModule, PyModuleMethods, PyResult}; - -pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { - register_enums(module)?; - module.add_class::()?; - NativeSqlTestArtifact::register_class(module)?; - RelationFixtureDiagnostic::register_class(module)?; - RelationFixtureCompletion::register_class(module)?; - FixtureColumnMetadata::register_class(module)?; - FixtureRelationMetadata::register_class(module)?; - RelationFixturePlanningContext::register_class(module)?; - SelectorExpansion::register_class(module)?; - ParsedScenarioArtifactName::register_class(module)?; - MissingUpstream::register_class(module)?; - WarehouseFingerprints::register_class(module)?; - CursorSnapshotScope::register_class(module)?; - ReferenceRename::register_class(module)?; - WarehouseSnapshot::register_class(module)?; - SelectionStalenessNodeKey::register_class(module)?; - SelectionStalenessGraph::register_class(module)?; - SelectionStalenessWarning::register_class(module)?; - BackfillResult::register_class(module)?; - ChangeDetectionResult::register_class(module)?; - UnitTestSelectorSplit::register_class(module)?; - ModelPlanContext::register_class(module)?; - CursorOverridePair::register_class(module)?; - PlanEntryBuildInputs::register_class(module)?; - FunctionChangeResult::register_class(module)?; - PlannerChangeResults::register_class(module)?; - ResolvedModelAction::register_class(module)?; - PlannerResolvedActions::register_class(module)?; - DirectModelVersionIdentities::register_class(module)?; - PlannerModelEntryResults::register_class(module)?; - SchemaAction::register_class(module)?; - PlanWarning::register_class(module)?; - ColumnRenameHint::register_class(module)?; - QueryProjection::register_class(module)?; - QueryShape::register_class(module)?; - PlanOutputExtras::register_class(module)?; - PlannedSqlTests::register_class(module)?; - NativeSqlTestPlan::register_class(module)?; - SqlTestPlanResult::register_class(module)?; - ScenarioGraphPlan::register_class(module)?; - ScenarioArtifactIdentity::register_class(module)?; - ScenarioArtifactName::register_class(module)?; - PlanProviderUsage::register_class(module)?; - PlannerRuntime::register_class(module)?; - PlannerScopeResolution::register_class(module)?; - PlannerWarehouseState::register_class(module)?; - ModelMigrationRequest::register_class(module)?; - ModelMigrationDeclaration::register_class(module)?; - ModelMigrationDiscovery::register_class(module)?; - MigrationCompatibilityResult::register_class(module)?; - ModelMigrationPlanning::register_class(module)?; - OldNameViewPlanning::register_class(module)?; - ColumnMigrationPlanning::register_class(module)?; - PlannerIdentityContext::register_class(module)?; - PlannerScopePruningResult::register_class(module)?; - PlannerChangeReconciliation::register_class(module)?; - PlannerEntryResults::register_class(module)?; - Ok(()) -} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_records/constants.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_records/constants.rs index a9e4a328d3..9d4703965a 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_records/constants.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_records/constants.rs @@ -1,7 +1,8 @@ //! Layouts of the planner, migration, cursor and fingerprint value records hosted natively. +use crate::bindings::_helpers::build_command::support::utc_now; use crate::bindings::_helpers::record_behaviours::planner::forward_only_backfill; -use crate::bindings::_helpers::records::models::{FieldDefault, FieldSpec, RecordSpec}; +use crate::bindings::_helpers::records::models::{EnumSpec, FieldDefault, FieldSpec, RecordSpec}; const R_PARSED_SELECTOR: RecordSpec = RecordSpec { name: "ParsedSelector", @@ -187,6 +188,7 @@ const R_TIMESTAMP_VALUE: RecordSpec = RecordSpec { name: "source_text", default: FieldDefault::None, compared: false, + shown: false, }, ], }; @@ -327,27 +329,32 @@ const R_MODEL_CURSOR_SNAPSHOT: RecordSpec = RecordSpec { name: "physical_target_max", default: FieldDefault::None, compared: false, + shown: false, }, FieldSpec::new("target_eligible_max", FieldDefault::None), FieldSpec { name: "target_relation", default: FieldDefault::None, compared: false, + shown: false, }, FieldSpec { name: "destination_cursor_column", default: FieldDefault::None, compared: false, + shown: false, }, FieldSpec { name: "input_evidence", default: FieldDefault::EmptyTuple, compared: false, + shown: false, }, FieldSpec { name: "expected_watermark_count", default: FieldDefault::Int(0), compared: false, + shown: false, }, FieldSpec::new("unavailable_watermark_tags", FieldDefault::EmptyTuple), FieldSpec::new( @@ -362,11 +369,13 @@ const R_MODEL_CURSOR_SNAPSHOT: RecordSpec = RecordSpec { name: "empty_input_names", default: FieldDefault::EmptyTuple, compared: false, + shown: false, }, FieldSpec { name: "unreadable_input_names", default: FieldDefault::EmptyTuple, compared: false, + shown: false, }, ], }; @@ -981,3 +990,707 @@ pub(crate) const PLANNER_RECORDS: &[RecordSpec] = &[ R_SCENARIO_EXECUTION_PLAN, R_PLAN_OUTPUT, ]; + +const R_NATIVE_SQL_TEST_ARTIFACT: RecordSpec = RecordSpec { + name: "NativeSqlTestArtifact", + doc: "One native-planned SQL-test artifact and its distinct planning error messages.", + fields: &[ + FieldSpec::required("sql"), + FieldSpec::required("model_names"), + FieldSpec::new("error_messages", FieldDefault::EmptyTuple), + ], +}; +const R_RELATION_FIXTURE_DIAGNOSTIC: RecordSpec = RecordSpec { + name: "RelationFixtureDiagnostic", + doc: "One statically provable invalid fixture condition.", + fields: &[FieldSpec::required("key"), FieldSpec::required("message")], +}; +const R_RELATION_FIXTURE_COMPLETION: RecordSpec = RecordSpec { + name: "RelationFixtureCompletion", + doc: "Completed fixture SQL plus metadata reused by SQL-test validation.", + fields: &[ + FieldSpec::required("fixture_sql_by_key"), + FieldSpec::required("inferred_by_fixture"), + FieldSpec::required("expected_types"), + FieldSpec::required("diagnostics"), + ], +}; +const R_FIXTURE_COLUMN_METADATA: RecordSpec = RecordSpec { + name: "FixtureColumnMetadata", + doc: "Known type and nullability for one relation fixture column.", + fields: &[ + FieldSpec::required("name"), + FieldSpec::required("type"), + FieldSpec::required("nullable"), + ], +}; +const R_FIXTURE_RELATION_METADATA: RecordSpec = RecordSpec { + name: "FixtureRelationMetadata", + doc: "Known columns and whether they form an authoritative relation shape.", + fields: &[ + FieldSpec::required("columns"), + FieldSpec::required("authoritative_names"), + ], +}; +const R_RELATION_FIXTURE_PLANNING_CONTEXT: RecordSpec = RecordSpec { + name: "RelationFixturePlanningContext", + doc: "Project-wide fixture metadata shared across test and scenario planning.", + fields: &[ + FieldSpec::required("models_by_name"), + FieldSpec::required("relations"), + FieldSpec::required("authoritative_columns"), + FieldSpec::required("expected_types"), + FieldSpec::uncompared_shown( + "fixture_inferences", + FieldDefault::Call("sqlbuild._native", "FixtureColumnInferences"), + ), + ], +}; +const R_SELECTOR_EXPANSION: RecordSpec = RecordSpec { + name: "SelectorExpansion", + doc: "One selector split into graph expansion flags and core text.", + fields: &[ + FieldSpec::required("core"), + FieldSpec::new("upstream", FieldDefault::False), + FieldSpec::new("downstream", FieldDefault::False), + ], +}; +const R_PARSED_SCENARIO_ARTIFACT_NAME: RecordSpec = RecordSpec { + name: "ParsedScenarioArtifactName", + doc: "Parsed physical name for one planner-owned scenario artifact.", + fields: &[ + FieldSpec::required("hash_prefix"), + FieldSpec::required("kind"), + FieldSpec::required("logical_name"), + ], +}; +const R_MISSING_UPSTREAM: RecordSpec = RecordSpec { + name: "MissingUpstream", + doc: "One upstream dependency missing from both scope and warehouse.", + fields: &[ + FieldSpec::required("key"), + FieldSpec::required("required_by"), + ], +}; +const R_WAREHOUSE_FINGERPRINTS: RecordSpec = RecordSpec { + name: "WarehouseFingerprints", + doc: "Latest direct fingerprints grouped by node type.", + fields: &[ + FieldSpec::new("models", FieldDefault::EmptyDict), + FieldSpec::new("functions", FieldDefault::EmptyDict), + FieldSpec::new("seeds", FieldDefault::EmptyDict), + FieldSpec::new("python_nodes", FieldDefault::EmptyDict), + FieldSpec::uncompared("unfiltered_schemas", FieldDefault::EmptyDict), + FieldSpec::uncompared("unfiltered_database", FieldDefault::None), + ], +}; +const R_CURSOR_SNAPSHOT_SCOPE: RecordSpec = RecordSpec { + name: "CursorSnapshotScope", + doc: "Execution selection used for cursor validation and runtime producer resolution.", + fields: &[ + FieldSpec::required("model_keys"), + FieldSpec::required("runtime_producer_keys"), + FieldSpec::new("invocation_time", FieldDefault::None), + FieldSpec::new("start_cursor_config", FieldDefault::None), + FieldSpec::new("cursor_overrides", FieldDefault::None), + ], +}; +const R_REFERENCE_RENAME: RecordSpec = RecordSpec { + name: "ReferenceRename", + doc: "One model rename that downstream references may have been rewritten for.", + fields: &[ + FieldSpec::required("new_name"), + FieldSpec::required("origin_name"), + FieldSpec::new("recorded_at", FieldDefault::None), + ], +}; +const R_WAREHOUSE_SNAPSHOT: RecordSpec = RecordSpec { + name: "WarehouseSnapshot", + doc: "Frozen point-in-time picture of warehouse state for planning.", + fields: &[ + FieldSpec::new("existing_relations", FieldDefault::EmptyDict), + FieldSpec::new("existing_columns", FieldDefault::EmptyDict), + FieldSpec::new( + "fingerprints", + FieldDefault::Call("sqlbuild._native", "WarehouseFingerprints"), + ), + FieldSpec::new("cursor_snapshots", FieldDefault::EmptyDict), + FieldSpec::new( + "source_freshness_state_schemas", + FieldDefault::EmptyFrozenset, + ), + FieldSpec::new("column_dialect", FieldDefault::None), + FieldSpec::new("renamed_models", FieldDefault::EmptyFrozenset), + FieldSpec::new("reference_renames", FieldDefault::EmptyTuple), + FieldSpec::new("migration_state_schemas", FieldDefault::EmptyFrozenset), + FieldSpec::new("old_name_view_state_schemas", FieldDefault::EmptyFrozenset), + FieldSpec::new("listed_state_schemas", FieldDefault::EmptyFrozenset), + ], +}; +const R_SELECTION_STALENESS_NODE_KEY: RecordSpec = RecordSpec { + name: "SelectionStalenessNodeKey", + doc: "Neutral node identity for shared selection-aware staleness classification.", + fields: &[ + FieldSpec::required("resource_type"), + FieldSpec::required("name"), + ], +}; +const R_SELECTION_STALENESS_GRAPH: RecordSpec = RecordSpec { + name: "SelectionStalenessGraph", + doc: "Neutral graph inputs for selected-model stale warning classification.", + fields: &[ + FieldSpec::required("upstream_deps"), + FieldSpec::required("selected_model_names"), + FieldSpec::required("run_model_names"), + FieldSpec::required("run_seed_names"), + FieldSpec::required("run_source_names"), + FieldSpec::required("changed_model_names"), + FieldSpec::required("changed_seed_names"), + FieldSpec::required("changed_source_names"), + ], +}; +const R_SELECTION_STALENESS_WARNING: RecordSpec = RecordSpec { + name: "SelectionStalenessWarning", + doc: "Neutral stale warning classification for one selected model.", + fields: &[ + FieldSpec::required("model_name"), + FieldSpec::required("trigger_names"), + ], +}; +const R_BACKFILL_RESULT: RecordSpec = RecordSpec { + name: "BackfillResult", + doc: "Resolved backfill action from a change detection policy.", + fields: &[ + FieldSpec::required("action"), + FieldSpec::new("duration", FieldDefault::None), + ], +}; +const R_CHANGE_DETECTION_RESULT: RecordSpec = RecordSpec { + name: "ChangeDetectionResult", + doc: "Per-model output from change detection and policy resolution.", + fields: &[ + FieldSpec::required("model_name"), + FieldSpec::required("change_kind"), + FieldSpec::new("query_changed", FieldDefault::False), + FieldSpec::new("config_changed", FieldDefault::False), + FieldSpec::new("fingerprint_metadata_json", FieldDefault::None), + FieldSpec::new("previous_metadata_json", FieldDefault::None), + FieldSpec::new("fingerprint_version_hash", FieldDefault::None), + FieldSpec::new("previous_version_hash", FieldDefault::None), + FieldSpec::new("schema_findings", FieldDefault::EmptyTuple), + FieldSpec::new("backfill", FieldDefault::Factory(forward_only_backfill)), + FieldSpec::new("recorded_build_relation_missing", FieldDefault::False), + FieldSpec::new("changed_functions", FieldDefault::EmptyTuple), + ], +}; +const R_UNIT_TEST_SELECTOR_SPLIT: RecordSpec = RecordSpec { + name: "UnitTestSelectorSplit", + doc: "Resource selectors left after unit-test selectors moved into a test selection.", + fields: &[ + FieldSpec::required("select"), + FieldSpec::required("exclude"), + FieldSpec::required("sql_test_selection"), + ], +}; +const R_MODEL_PLAN_CONTEXT: RecordSpec = RecordSpec { + name: "ModelPlanContext", + doc: "Relation lookups and source columns for building one model plan entry.", + fields: &[ + FieldSpec::required("model_locations"), + FieldSpec::required("models_by_name"), + FieldSpec::required("seed_locations"), + FieldSpec::required("function_locations"), + FieldSpec::required("source_map"), + FieldSpec::required("source_warehouse_columns"), + FieldSpec::required("star_exclude_keyword"), + FieldSpec::new("functions_by_name", FieldDefault::EmptyDict), + FieldSpec::new( + "runtime_cursor_producer_names", + FieldDefault::EmptyFrozenset, + ), + FieldSpec::new("future_cursor_config", FieldDefault::None), + FieldSpec::new("start_cursor_config", FieldDefault::None), + FieldSpec::new("invocation_time", FieldDefault::None), + ], +}; +const R_CURSOR_OVERRIDE_PAIR: RecordSpec = RecordSpec { + name: "CursorOverridePair", + doc: "Resolved per-model cursor start/end overrides for plan entry construction.", + fields: &[ + FieldSpec::new("start_cursor_override", FieldDefault::None), + FieldSpec::new("end_cursor_override", FieldDefault::None), + ], +}; +const R_PLAN_ENTRY_BUILD_INPUTS: RecordSpec = RecordSpec { + name: "PlanEntryBuildInputs", + doc: "Blocked models and cursor overrides for plan entry building.", + fields: &[ + FieldSpec::new( + "source_freshness_blocked_model_names", + FieldDefault::EmptyFrozenset, + ), + FieldSpec::new("external_blocked_model_names", FieldDefault::EmptyFrozenset), + FieldSpec::new("start_cursor_override", FieldDefault::None), + FieldSpec::new("end_cursor_override", FieldDefault::None), + FieldSpec::new("future_cursor_config", FieldDefault::None), + FieldSpec::new("start_cursor_config", FieldDefault::None), + FieldSpec::new("invocation_time", FieldDefault::None), + FieldSpec::new("max_microbatches", FieldDefault::None), + FieldSpec::new("max_microbatches_is_override", FieldDefault::False), + FieldSpec::new( + "microbatch_limit_action", + FieldDefault::Member("sqlbuild._native", "MicrobatchLimitAction", "ERROR"), + ), + ], +}; +const R_FUNCTION_CHANGE_RESULT: RecordSpec = RecordSpec { + name: "FunctionChangeResult", + doc: "Per-function output from change detection.", + fields: &[ + FieldSpec::required("fingerprint_sql"), + FieldSpec::new( + "reason", + FieldDefault::Member("sqlbuild._native", "PlanReason", "NO_CHANGE"), + ), + ], +}; +const R_PLANNER_CHANGE_RESULTS: RecordSpec = RecordSpec { + name: "PlannerChangeResults", + doc: "Change detection output for selected planner resources.", + fields: &[ + FieldSpec::required("models"), + FieldSpec::required("functions"), + ], +}; +const R_RESOLVED_MODEL_ACTION: RecordSpec = RecordSpec { + name: "ResolvedModelAction", + doc: "Effective model change and backfill decided from the model's own change.", + fields: &[ + FieldSpec::required("change"), + FieldSpec::required("backfill"), + ], +}; +const R_PLANNER_RESOLVED_ACTIONS: RecordSpec = RecordSpec { + name: "PlannerResolvedActions", + doc: "Per-model planning decisions keyed by model name.", + fields: &[FieldSpec::required("models")], +}; +const R_DIRECT_MODEL_VERSION_IDENTITIES: RecordSpec = RecordSpec { + name: "DirectModelVersionIdentities", + doc: "Current direct model version identity values by model name.", + fields: &[ + FieldSpec::required("function_local_hashes"), + FieldSpec::required("seed_version_hashes"), + FieldSpec::required("seed_metadata_jsons"), + FieldSpec::required("model_metadata_jsons"), + FieldSpec::required("model_local_hashes"), + FieldSpec::required("model_version_hashes"), + ], +}; +const R_PLANNER_MODEL_ENTRY_RESULTS: RecordSpec = RecordSpec { + name: "PlannerModelEntryResults", + doc: "Model plan-entry phase output.", + fields: &[ + FieldSpec::required("entries"), + FieldSpec::new("warnings", FieldDefault::EmptyTuple), + ], +}; +const R_SCHEMA_ACTION: RecordSpec = RecordSpec { + name: "SchemaAction", + doc: "One concrete schema change action to apply to a target table.", + fields: &[ + FieldSpec::required("kind"), + FieldSpec::required("column_name"), + FieldSpec::new("column_type", FieldDefault::None), + ], +}; +const R_PLAN_WARNING: RecordSpec = RecordSpec { + name: "PlanWarning", + doc: "One warning or error produced during plan resolution.", + fields: &[ + FieldSpec::required("model_name"), + FieldSpec::required("severity"), + FieldSpec::required("message"), + FieldSpec::new("code", FieldDefault::None), + ], +}; +const R_COLUMN_RENAME_HINT: RecordSpec = RecordSpec { + name: "ColumnRenameHint", + doc: "A removed column that an added column may have been renamed from.", + fields: &[ + FieldSpec::required("model_name"), + FieldSpec::required("added_column"), + FieldSpec::required("candidate_columns"), + FieldSpec::new("identical", FieldDefault::False), + ], +}; +const R_QUERY_PROJECTION: RecordSpec = RecordSpec { + name: "QueryProjection", + doc: "One named top-level output column of a query and its canonical expression.", + fields: &[ + FieldSpec::required("name"), + FieldSpec::required("expression"), + FieldSpec::required("references"), + FieldSpec::new("passthrough", FieldDefault::False), + ], +}; +const R_QUERY_SHAPE: RecordSpec = RecordSpec { + name: "QueryShape", + doc: "Canonical top-level projections and the rest of one query, ignoring formatting.", + fields: &[ + FieldSpec::required("projections"), + FieldSpec::required("body"), + FieldSpec::new("alias_clauses", FieldDefault::Text("{}")), + FieldSpec::new("positional", FieldDefault::False), + FieldSpec::new("input_relations", FieldDefault::None), + FieldSpec::new("local_input_columns", FieldDefault::EmptyFrozenset), + FieldSpec::new("clause_references", FieldDefault::EmptyFrozenset), + ], +}; +const R_PLAN_OUTPUT_EXTRAS: RecordSpec = RecordSpec { + name: "PlanOutputExtras", + doc: "Optional supplemental seed fingerprints and precomputed SQL tests for plan assembly.", + fields: &[ + FieldSpec::new("seed_version_hashes", FieldDefault::None), + FieldSpec::new("seed_metadata_jsons", FieldDefault::None), + FieldSpec::new("seed_plan_reasons", FieldDefault::None), + FieldSpec::new("planned_sql_tests", FieldDefault::None), + ], +}; +const R_PLANNED_SQL_TESTS: RecordSpec = RecordSpec { + name: "PlannedSqlTests", + doc: "SQL test entries planned for one selection; reusable only for the same selection.", + fields: &[ + FieldSpec::required("selected_keys"), + FieldSpec::new( + "sql_test_selection", + FieldDefault::Call("sqlbuild._native", "SqlTestSelection"), + ), + FieldSpec::new("entries", FieldDefault::EmptyTuple), + FieldSpec::new("warnings", FieldDefault::EmptyTuple), + ], +}; +const R_NATIVE_SQL_TEST_PLAN: RecordSpec = RecordSpec { + name: "NativeSqlTestPlan", + doc: "Native chain and assertion steps for one SQL test, with SQL when rendered.", + fields: &[ + FieldSpec::required("chain"), + FieldSpec::required("assertions"), + FieldSpec::required("model_names"), + FieldSpec::required("warnings"), + FieldSpec::required("error_messages"), + FieldSpec::new("sql", FieldDefault::None), + ], +}; +const R_SQL_TEST_PLAN_RESULT: RecordSpec = RecordSpec { + name: "SqlTestPlanResult", + doc: "One SQL test's plan entry and warnings, or the fixture diagnostics preventing it.", + fields: &[ + FieldSpec::required("entry"), + FieldSpec::new("warnings", FieldDefault::EmptyTuple), + FieldSpec::new("fixture_diagnostics", FieldDefault::EmptyTuple), + ], +}; +const R_SCENARIO_GRAPH_PLAN: RecordSpec = RecordSpec { + name: "ScenarioGraphPlan", + doc: "Inferred graph slice and fixture boundaries for one SQL scenario.", + fields: &[ + FieldSpec::required("key"), + FieldSpec::required("name"), + FieldSpec::new("target_model_names", FieldDefault::EmptyTuple), + FieldSpec::new("assertion_target_model_names", FieldDefault::EmptyTuple), + FieldSpec::new("model_names", FieldDefault::EmptyTuple), + FieldSpec::new("source_fixture_names", FieldDefault::EmptyTuple), + FieldSpec::new("ref_fixture_names", FieldDefault::EmptyTuple), + FieldSpec::new("seed_names", FieldDefault::EmptyTuple), + FieldSpec::new("seed_fixture_names", FieldDefault::EmptyTuple), + FieldSpec::new("dbt_ref_fixture_names", FieldDefault::EmptyTuple), + FieldSpec::new("function_deps", FieldDefault::EmptyTuple), + ], +}; +const R_SCENARIO_ARTIFACT_IDENTITY: RecordSpec = RecordSpec { + name: "ScenarioArtifactIdentity", + doc: "Logical identity for one scenario-owned physical artifact.", + fields: &[ + FieldSpec::required("kind"), + FieldSpec::required("logical_name"), + ], +}; +const R_SCENARIO_ARTIFACT_NAME: RecordSpec = RecordSpec { + name: "ScenarioArtifactName", + doc: "Resolved physical relation name for one scenario artifact.", + fields: &[ + FieldSpec::required("identity"), + FieldSpec::required("physical_name"), + ], +}; +const R_PLAN_PROVIDER_USAGE: RecordSpec = RecordSpec { + name: "PlanProviderUsage", + doc: "Provider usage metadata scoped to planned work.", + fields: &[ + FieldSpec::required("provider_name"), + FieldSpec::required("consumer_kind"), + FieldSpec::required("consumer_name"), + FieldSpec::required("parameter_name"), + FieldSpec::new("annotation_class_name", FieldDefault::None), + FieldSpec::new("annotation_module", FieldDefault::None), + ], +}; +const R_PLANNER_RUNTIME: RecordSpec = RecordSpec { + name: "PlannerRuntime", + doc: "Resolved execution environment shared by planner phases.", + fields: &[ + FieldSpec::required("project"), + FieldSpec::required("adapter"), + FieldSpec::required("connection"), + FieldSpec::new("project_config", FieldDefault::None), + FieldSpec::new("local_config", FieldDefault::None), + FieldSpec::new("on_progress", FieldDefault::None), + FieldSpec::new("invocation_time", FieldDefault::Factory(utc_now)), + ], +}; +const R_PLANNER_SCOPE_RESOLUTION: RecordSpec = RecordSpec { + name: "PlannerScopeResolution", + doc: "Resolved planner scopes for one plan build.", + fields: &[ + FieldSpec::required("selected_scope"), + FieldSpec::required("stale_warning_scope"), + FieldSpec::required("inspection_scope"), + ], +}; +const R_PLANNER_WAREHOUSE_STATE: RecordSpec = RecordSpec { + name: "PlannerWarehouseState", + doc: "Warehouse snapshot and inspection relations gathered once per plan.", + fields: &[ + FieldSpec::required("snapshot"), + FieldSpec::required("inspection_relations"), + FieldSpec::new("migration_entries", FieldDefault::EmptyTuple), + FieldSpec::new("migration_warnings", FieldDefault::EmptyTuple), + FieldSpec::new("column_migration_entries", FieldDefault::EmptyTuple), + FieldSpec::new("column_rename_hints", FieldDefault::EmptyTuple), + FieldSpec::new("old_name_view_entries", FieldDefault::EmptyTuple), + FieldSpec::new("old_name_views", FieldDefault::EmptyTuple), + FieldSpec::new( + "migration_fingerprints", + FieldDefault::Call("sqlbuild._native", "MigrationFingerprintCache"), + ), + ], +}; +const R_MODEL_MIGRATION_REQUEST: RecordSpec = RecordSpec { + name: "ModelMigrationRequest", + doc: "One destination model whose data should come from an earlier relation.", + fields: &[ + FieldSpec::required("model"), + FieldSpec::required("discovery"), + FieldSpec::new("raw_origin", FieldDefault::None), + FieldSpec::new("origin_location", FieldDefault::None), + FieldSpec::new("origin_model", FieldDefault::None), + FieldSpec::new("force", FieldDefault::False), + FieldSpec::new("identity_only", FieldDefault::False), + ], +}; +const R_MODEL_MIGRATION_DECLARATION: RecordSpec = RecordSpec { + name: "ModelMigrationDeclaration", + doc: "One project model's explicit migrate_from, resolved to a relation.", + fields: &[ + FieldSpec::required("model_name"), + FieldSpec::required("origin_location"), + FieldSpec::required("origin_model"), + ], +}; +const R_MODEL_MIGRATION_DISCOVERY: RecordSpec = RecordSpec { + name: "ModelMigrationDiscovery", + doc: "Manual and automatically discovered migration requests for one plan.", + fields: &[ + FieldSpec::new("requests", FieldDefault::EmptyTuple), + FieldSpec::new("warnings", FieldDefault::EmptyTuple), + FieldSpec::new("destination_fingerprints", FieldDefault::EmptyDict), + ], +}; +const R_MIGRATION_COMPATIBILITY_RESULT: RecordSpec = RecordSpec { + name: "MigrationCompatibilityResult", + doc: "Compatibility status plus human-readable blocking findings.", + fields: &[ + FieldSpec::required("status"), + FieldSpec::new("findings", FieldDefault::EmptyTuple), + ], +}; +const R_MODEL_MIGRATION_PLANNING: RecordSpec = RecordSpec { + name: "ModelMigrationPlanning", + doc: "Migration decisions plus the warehouse snapshot as it will look after moves.", + fields: &[ + FieldSpec::required("snapshot"), + FieldSpec::new("entries", FieldDefault::EmptyTuple), + FieldSpec::new("warnings", FieldDefault::EmptyTuple), + ], +}; +const R_OLD_NAME_VIEW_PLANNING: RecordSpec = RecordSpec { + name: "OldNameViewPlanning", + doc: "Old-name steps for this build plus every compatibility view still recorded.", + fields: &[ + FieldSpec::new("entries", FieldDefault::EmptyTuple), + FieldSpec::new("views", FieldDefault::EmptyTuple), + FieldSpec::new("warnings", FieldDefault::EmptyTuple), + ], +}; +const R_COLUMN_MIGRATION_PLANNING: RecordSpec = RecordSpec { + name: "ColumnMigrationPlanning", + doc: "Column rename decisions plus the warehouse snapshot as it will look after renames.", + fields: &[ + FieldSpec::required("snapshot"), + FieldSpec::new("entries", FieldDefault::EmptyTuple), + FieldSpec::new("hints", FieldDefault::EmptyTuple), + FieldSpec::new("warnings", FieldDefault::EmptyTuple), + ], +}; +const R_PLANNER_IDENTITY_CONTEXT: RecordSpec = RecordSpec { + name: "PlannerIdentityContext", + doc: "Expected version identities for inspection and stale-warning scopes.", + fields: &[ + FieldSpec::required("version_identities"), + FieldSpec::required("stale_warning_identities"), + ], +}; +const R_PLANNER_SCOPE_PRUNING_RESULT: RecordSpec = RecordSpec { + name: "PlannerScopePruningResult", + doc: "Scope pruning outcome with pruned scopes, actions, and staleness state.", + fields: &[ + FieldSpec::required("inspection_scope"), + FieldSpec::required("execution_scope"), + FieldSpec::required("resolved_actions"), + FieldSpec::required("pruned_direct_model_names"), + FieldSpec::required("direct_identity_stale_model_names"), + ], +}; +const R_PLANNER_CHANGE_RECONCILIATION: RecordSpec = RecordSpec { + name: "PlannerChangeReconciliation", + doc: "Write-hash-honest changes merged back into resolved actions.", + fields: &[ + FieldSpec::required("changes"), + FieldSpec::required("resolved_actions"), + ], +}; +const R_PLANNER_ENTRY_RESULTS: RecordSpec = RecordSpec { + name: "PlannerEntryResults", + doc: "Model plan entries for one plan build.", + fields: &[FieldSpec::required("model_entry_results")], +}; + +/// Planner value records, in registration order. +pub(crate) const PLANNER_VALUE_RECORDS: &[RecordSpec] = &[ + R_NATIVE_SQL_TEST_ARTIFACT, + R_RELATION_FIXTURE_DIAGNOSTIC, + R_RELATION_FIXTURE_COMPLETION, + R_FIXTURE_COLUMN_METADATA, + R_FIXTURE_RELATION_METADATA, + R_RELATION_FIXTURE_PLANNING_CONTEXT, + R_SELECTOR_EXPANSION, + R_PARSED_SCENARIO_ARTIFACT_NAME, + R_MISSING_UPSTREAM, + R_WAREHOUSE_FINGERPRINTS, + R_CURSOR_SNAPSHOT_SCOPE, + R_REFERENCE_RENAME, + R_WAREHOUSE_SNAPSHOT, + R_SELECTION_STALENESS_NODE_KEY, + R_SELECTION_STALENESS_GRAPH, + R_SELECTION_STALENESS_WARNING, + R_BACKFILL_RESULT, + R_CHANGE_DETECTION_RESULT, + R_UNIT_TEST_SELECTOR_SPLIT, + R_MODEL_PLAN_CONTEXT, + R_CURSOR_OVERRIDE_PAIR, + R_PLAN_ENTRY_BUILD_INPUTS, + R_FUNCTION_CHANGE_RESULT, + R_PLANNER_CHANGE_RESULTS, + R_RESOLVED_MODEL_ACTION, + R_PLANNER_RESOLVED_ACTIONS, + R_DIRECT_MODEL_VERSION_IDENTITIES, + R_PLANNER_MODEL_ENTRY_RESULTS, + R_SCHEMA_ACTION, + R_PLAN_WARNING, + R_COLUMN_RENAME_HINT, + R_QUERY_PROJECTION, + R_QUERY_SHAPE, + R_PLAN_OUTPUT_EXTRAS, + R_PLANNED_SQL_TESTS, + R_NATIVE_SQL_TEST_PLAN, + R_SQL_TEST_PLAN_RESULT, + R_SCENARIO_GRAPH_PLAN, + R_SCENARIO_ARTIFACT_IDENTITY, + R_SCENARIO_ARTIFACT_NAME, + R_PLAN_PROVIDER_USAGE, + R_PLANNER_RUNTIME, + R_PLANNER_SCOPE_RESOLUTION, + R_PLANNER_WAREHOUSE_STATE, + R_MODEL_MIGRATION_REQUEST, + R_MODEL_MIGRATION_DECLARATION, + R_MODEL_MIGRATION_DISCOVERY, + R_MIGRATION_COMPATIBILITY_RESULT, + R_MODEL_MIGRATION_PLANNING, + R_OLD_NAME_VIEW_PLANNING, + R_COLUMN_MIGRATION_PLANNING, + R_PLANNER_IDENTITY_CONTEXT, + R_PLANNER_SCOPE_PRUNING_RESULT, + R_PLANNER_CHANGE_RECONCILIATION, + R_PLANNER_ENTRY_RESULTS, +]; + +/// Planner string enums, in declaration order. +pub(crate) const PLANNER_ENUMS: &[EnumSpec] = &[ + EnumSpec { + name: "GraphResourceKind", + doc: "", + members: &[ + ("MODEL", "model"), + ("SEED", "seed"), + ("SOURCE", "source"), + ("FUNCTION", "function"), + ], + }, + EnumSpec { + name: "ChangeKind", + doc: "", + members: &[ + ("FIRST_RUN", "first_run"), + ("RENAMED", "renamed"), + ("QUERY_CHANGED", "query_changed"), + ("FUNCTION_CHANGED", "function_changed"), + ("CONFIG_CHANGED", "config_changed"), + ("SCHEMA_CHANGED", "schema_changed"), + ("NO_CHANGE", "no_change"), + ], + }, + EnumSpec { + name: "SchemaChangeKind", + doc: "", + members: &[ + ("COLUMN_ADDED", "column_added"), + ("COLUMN_REMOVED", "column_removed"), + ("COLUMN_TYPE_CHANGED", "column_type_changed"), + ], + }, + EnumSpec { + name: "SchemaColumnSource", + doc: "", + members: &[("YML", "yml"), ("SQL_ANALYSIS", "sql_analysis")], + }, + EnumSpec { + name: "CursorWatermarkMode", + doc: "", + members: &[("ALL", "all"), ("ANY", "any")], + }, + EnumSpec { + name: "SchemaActionKind", + doc: "", + members: &[ + ("ADD_COLUMN", "add_column"), + ("DROP_COLUMN", "drop_column"), + ("ALTER_COLUMN_TYPE", "alter_column_type"), + ], + }, + EnumSpec { + name: "WarningSeverity", + doc: "", + members: &[("ERROR", "error"), ("WARNING", "warning"), ("INFO", "info")], + }, +]; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_registration/registration.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_registration/registration.rs index 62cfa9cd58..a4c60b4b08 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_registration/registration.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_registration/registration.rs @@ -3,13 +3,12 @@ use pyo3::prelude::{Bound, PyModule, PyResult}; use crate::bindings::_helpers::{ - planner_changes, planner_entries, planner_identity, planner_migrations, planner_models, - planner_phases, planner_resolution, planner_scenario, planner_scope, planner_selection, - planner_strategy, planner_warehouse, relation_naming, + planner_changes, planner_entries, planner_identity, planner_migrations, planner_phases, + planner_resolution, planner_scenario, planner_scope, planner_selection, planner_strategy, + planner_warehouse, relation_naming, }; pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { - planner_models::registration::register(module)?; planner_changes::registration::register(module)?; planner_migrations::registration::register(module)?; planner_phases::registration::register(module)?; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/record_behaviours/constants.rs b/crates/sqlbuild-python/src/bindings/_helpers/record_behaviours/constants.rs index ecdc5270ee..8025590a3b 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/record_behaviours/constants.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/record_behaviours/constants.rs @@ -50,10 +50,11 @@ use crate::bindings::_helpers::record_behaviours::migrations::{ }; use crate::bindings::_helpers::record_behaviours::planner::{ column_migration_blocks_build, column_migration_origin_missing, finish_cursor_overrides, - materialization_is_table_backed, model_migration_blocks_build, model_migration_origin_hidden, - model_migration_origin_missing, model_migration_storage_transition, - model_migration_target_label, old_name_runs_steps, overrides_have_untyped, overrides_typed_as, - plan_python_source_entries, retention_decreases, + finish_scenario_artifact_kind, materialization_is_table_backed, model_migration_blocks_build, + model_migration_origin_hidden, model_migration_origin_missing, + model_migration_storage_transition, model_migration_target_label, old_name_runs_steps, + overrides_have_untyped, overrides_typed_as, plan_python_source_entries, projection_key, + rename_hint_message, retention_decreases, }; use crate::bindings::_helpers::record_behaviours::python_nodes::graph_selected_sql_refs; use crate::bindings::_helpers::record_behaviours::references::{ @@ -75,6 +76,7 @@ use crate::bindings::_helpers::records::types::{ClassMethod, ClassProperty, Reco /// Compiler records whose bound values are normalized or validated natively. pub(crate) const BEHAVIOUR_FINISHERS: &[(&str, RecordFinish)] = &[ + ("ScenarioArtifactIdentity", finish_scenario_artifact_kind), ("MeasurementThresholdBound", finish_threshold_bound), ("MeasurementThresholds", finish_thresholds), ("MeasurementContract", finish_measurement_contract), @@ -98,6 +100,8 @@ pub(crate) const BEHAVIOUR_FINISHERS: &[(&str, RecordFinish)] = &[ ]; /// Computed attributes of compiler shapes. pub(crate) const BEHAVIOUR_PROPERTIES: &[(&str, &str, ClassProperty)] = &[ + ("ColumnRenameHint", "message", rename_hint_message), + ("QueryProjection", "key", projection_key), ("ScopeCompleteness", "complete", completeness_complete), ("ScopeTargetQuery", "unknown", target_query_unknown), ("SettingsConfig", "sql_validation", settings_sql_validation), diff --git a/crates/sqlbuild-python/src/bindings/_helpers/record_behaviours/planner.rs b/crates/sqlbuild-python/src/bindings/_helpers/record_behaviours/planner.rs index 5ccafc6aea..8c6fbbaffd 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/record_behaviours/planner.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/record_behaviours/planner.rs @@ -309,3 +309,45 @@ pub(crate) fn forward_only_backfill(py: Python<'_>) -> PyResult )?; imported(py, NATIVE_MODULE, "BackfillResult")?.call((), Some(&keywords)) } + +/// `ScenarioArtifactIdentity.kind` coerced to `ScenarioArtifactKind`. +pub(crate) fn finish_scenario_artifact_kind( + py: Python<'_>, + mut values: Vec>, +) -> PyResult>> { + if let Some(kind) = values.first_mut() { + *kind = imported(py, NATIVE_MODULE, "ScenarioArtifactKind")? + .call1((kind.bind(py),))? + .unbind(); + } + Ok(values) +} + +/// `ColumnRenameHint.message`: the one-line hint shown beside the added column. +pub(crate) fn rename_hint_message(hint: &Bound<'_, PyAny>) -> PyResult> { + let relation: &str = if hint.getattr("identical")?.is_truthy()? { + "same expression as" + } else { + "similar to" + }; + let candidates: Vec = hint + .getattr("candidate_columns")? + .try_iter()? + .map(|candidate| Ok(candidate?.str()?.to_string())) + .collect::>()?; + let origin: &str = match candidates.as_slice() { + [only] => only, + _ => "", + }; + format!( + "{relation} {}; if this is a rename, add {} (migrate_from {origin})", + candidates.join(", "), + hint.getattr("added_column")?.str()? + ) + .into_py_any(hint.py()) +} + +/// `QueryProjection.key`: the case-insensitive output name. +pub(crate) fn projection_key(projection: &Bound<'_, PyAny>) -> PyResult> { + Ok(projection.getattr("name")?.call_method0("lower")?.unbind()) +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/records/constants.rs b/crates/sqlbuild-python/src/bindings/_helpers/records/constants.rs index b36842fba5..09870bba21 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/records/constants.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/records/constants.rs @@ -6,7 +6,9 @@ use crate::bindings::_helpers::compiler_errors::constants::COMPILER_ERRORS; use crate::bindings::_helpers::compiler_records::constants::COMPILER_RECORDS; use crate::bindings::_helpers::contract_records::constants::CONTRACT_RECORDS; use crate::bindings::_helpers::freshness::observation::normalize_observation; -use crate::bindings::_helpers::planner_records::constants::PLANNER_RECORDS; +use crate::bindings::_helpers::planner_records::constants::{ + PLANNER_ENUMS, PLANNER_RECORDS, PLANNER_VALUE_RECORDS, +}; use crate::bindings::_helpers::record_behaviours::constants::{ BEHAVIOUR_CACHED_PROPERTIES, BEHAVIOUR_FINISHERS, BEHAVIOUR_METHODS, BEHAVIOUR_ORDERED, BEHAVIOUR_PROPERTIES, BEHAVIOUR_SEQUENCES, @@ -138,6 +140,7 @@ pub(crate) const ENUMS: &[&[EnumSpec]] = &[ DBT_ENUMS, PRESENTATION_ENUMS, COMPILER_ENUMS, + PLANNER_ENUMS, ]; /// Every native record group, in registration order. @@ -155,6 +158,7 @@ pub(crate) const RECORDS: &[&[RecordSpec]] = &[ COMPILE_VALUE_RECORDS, SCOPE_RECORDS, PLANNER_RECORDS, + PLANNER_VALUE_RECORDS, RULE_RECORDS, CONTRACT_RECORDS, ]; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/records/fields.rs b/crates/sqlbuild-python/src/bindings/_helpers/records/fields.rs index 5ce038f7a2..02c32155fd 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/records/fields.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/records/fields.rs @@ -119,7 +119,7 @@ fn default_value<'py>( } } -/// The values of the fields that take part in equality, hashing and repr. +/// The values of the fields that take part in equality and hashing. pub(crate) fn compared_values<'py>( record: &Bound<'py, NativeRecord>, ) -> PyResult> { @@ -135,14 +135,14 @@ pub(crate) fn compared_values<'py>( PyTuple::new(py, values) } -/// `Name(field=value!r, ...)` over the compared fields, as a dataclass repr reads. +/// `Name(field=value!r, ...)` over the shown fields, as a dataclass repr reads. pub(crate) fn repr_text(record: &Bound<'_, NativeRecord>) -> PyResult { let py: Python<'_> = record.py(); let cls: Bound<'_, PyType> = record.get_type(); let spec: &RecordSpec = record_spec(&cls)?; let mut parts: Vec = Vec::with_capacity(spec.fields.len()); for (field, value) in spec.fields.iter().zip(record.get().values.iter()) { - if field.compared { + if field.shown { parts.push(format!("{}={}", field.name, value.bind(py).repr()?)); } } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/records/models.rs b/crates/sqlbuild-python/src/bindings/_helpers/records/models.rs index e394ec6148..4c2b8657de 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/records/models.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/records/models.rs @@ -40,12 +40,13 @@ pub(crate) enum FieldDefault { CallText(&'static str, &'static str, &'static str), } -/// One record field: its name, default and whether it takes part in equality and repr. +/// One record field: its name, default, and whether it takes part in equality and in repr. #[derive(Debug, Clone, Copy)] pub(crate) struct FieldSpec { pub(crate) name: &'static str, pub(crate) default: FieldDefault, pub(crate) compared: bool, + pub(crate) shown: bool, } /// The layout of one native record class. @@ -290,6 +291,27 @@ impl FieldSpec { name, default, compared: true, + shown: true, + } + } + + /// A field left out of equality, hashing and repr. + pub(crate) const fn uncompared(name: &'static str, default: FieldDefault) -> Self { + Self { + name, + default, + compared: false, + shown: false, + } + } + + /// A field left out of equality and hashing but shown in repr. + pub(crate) const fn uncompared_shown(name: &'static str, default: FieldDefault) -> Self { + Self { + name, + default, + compared: false, + shown: true, } } } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/records/registration.rs b/crates/sqlbuild-python/src/bindings/_helpers/records/registration.rs index 09501ed91e..e26703adf0 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/records/registration.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/records/registration.rs @@ -132,7 +132,7 @@ fn dataclass_fields<'py>( for field in spec.fields { let options: Bound<'py, PyDict> = PyDict::new(py); options.set_item("compare", field.compared)?; - options.set_item("repr", field.compared)?; + options.set_item("repr", field.shown)?; let declared: Bound<'py, PyAny> = dataclasses.getattr("field")?.call((), Some(&options))?; declared.setattr("name", field.name)?; declared.setattr("type", "object")?; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/scope_records/constants.rs b/crates/sqlbuild-python/src/bindings/_helpers/scope_records/constants.rs index 8cdc552555..1e77649b84 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/scope_records/constants.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/scope_records/constants.rs @@ -206,6 +206,7 @@ const R_SCOPE_LOOKUP: RecordSpec = RecordSpec { name: "native", default: FieldDefault::None, compared: false, + shown: false, }, ], }; diff --git a/src/sqlbuild/_native.pyi b/src/sqlbuild/_native.pyi index be0c342019..f0004ab238 100644 --- a/src/sqlbuild/_native.pyi +++ b/src/sqlbuild/_native.pyi @@ -664,11 +664,9 @@ class FixtureColumnInferences: ) -> FixtureColumnInference | None: """Return the inference for one fixture query, computing each distinct query once.""" -class NativeSqlTestArtifact: +class NativeSqlTestArtifact(NativeRecord): """One native-planned SQL-test artifact and its distinct planning error messages.""" - __dataclass_fields__: ClassVar[dict[str, Any]] - sql: str model_names: tuple[str, ...] error_messages: tuple[str, ...] @@ -677,21 +675,17 @@ class NativeSqlTestArtifact: ) -> None: ... def __replace__(self, **changes: Any) -> NativeSqlTestArtifact: ... -class RelationFixtureDiagnostic: +class RelationFixtureDiagnostic(NativeRecord): """One statically provable invalid fixture condition.""" - __dataclass_fields__: ClassVar[dict[str, Any]] - key: FixtureKey message: str def __init__(self, key: FixtureKey, message: str) -> None: ... def __replace__(self, **changes: Any) -> RelationFixtureDiagnostic: ... -class RelationFixtureCompletion: +class RelationFixtureCompletion(NativeRecord): """Completed fixture SQL plus metadata reused by SQL-test validation.""" - __dataclass_fields__: ClassVar[dict[str, Any]] - fixture_sql_by_key: dict[FixtureKey, str] inferred_by_fixture: dict[FixtureKey, tuple[InferredColumn, ...]] expected_types: Mapping[FixtureKey, dict[str, str]] @@ -705,22 +699,18 @@ class RelationFixtureCompletion: ) -> None: ... def __replace__(self, **changes: Any) -> RelationFixtureCompletion: ... -class FixtureColumnMetadata: +class FixtureColumnMetadata(NativeRecord): """Known type and nullability for one relation fixture column.""" - __dataclass_fields__: ClassVar[dict[str, Any]] - name: str type: str | None nullable: bool | None def __init__(self, name: str, type: str | None, nullable: bool | None) -> None: ... def __replace__(self, **changes: Any) -> FixtureColumnMetadata: ... -class FixtureRelationMetadata: +class FixtureRelationMetadata(NativeRecord): """Known columns and whether they form an authoritative relation shape.""" - __dataclass_fields__: ClassVar[dict[str, Any]] - columns: tuple[FixtureColumnMetadata, ...] authoritative_names: bool def __init__( @@ -728,11 +718,9 @@ class FixtureRelationMetadata: ) -> None: ... def __replace__(self, **changes: Any) -> FixtureRelationMetadata: ... -class RelationFixturePlanningContext: +class RelationFixturePlanningContext(NativeRecord): """Project-wide fixture metadata shared across test and scenario planning.""" - __dataclass_fields__: ClassVar[dict[str, Any]] - models_by_name: dict[str, CompiledModelView] relations: dict[FixtureKey, FixtureRelationMetadata] authoritative_columns: dict[FixtureKey, frozenset[str]] @@ -748,33 +736,27 @@ class RelationFixturePlanningContext: ) -> None: ... def __replace__(self, **changes: Any) -> RelationFixturePlanningContext: ... -class SelectorExpansion: +class SelectorExpansion(NativeRecord): """One selector split into graph expansion flags and core text.""" - __dataclass_fields__: ClassVar[dict[str, Any]] - core: str upstream: bool downstream: bool def __init__(self, core: str, upstream: bool = ..., downstream: bool = ...) -> None: ... def __replace__(self, **changes: Any) -> SelectorExpansion: ... -class ParsedScenarioArtifactName: +class ParsedScenarioArtifactName(NativeRecord): """Parsed physical name for one planner-owned scenario artifact.""" - __dataclass_fields__: ClassVar[dict[str, Any]] - hash_prefix: str kind: str logical_name: str def __init__(self, hash_prefix: str, kind: str, logical_name: str) -> None: ... def __replace__(self, **changes: Any) -> ParsedScenarioArtifactName: ... -class MissingUpstream: +class MissingUpstream(NativeRecord): """One upstream dependency missing from both scope and warehouse.""" - __dataclass_fields__: ClassVar[dict[str, Any]] - key: CompiledObjectKey required_by: tuple[CompiledObjectKey, ...] def __init__( @@ -782,11 +764,9 @@ class MissingUpstream: ) -> None: ... def __replace__(self, **changes: Any) -> MissingUpstream: ... -class WarehouseFingerprints: +class WarehouseFingerprints(NativeRecord): """Latest direct fingerprints grouped by node type.""" - __dataclass_fields__: ClassVar[dict[str, Any]] - models: dict[str, Fingerprint] functions: dict[str, Fingerprint] seeds: dict[str, Fingerprint] @@ -804,11 +784,9 @@ class WarehouseFingerprints: ) -> None: ... def __replace__(self, **changes: Any) -> WarehouseFingerprints: ... -class CursorSnapshotScope: +class CursorSnapshotScope(NativeRecord): """Execution selection used for cursor validation and runtime producer resolution.""" - __dataclass_fields__: ClassVar[dict[str, Any]] - model_keys: frozenset[CompiledObjectKey] runtime_producer_keys: frozenset[CompiledObjectKey] invocation_time: datetime | None @@ -824,11 +802,9 @@ class CursorSnapshotScope: ) -> None: ... def __replace__(self, **changes: Any) -> CursorSnapshotScope: ... -class ReferenceRename: +class ReferenceRename(NativeRecord): """One model rename that downstream references may have been rewritten for.""" - __dataclass_fields__: ClassVar[dict[str, Any]] - new_name: str origin_name: str recorded_at: datetime | None @@ -837,11 +813,9 @@ class ReferenceRename: ) -> None: ... def __replace__(self, **changes: Any) -> ReferenceRename: ... -class WarehouseSnapshot: +class WarehouseSnapshot(NativeRecord): """Frozen point-in-time picture of warehouse state for planning.""" - __dataclass_fields__: ClassVar[dict[str, Any]] - existing_relations: dict[str, RelationInfo] existing_columns: dict[str, tuple[ColumnInfo, ...]] fingerprints: WarehouseFingerprints @@ -869,21 +843,17 @@ class WarehouseSnapshot: ) -> None: ... def __replace__(self, **changes: Any) -> WarehouseSnapshot: ... -class SelectionStalenessNodeKey: +class SelectionStalenessNodeKey(NativeRecord): """Neutral node identity for shared selection-aware staleness classification.""" - __dataclass_fields__: ClassVar[dict[str, Any]] - resource_type: str name: str def __init__(self, resource_type: str, name: str) -> None: ... def __replace__(self, **changes: Any) -> SelectionStalenessNodeKey: ... -class SelectionStalenessGraph: +class SelectionStalenessGraph(NativeRecord): """Neutral graph inputs for selected-model stale warning classification.""" - __dataclass_fields__: ClassVar[dict[str, Any]] - upstream_deps: dict[SelectionStalenessNodeKey, tuple[SelectionStalenessNodeKey, ...]] selected_model_names: frozenset[str] run_model_names: frozenset[str] @@ -905,31 +875,25 @@ class SelectionStalenessGraph: ) -> None: ... def __replace__(self, **changes: Any) -> SelectionStalenessGraph: ... -class SelectionStalenessWarning: +class SelectionStalenessWarning(NativeRecord): """Neutral stale warning classification for one selected model.""" - __dataclass_fields__: ClassVar[dict[str, Any]] - model_name: str trigger_names: tuple[str, ...] def __init__(self, model_name: str, trigger_names: tuple[str, ...]) -> None: ... def __replace__(self, **changes: Any) -> SelectionStalenessWarning: ... -class BackfillResult: +class BackfillResult(NativeRecord): """Resolved backfill action from a change detection policy.""" - __dataclass_fields__: ClassVar[dict[str, Any]] - action: BackfillAction duration: str | None def __init__(self, action: BackfillAction, duration: str | None = ...) -> None: ... def __replace__(self, **changes: Any) -> BackfillResult: ... -class ChangeDetectionResult: +class ChangeDetectionResult(NativeRecord): """Per-model output from change detection and policy resolution.""" - __dataclass_fields__: ClassVar[dict[str, Any]] - model_name: str change_kind: ChangeKind query_changed: bool @@ -959,11 +923,9 @@ class ChangeDetectionResult: ) -> None: ... def __replace__(self, **changes: Any) -> ChangeDetectionResult: ... -class UnitTestSelectorSplit: +class UnitTestSelectorSplit(NativeRecord): """Resource selectors left after unit-test selectors moved into a test selection.""" - __dataclass_fields__: ClassVar[dict[str, Any]] - select: tuple[str, ...] exclude: tuple[str, ...] sql_test_selection: SqlTestSelection @@ -975,11 +937,9 @@ class UnitTestSelectorSplit: ) -> None: ... def __replace__(self, **changes: Any) -> UnitTestSelectorSplit: ... -class ModelPlanContext: +class ModelPlanContext(NativeRecord): """Relation lookups and source columns for building one model plan entry.""" - __dataclass_fields__: ClassVar[dict[str, Any]] - model_locations: dict[str, CompiledRelationLocation] models_by_name: dict[str, CompiledModelView] seed_locations: dict[str, CompiledRelationLocation] @@ -1009,11 +969,9 @@ class ModelPlanContext: ) -> None: ... def __replace__(self, **changes: Any) -> ModelPlanContext: ... -class CursorOverridePair: +class CursorOverridePair(NativeRecord): """Resolved per-model cursor start/end overrides for plan entry construction.""" - __dataclass_fields__: ClassVar[dict[str, Any]] - start_cursor_override: str | None end_cursor_override: str | None def __init__( @@ -1021,11 +979,9 @@ class CursorOverridePair: ) -> None: ... def __replace__(self, **changes: Any) -> CursorOverridePair: ... -class PlanEntryBuildInputs: +class PlanEntryBuildInputs(NativeRecord): """Blocked models and cursor overrides for plan entry building.""" - __dataclass_fields__: ClassVar[dict[str, Any]] - source_freshness_blocked_model_names: frozenset[str] external_blocked_model_names: frozenset[str] start_cursor_override: str | None @@ -1051,21 +1007,17 @@ class PlanEntryBuildInputs: ) -> None: ... def __replace__(self, **changes: Any) -> PlanEntryBuildInputs: ... -class FunctionChangeResult: +class FunctionChangeResult(NativeRecord): """Per-function output from change detection.""" - __dataclass_fields__: ClassVar[dict[str, Any]] - fingerprint_sql: str reason: PlanReason def __init__(self, fingerprint_sql: str, reason: PlanReason = ...) -> None: ... def __replace__(self, **changes: Any) -> FunctionChangeResult: ... -class PlannerChangeResults: +class PlannerChangeResults(NativeRecord): """Change detection output for selected planner resources.""" - __dataclass_fields__: ClassVar[dict[str, Any]] - models: dict[str, ChangeDetectionResult] functions: dict[str, FunctionChangeResult] def __init__( @@ -1073,30 +1025,24 @@ class PlannerChangeResults: ) -> None: ... def __replace__(self, **changes: Any) -> PlannerChangeResults: ... -class ResolvedModelAction: +class ResolvedModelAction(NativeRecord): """Effective model change and backfill decided from the model's own change.""" - __dataclass_fields__: ClassVar[dict[str, Any]] - change: ChangeDetectionResult backfill: BackfillResult def __init__(self, change: ChangeDetectionResult, backfill: BackfillResult) -> None: ... def __replace__(self, **changes: Any) -> ResolvedModelAction: ... -class PlannerResolvedActions: +class PlannerResolvedActions(NativeRecord): """Per-model planning decisions keyed by model name.""" - __dataclass_fields__: ClassVar[dict[str, Any]] - models: dict[str, ResolvedModelAction] def __init__(self, models: dict[str, ResolvedModelAction]) -> None: ... def __replace__(self, **changes: Any) -> PlannerResolvedActions: ... -class DirectModelVersionIdentities: +class DirectModelVersionIdentities(NativeRecord): """Current direct model version identity values by model name.""" - __dataclass_fields__: ClassVar[dict[str, Any]] - function_local_hashes: dict[str, str] seed_version_hashes: dict[str, str] seed_metadata_jsons: dict[str, str] @@ -1114,11 +1060,9 @@ class DirectModelVersionIdentities: ) -> None: ... def __replace__(self, **changes: Any) -> DirectModelVersionIdentities: ... -class PlannerModelEntryResults: +class PlannerModelEntryResults(NativeRecord): """Model plan-entry phase output.""" - __dataclass_fields__: ClassVar[dict[str, Any]] - entries: tuple[ModelPlanEntry, ...] warnings: tuple[PlanWarning, ...] def __init__( @@ -1126,11 +1070,9 @@ class PlannerModelEntryResults: ) -> None: ... def __replace__(self, **changes: Any) -> PlannerModelEntryResults: ... -class SchemaAction: +class SchemaAction(NativeRecord): """One concrete schema change action to apply to a target table.""" - __dataclass_fields__: ClassVar[dict[str, Any]] - kind: SchemaActionKind column_name: str column_type: str | None @@ -1139,11 +1081,9 @@ class SchemaAction: ) -> None: ... def __replace__(self, **changes: Any) -> SchemaAction: ... -class PlanWarning: +class PlanWarning(NativeRecord): """One warning or error produced during plan resolution.""" - __dataclass_fields__: ClassVar[dict[str, Any]] - model_name: str | None severity: WarningSeverity message: str @@ -1157,11 +1097,9 @@ class PlanWarning: ) -> None: ... def __replace__(self, **changes: Any) -> PlanWarning: ... -class ColumnRenameHint: +class ColumnRenameHint(NativeRecord): """A removed column that an added column may have been renamed from.""" - __dataclass_fields__: ClassVar[dict[str, Any]] - model_name: str added_column: str candidate_columns: tuple[str, ...] @@ -1178,11 +1116,9 @@ class ColumnRenameHint: def message(self) -> str: """The one-line hint shown beside the added column.""" -class QueryProjection: +class QueryProjection(NativeRecord): """One named top-level output column of a query and its canonical expression.""" - __dataclass_fields__: ClassVar[dict[str, Any]] - name: str expression: str references: frozenset[str] @@ -1195,11 +1131,9 @@ class QueryProjection: def key(self) -> str: """The case-insensitive output name.""" -class QueryShape: +class QueryShape(NativeRecord): """Canonical top-level projections and the rest of one query, ignoring formatting.""" - __dataclass_fields__: ClassVar[dict[str, Any]] - projections: tuple[QueryProjection, ...] body: str alias_clauses: str @@ -1219,11 +1153,9 @@ class QueryShape: ) -> None: ... def __replace__(self, **changes: Any) -> QueryShape: ... -class PlanOutputExtras: +class PlanOutputExtras(NativeRecord): """Optional supplemental seed fingerprints and precomputed SQL tests for plan assembly.""" - __dataclass_fields__: ClassVar[dict[str, Any]] - seed_version_hashes: dict[str, str] | None seed_metadata_jsons: dict[str, str] | None seed_plan_reasons: dict[str, PlanReason] | None @@ -1237,11 +1169,9 @@ class PlanOutputExtras: ) -> None: ... def __replace__(self, **changes: Any) -> PlanOutputExtras: ... -class PlannedSqlTests: +class PlannedSqlTests(NativeRecord): """SQL test entries planned for one selection; reusable only for the same selection.""" - __dataclass_fields__: ClassVar[dict[str, Any]] - selected_keys: frozenset[CompiledObjectKey] sql_test_selection: SqlTestSelection entries: tuple[SqlTestPlanEntry, ...] @@ -1255,11 +1185,9 @@ class PlannedSqlTests: ) -> None: ... def __replace__(self, **changes: Any) -> PlannedSqlTests: ... -class NativeSqlTestPlan: +class NativeSqlTestPlan(NativeRecord): """Native chain and assertion steps for one SQL test, with SQL when rendered.""" - __dataclass_fields__: ClassVar[dict[str, Any]] - chain: tuple[ChainStep, ...] assertions: tuple[SqlTestAssertionStep, ...] model_names: tuple[str, ...] @@ -1277,11 +1205,9 @@ class NativeSqlTestPlan: ) -> None: ... def __replace__(self, **changes: Any) -> NativeSqlTestPlan: ... -class SqlTestPlanResult: +class SqlTestPlanResult(NativeRecord): """One SQL test's plan entry and warnings, or the fixture diagnostics preventing it.""" - __dataclass_fields__: ClassVar[dict[str, Any]] - entry: SqlTestPlanEntry | None warnings: tuple[PlanWarning, ...] fixture_diagnostics: tuple[str, ...] @@ -1293,11 +1219,9 @@ class SqlTestPlanResult: ) -> None: ... def __replace__(self, **changes: Any) -> SqlTestPlanResult: ... -class ScenarioGraphPlan: +class ScenarioGraphPlan(NativeRecord): """Inferred graph slice and fixture boundaries for one SQL scenario.""" - __dataclass_fields__: ClassVar[dict[str, Any]] - key: CompiledObjectKey name: str target_model_names: tuple[str, ...] @@ -1325,31 +1249,25 @@ class ScenarioGraphPlan: ) -> None: ... def __replace__(self, **changes: Any) -> ScenarioGraphPlan: ... -class ScenarioArtifactIdentity: +class ScenarioArtifactIdentity(NativeRecord): """Logical identity for one scenario-owned physical artifact.""" - __dataclass_fields__: ClassVar[dict[str, Any]] - kind: ScenarioArtifactKind | str logical_name: str def __init__(self, kind: ScenarioArtifactKind | str, logical_name: str) -> None: ... def __replace__(self, **changes: Any) -> ScenarioArtifactIdentity: ... -class ScenarioArtifactName: +class ScenarioArtifactName(NativeRecord): """Resolved physical relation name for one scenario artifact.""" - __dataclass_fields__: ClassVar[dict[str, Any]] - identity: ScenarioArtifactIdentity physical_name: str def __init__(self, identity: ScenarioArtifactIdentity, physical_name: str) -> None: ... def __replace__(self, **changes: Any) -> ScenarioArtifactName: ... -class PlanProviderUsage: +class PlanProviderUsage(NativeRecord): """Provider usage metadata scoped to planned work.""" - __dataclass_fields__: ClassVar[dict[str, Any]] - provider_name: str consumer_kind: str consumer_name: str @@ -1367,11 +1285,9 @@ class PlanProviderUsage: ) -> None: ... def __replace__(self, **changes: Any) -> PlanProviderUsage: ... -class PlannerRuntime: +class PlannerRuntime(NativeRecord): """Resolved execution environment shared by planner phases.""" - __dataclass_fields__: ClassVar[dict[str, Any]] - project: CompiledProjectView adapter: BaseAdapter connection: Any @@ -1391,11 +1307,9 @@ class PlannerRuntime: ) -> None: ... def __replace__(self, **changes: Any) -> PlannerRuntime: ... -class PlannerScopeResolution: +class PlannerScopeResolution(NativeRecord): """Resolved planner scopes for one plan build.""" - __dataclass_fields__: ClassVar[dict[str, Any]] - selected_scope: PlannerScope stale_warning_scope: PlannerScope inspection_scope: PlannerScope @@ -1407,11 +1321,9 @@ class PlannerScopeResolution: ) -> None: ... def __replace__(self, **changes: Any) -> PlannerScopeResolution: ... -class PlannerWarehouseState: +class PlannerWarehouseState(NativeRecord): """Warehouse snapshot and inspection relations gathered once per plan.""" - __dataclass_fields__: ClassVar[dict[str, Any]] - snapshot: WarehouseSnapshot inspection_relations: PlannerRelationsContext migration_entries: tuple[ModelMigrationPlanEntry, ...] @@ -1435,11 +1347,9 @@ class PlannerWarehouseState: ) -> None: ... def __replace__(self, **changes: Any) -> PlannerWarehouseState: ... -class ModelMigrationRequest: +class ModelMigrationRequest(NativeRecord): """One destination model whose data should come from an earlier relation.""" - __dataclass_fields__: ClassVar[dict[str, Any]] - model: CompiledModelView discovery: MigrationDiscovery raw_origin: str | None @@ -1459,11 +1369,9 @@ class ModelMigrationRequest: ) -> None: ... def __replace__(self, **changes: Any) -> ModelMigrationRequest: ... -class ModelMigrationDeclaration: +class ModelMigrationDeclaration(NativeRecord): """One project model's explicit migrate_from, resolved to a relation.""" - __dataclass_fields__: ClassVar[dict[str, Any]] - model_name: str origin_location: CompiledRelationLocation origin_model: str | None @@ -1472,11 +1380,9 @@ class ModelMigrationDeclaration: ) -> None: ... def __replace__(self, **changes: Any) -> ModelMigrationDeclaration: ... -class ModelMigrationDiscovery: +class ModelMigrationDiscovery(NativeRecord): """Manual and automatically discovered migration requests for one plan.""" - __dataclass_fields__: ClassVar[dict[str, Any]] - requests: tuple[ModelMigrationRequest, ...] warnings: tuple[PlanWarning, ...] destination_fingerprints: dict[str, tuple[str, ...]] @@ -1488,21 +1394,17 @@ class ModelMigrationDiscovery: ) -> None: ... def __replace__(self, **changes: Any) -> ModelMigrationDiscovery: ... -class MigrationCompatibilityResult: +class MigrationCompatibilityResult(NativeRecord): """Compatibility status plus human-readable blocking findings.""" - __dataclass_fields__: ClassVar[dict[str, Any]] - status: MigrationCompatibility findings: tuple[str, ...] def __init__(self, status: MigrationCompatibility, findings: tuple[str, ...] = ...) -> None: ... def __replace__(self, **changes: Any) -> MigrationCompatibilityResult: ... -class ModelMigrationPlanning: +class ModelMigrationPlanning(NativeRecord): """Migration decisions plus the warehouse snapshot as it will look after moves.""" - __dataclass_fields__: ClassVar[dict[str, Any]] - snapshot: WarehouseSnapshot entries: tuple[ModelMigrationPlanEntry, ...] warnings: tuple[PlanWarning, ...] @@ -1514,11 +1416,9 @@ class ModelMigrationPlanning: ) -> None: ... def __replace__(self, **changes: Any) -> ModelMigrationPlanning: ... -class OldNameViewPlanning: +class OldNameViewPlanning(NativeRecord): """Old-name steps for this build plus every compatibility view still recorded.""" - __dataclass_fields__: ClassVar[dict[str, Any]] - entries: tuple[OldNameViewPlanEntry, ...] views: tuple[OldNameView, ...] warnings: tuple[PlanWarning, ...] @@ -1530,11 +1430,9 @@ class OldNameViewPlanning: ) -> None: ... def __replace__(self, **changes: Any) -> OldNameViewPlanning: ... -class ColumnMigrationPlanning: +class ColumnMigrationPlanning(NativeRecord): """Column rename decisions plus the warehouse snapshot as it will look after renames.""" - __dataclass_fields__: ClassVar[dict[str, Any]] - snapshot: WarehouseSnapshot entries: tuple[ColumnMigrationPlanEntry, ...] hints: tuple[ColumnRenameHint, ...] @@ -1548,11 +1446,9 @@ class ColumnMigrationPlanning: ) -> None: ... def __replace__(self, **changes: Any) -> ColumnMigrationPlanning: ... -class PlannerIdentityContext: +class PlannerIdentityContext(NativeRecord): """Expected version identities for inspection and stale-warning scopes.""" - __dataclass_fields__: ClassVar[dict[str, Any]] - version_identities: DirectModelVersionIdentities stale_warning_identities: DirectModelVersionIdentities def __init__( @@ -1562,11 +1458,9 @@ class PlannerIdentityContext: ) -> None: ... def __replace__(self, **changes: Any) -> PlannerIdentityContext: ... -class PlannerScopePruningResult: +class PlannerScopePruningResult(NativeRecord): """Scope pruning outcome with pruned scopes, actions, and staleness state.""" - __dataclass_fields__: ClassVar[dict[str, Any]] - inspection_scope: PlannerScope execution_scope: PlannerScope resolved_actions: PlannerResolvedActions @@ -1582,11 +1476,9 @@ class PlannerScopePruningResult: ) -> None: ... def __replace__(self, **changes: Any) -> PlannerScopePruningResult: ... -class PlannerChangeReconciliation: +class PlannerChangeReconciliation(NativeRecord): """Write-hash-honest changes merged back into resolved actions.""" - __dataclass_fields__: ClassVar[dict[str, Any]] - changes: PlannerChangeResults resolved_actions: PlannerResolvedActions def __init__( @@ -1594,11 +1486,9 @@ class PlannerChangeReconciliation: ) -> None: ... def __replace__(self, **changes: Any) -> PlannerChangeReconciliation: ... -class PlannerEntryResults: +class PlannerEntryResults(NativeRecord): """Model plan entries for one plan build.""" - __dataclass_fields__: ClassVar[dict[str, Any]] - model_entry_results: PlannerModelEntryResults def __init__(self, model_entry_results: PlannerModelEntryResults) -> None: ... def __replace__(self, **changes: Any) -> PlannerEntryResults: ... From 40f0204658855755bbbf9cd90ae93b6ff0e7202a Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Thu, 15 Oct 2026 22:52:16 +0100 Subject: [PATCH 24/48] refactor: export Rust-owned constants from sqlbuild._native Constants needed by both Rust and stays Python are now defined once in Rust and registered on sqlbuild._native; Python keeps no copy. - Register text, integer, tuple and frozenset constants plus GRAIN_ORDER (built from the Rust cursor grain order) in a new exported_constants binding module, typed in _native.pyi. - Consolidate duplicated Rust literals (project config file name, state table names, settings keys, Python node root, __init__ stem, CTE AST kind, custom-rule test-case threshold key, zero-day duration) into one definition each. - Native code no longer reads fingerprint, output-capture or rule-engine constants back from Python modules. - Delete compiler/planner, compiler/discovery, cursor_algebra, compiler/frontier, cost, compiler/migrations and runtime/execution_history constants modules and their stays entries; trim the remaining stays constants files to Python-only values. - Relabel compiler/sql_analysis/constants.py as rule 4 polyglot_sql glue. --- .../_helpers/dynamic_pivot.rs | 8 +- .../assembly/analysis_session/constants.rs | 1 - .../src/contracts/_helpers/promotion.rs | 6 +- .../src/contracts/constants.rs | 5 +- crates/sqlbuild-analysis/src/contracts/mod.rs | 2 +- .../src/fingerprint_storage/constants.rs | 2 + .../src/janitor_planning/constants.rs | 16 ++- .../main/stored_migration_fingerprint.rs | 4 +- .../src/python_selection/_helpers/nodes.rs | 8 +- .../src/python_selection/_helpers/unified.rs | 7 +- .../src/python_selection/constants.rs | 1 - .../_helpers/explanation/opt_outs.rs | 5 +- .../src/semantic_checks/constants.rs | 2 - .../src/snowflake_cost/constants.rs | 2 +- .../src/snowflake_cost/mod.rs | 2 +- .../src/audits/_helpers/policies.rs | 2 +- .../src/command_connection/constants.rs | 1 - .../main/authored_warehouse.rs | 3 +- .../src/project/_helpers/defaults.rs | 6 +- .../src/project/_helpers/documents.rs | 6 +- .../src/project/_helpers/sections.rs | 3 +- .../sqlbuild-config/src/project/constants.rs | 3 - .../project/main/explicit_references_help.rs | 5 +- .../src/project/main/microbatch_guidance.rs | 7 +- .../project/main/sql_analysis_off_guidance.rs | 6 +- .../src/workspace/_helpers/skill_settings.rs | 5 +- .../src/workspace/constants.rs | 1 - crates/sqlbuild-core/src/constants.rs | 26 ++++ .../main/render_dbt_init_summary.rs | 3 +- .../src/debug_report/constants.rs | 1 - .../debug_report/main/debug_config_files.rs | 2 +- .../src/init_command/constants.rs | 3 - .../init_command/main/render_init_summary.rs | 3 +- .../main/janitor_disabled_text.rs | 3 +- crates/sqlbuild-discovery/src/constants.rs | 4 +- .../_helpers/listing/module_paths.rs | 3 +- .../src/declaration_files/constants.rs | 2 - .../declarations/_helpers/declaration_tree.rs | 5 +- .../model_validation/_helpers/incremental.rs | 3 +- .../src/model_validation/constants.rs | 2 - .../main/retention_override.rs | 5 +- .../src/rule_evidence/_helpers/paths.rs | 3 +- .../src/rule_evidence/constants.rs | 1 - .../analysis_session/model_analyses.rs | 5 +- .../_helpers/attachments/cursor_intrinsics.rs | 6 +- .../_helpers/command_runtime/connection.rs | 3 +- .../compile_attachment/model_inputs.rs | 2 +- .../_helpers/compile_command/constants.rs | 12 +- .../_helpers/compile_command/reuse_attempt.rs | 8 +- .../compile_driver/config_validation.rs | 2 +- .../_helpers/compile_driver/project.rs | 4 +- .../_helpers/compile_driver/targets.rs | 2 +- .../_helpers/contracts/project_config.rs | 3 +- .../bindings/_helpers/cursor/conversion.rs | 3 +- .../_helpers/dbt_command/auto_init.rs | 2 +- .../_helpers/dbt_command/init_project.rs | 2 +- .../discovery/imported_python_nodes.rs | 2 +- .../_helpers/discovery/scope_snapshot.rs | 2 +- .../_helpers/exported_constants/constants.rs | 135 ++++++++++++++++++ .../_helpers/exported_constants/mod.rs | 2 + .../exported_constants/registration.rs | 32 +++++ .../_helpers/fingerprints/storage_io.rs | 2 +- .../_helpers/freshness/freshness_command.rs | 2 +- .../bindings/_helpers/freshness/state_io.rs | 3 +- .../_helpers/frontier/stage_capture.rs | 2 +- .../bindings/_helpers/init_command/command.rs | 2 +- .../_helpers/init_command/scaffold.rs | 3 +- .../_helpers/janitor/janitor_facts.rs | 4 +- .../_helpers/janitor/janitor_old_names.rs | 4 +- .../src/bindings/_helpers/mod.rs | 1 + .../_helpers/plan_output/cursor_details.rs | 7 +- .../_helpers/planner_entries/command_plans.rs | 14 +- .../_helpers/planner_migrations/constants.rs | 2 - .../planner_migrations/state_inspection.rs | 11 +- .../_helpers/planner_warehouse/constants.rs | 6 - .../_helpers/planner_warehouse/snapshot.rs | 12 +- .../_helpers/result_output/event_writer.rs | 2 +- .../src/bindings/_helpers/rules/constants.rs | 5 + .../_helpers/rules/custom_evaluation.rs | 7 +- .../bindings/_helpers/rules/custom_hosts.rs | 12 +- .../_helpers/semantic_shapes/shapes.rs | 6 +- .../_helpers/state_rows/migration_io.rs | 2 +- .../_helpers/state_rows/migration_objects.rs | 6 +- .../src/bindings/main/register.rs | 9 +- .../src/configuration/_helpers/loading.rs | 6 +- .../main/forbidden_exceptions.rs | 2 +- .../src/configuration/main/setting_help.rs | 3 +- crates/sqlbuild-rules/src/constants.rs | 3 +- .../sqlbuild-rules/src/lint_text/constants.rs | 1 - .../lint_text/main/resolve_format_settings.rs | 3 +- .../src/rule_catalogue/constants.rs | 4 +- .../src/rules/_helpers/evaluation.rs | 3 +- .../src/rules/_helpers/model_layers.rs | 3 +- .../src/rules/_helpers/sql_test_rules.rs | 2 +- .../src/scope_cache/constants.rs | 4 +- crates/sqlbuild-state/src/events/constants.rs | 3 - .../sqlbuild-state/src/freshness/constants.rs | 1 - python-boundary.toml | 51 ++----- src/sqlbuild/_native.pyi | 59 ++++++++ .../contract/classes/duckdb_backed_adapter.py | 4 +- .../bigquery/classes/bigquery_adapter.py | 4 +- .../databricks/classes/databricks_adapter.py | 4 +- .../postgres/classes/postgres_adapter.py | 8 +- .../snowflake/classes/snowflake_adapter.py | 4 +- .../snowflake/classes/snowflake_cursor.py | 3 +- .../sqlserver/classes/sqlserver_adapter.py | 19 ++- .../entrypoint/_dispatch_with_compute_logs.py | 8 +- src/sqlbuild/cli/entry/constants.py | 1 - src/sqlbuild/cli/entry/main/launched.py | 2 +- src/sqlbuild/cli/output/constants.py | 2 +- .../_helpers/filesystem/project_modules.py | 6 +- src/sqlbuild/compiler/discovery/constants.py | 13 -- .../compiler/fingerprints/constants.py | 19 --- src/sqlbuild/compiler/frontier/constants.py | 6 - .../_helpers/store_environment.py | 3 +- .../compiler/macro_bridge/constants.py | 12 -- src/sqlbuild/compiler/migrations/constants.py | 10 -- src/sqlbuild/compiler/planner/constants.py | 9 -- .../compiler/source_freshness/constants.py | 3 +- src/sqlbuild/compute_logs.py | 4 +- src/sqlbuild/cost/constants.py | 3 - src/sqlbuild/cursor_algebra/constants.py | 12 -- src/sqlbuild/execution_history.py | 2 +- .../executor/build/_helpers/scheduler.py | 3 +- .../executor/clone/_helpers/fingerprinting.py | 8 +- .../executor/diff/_helpers/query_artifacts.py | 6 +- .../executor/migrations/_helpers/columns.py | 2 +- .../migrations/_helpers/old_name_facts.py | 2 +- .../migrations/_helpers/old_name_views.py | 2 +- .../executor/migrations/_helpers/promotion.py | 2 +- .../run/_helpers/materializations/snapshot.py | 6 +- .../_helpers/reuse/fingerprint_metadata.py | 4 +- .../run/_helpers/reuse/fingerprinting.py | 2 +- .../run/_helpers/validation/cursor_bounds.py | 7 +- src/sqlbuild/executor/run/main/_execute.py | 14 +- .../executor/seed/_helpers/fingerprinting.py | 3 +- .../_helpers/engine/_custom_catalogue.py | 6 +- .../_helpers/engine/rule_harness.py | 2 +- .../_helpers/host/custom_evaluation.py | 6 +- .../rule_engine/_helpers/host/custom_host.py | 6 +- .../rule_engine/classes/contract_facts.py | 3 +- .../rule_engine/classes/sql_document.py | 3 +- src/sqlbuild/rule_engine/constants.py | 10 -- .../runtime/compute_logs/constants.py | 7 - src/sqlbuild/runtime/compute_logs/types.py | 2 +- .../runtime/execution_history/constants.py | 9 -- .../runtime/execution_history/types.py | 2 +- .../runtime/output_capture/constants.py | 36 +---- src/sqlbuild/runtime/output_capture/models.py | 6 +- src/sqlbuild/spec/contracts/constants.py | 1 - tests/conftest.py | 4 +- .../main/compile/fast_lineage/helpers.py | 4 +- .../cli/commands/main/compile/helpers.py | 4 +- .../main/compile/semantic_checks/helpers.py | 4 +- .../cli/commands/main/scenario/helpers.py | 4 +- .../sqlbuild/adapters/bigquery/test_client.py | 2 +- .../adapters/databricks/test_client.py | 2 +- .../commands/main/compile_outputs/helpers.py | 11 +- .../commands/main/model_migrations/helpers.py | 2 +- .../cli/commands/main/test_discovery_work.py | 2 +- .../main/test_startup_import_footprint.py | 4 +- .../src/sqlbuild/cli/entry/helpers.py | 4 +- .../fingerprints/main/test_read_write.py | 2 +- .../source_freshness/main/test_read_write.py | 2 +- .../incremental/test_incremental_execution.py | 3 +- .../executor/run/microbatch/helpers.py | 6 +- .../microbatch/test_microbatch_execution.py | 6 +- .../main/entry/test_observability_cleanup.py | 2 +- .../test_dispatch_with_output_capture.py | 3 +- .../cli/compile_reuse/main/helpers.py | 4 +- .../main/shared/_helpers/test_sql.py | 2 +- .../compiler/frontier/test_stage_capture.py | 8 +- .../planner/_helpers/resolve/test_cursor.py | 6 +- .../test_plan_entry_cursor_overrides.py | 3 +- .../_helpers/test_selection_staleness.py | 2 +- .../compiler/source_freshness/main/helpers.py | 2 +- .../main/shared/_helpers/test_sql.py | 2 +- .../clone/_helpers/fingerprinting/helpers.py | 2 +- .../fingerprinting/test_fingerprinting.py | 3 +- .../sqlbuild/executor/janitor/main/helpers.py | 4 +- .../executor/janitor/main/test_plan.py | 4 +- .../run/_helpers/test_fingerprint_metadata.py | 3 +- .../run/_helpers/test_snapshot_lifecycle.py | 3 +- .../executor/seed/main/test_execute.py | 2 +- .../rule_engine/_helpers/engine/helpers.py | 2 +- .../main/evaluate_rule/test_evaluate_rule.py | 9 +- .../classes/test_dispatcher.py | 2 +- .../execution_history/conformance/conftest.py | 10 +- .../main/test_invocation_context.py | 3 +- 189 files changed, 613 insertions(+), 542 deletions(-) create mode 100644 crates/sqlbuild-python/src/bindings/_helpers/exported_constants/constants.rs create mode 100644 crates/sqlbuild-python/src/bindings/_helpers/exported_constants/mod.rs create mode 100644 crates/sqlbuild-python/src/bindings/_helpers/exported_constants/registration.rs delete mode 100644 src/sqlbuild/compiler/discovery/constants.py delete mode 100644 src/sqlbuild/compiler/frontier/constants.py delete mode 100644 src/sqlbuild/compiler/migrations/constants.py delete mode 100644 src/sqlbuild/compiler/planner/constants.py delete mode 100644 src/sqlbuild/cost/constants.py delete mode 100644 src/sqlbuild/cursor_algebra/constants.py delete mode 100644 src/sqlbuild/runtime/execution_history/constants.py diff --git a/crates/sqlbuild-analysis/src/assembly/analysis_session/_helpers/dynamic_pivot.rs b/crates/sqlbuild-analysis/src/assembly/analysis_session/_helpers/dynamic_pivot.rs index d1589b8930..b37e6c4e4b 100644 --- a/crates/sqlbuild-analysis/src/assembly/analysis_session/_helpers/dynamic_pivot.rs +++ b/crates/sqlbuild-analysis/src/assembly/analysis_session/_helpers/dynamic_pivot.rs @@ -13,15 +13,15 @@ use crate::assembly::analysis_session::_helpers::dict_walk::{ parse_one_or_error, payload, relation_name, render_type, to_dict, truthy, upper, }; use crate::assembly::analysis_session::constants::{ - ALIAS_AST_KIND, CAST_AST_KIND, CTE_AST_KIND, DUCKDB_DIALECT, DYNAMIC_VALUE_SOURCE_KINDS, - MOTHERDUCK_DIALECT, PIVOT_AST_KIND, QUERY_AST_KINDS, SELECT_AST_KIND, - SIMPLIFIED_PIVOT_DIALECTS, SUPPORTED_PIVOT_DIALECTS, TYPE_PRESERVING_AGGREGATES, - UNKNOWN_NULLABILITY, UNKNOWN_TYPE, + ALIAS_AST_KIND, CAST_AST_KIND, DUCKDB_DIALECT, DYNAMIC_VALUE_SOURCE_KINDS, MOTHERDUCK_DIALECT, + PIVOT_AST_KIND, QUERY_AST_KINDS, SELECT_AST_KIND, SIMPLIFIED_PIVOT_DIALECTS, + SUPPORTED_PIVOT_DIALECTS, TYPE_PRESERVING_AGGREGATES, UNKNOWN_NULLABILITY, UNKNOWN_TYPE, }; use crate::assembly::analysis_session::models::{ ColumnFact, ContractProof, DynamicFamily, PivotModel, PivotOutcome, PivotTables, }; use crate::assembly::analysis_session::types::Shapes; +use sqlbuild_core::constants::CTE_AST_KIND; /// What an internal native failure of a pivot proof names. const PIVOT_CONTEXT: &str = "native dynamic pivot proof"; diff --git a/crates/sqlbuild-analysis/src/assembly/analysis_session/constants.rs b/crates/sqlbuild-analysis/src/assembly/analysis_session/constants.rs index c9a57102f7..2b786482a0 100644 --- a/crates/sqlbuild-analysis/src/assembly/analysis_session/constants.rs +++ b/crates/sqlbuild-analysis/src/assembly/analysis_session/constants.rs @@ -72,7 +72,6 @@ pub(crate) const COLUMN_AST_KIND: &str = "column"; pub(crate) const TABLE_AST_KIND: &str = "table"; pub(crate) const FUNCTION_AST_KIND: &str = "function"; pub(crate) const ALIAS_AST_KIND: &str = "alias"; -pub(crate) const CTE_AST_KIND: &str = "cte"; pub(crate) const STAR_AST_KIND: &str = "star"; pub(crate) const DYNAMIC_VALUE_SOURCE_KINDS: [&str; 3] = ["pivot_any", "select", "query"]; pub(crate) const DEPENDENCY_FUNCTION_NAMES: [&str; 3] = ["__ref", "__source", "__seed"]; diff --git a/crates/sqlbuild-analysis/src/contracts/_helpers/promotion.rs b/crates/sqlbuild-analysis/src/contracts/_helpers/promotion.rs index a445125ab0..270450768e 100644 --- a/crates/sqlbuild-analysis/src/contracts/_helpers/promotion.rs +++ b/crates/sqlbuild-analysis/src/contracts/_helpers/promotion.rs @@ -5,10 +5,12 @@ use crate::contracts::_helpers::setting_help::{ }; use crate::contracts::constants::{ CONTRACT_ENFORCED, IMMEDIATE_PROMOTION_MODE, MICROBATCH_INCREMENTAL_MODE, - PROMOTION_CONFLICT_CODE, PROMOTION_REMOVE_CONTRACT_HELP, SETTINGS_SECTION, - STAGED_LIFECYCLE_MATERIALIZATIONS, STAGED_PROMOTION_MODE, TABLE_PROMOTION_MODE_SETTING_KEY, + PROMOTION_CONFLICT_CODE, PROMOTION_REMOVE_CONTRACT_HELP, STAGED_LIFECYCLE_MATERIALIZATIONS, }; use crate::contracts::models::{PromotionConflict, PromotionModel, PromotionRequest}; +use sqlbuild_core::constants::SETTINGS_SECTION; +use sqlbuild_core::constants::STAGED_PROMOTION_MODE; +use sqlbuild_core::constants::TABLE_PROMOTION_MODE_SETTING_KEY; /// One conflict per enforced-contract table model when the effective promotion is immediate. pub(crate) fn promotion_conflicts(request: &PromotionRequest) -> Vec { diff --git a/crates/sqlbuild-analysis/src/contracts/constants.rs b/crates/sqlbuild-analysis/src/contracts/constants.rs index 112d6458f2..26e6e274b7 100644 --- a/crates/sqlbuild-analysis/src/contracts/constants.rs +++ b/crates/sqlbuild-analysis/src/contracts/constants.rs @@ -9,14 +9,11 @@ pub(crate) const MISSING_DECLARATIONS_CODE: &str = "K006"; pub(crate) const DYNAMIC_OUTPUT_NOT_PROVEN_CODE: &str = "K011"; pub(crate) const PROMOTION_CONFLICT_CODE: &str = "K011"; -pub(crate) const CONTRACT_ENFORCED: &str = "enforced"; +pub const CONTRACT_ENFORCED: &str = "enforced"; pub(crate) const CONTRACT_NONE: &str = "none"; pub(crate) const STAGED_LIFECYCLE_MATERIALIZATIONS: [&str; 2] = ["table", "incremental"]; pub(crate) const MICROBATCH_INCREMENTAL_MODE: &str = "microbatch"; pub(crate) const IMMEDIATE_PROMOTION_MODE: &str = "immediate"; -pub(crate) const STAGED_PROMOTION_MODE: &str = "staged"; -pub(crate) const SETTINGS_SECTION: &str = "settings"; -pub(crate) const TABLE_PROMOTION_MODE_SETTING_KEY: &str = "table_promotion_mode"; pub(crate) const NO_COMPILER_EVIDENCE: &str = "no compiler evidence"; pub(crate) const DYNAMIC_OUTPUT_NOT_PROVEN_HELP: &str = "project one supported runtime-dynamic pivot through a wildcard output boundary and declare every generated family"; diff --git a/crates/sqlbuild-analysis/src/contracts/mod.rs b/crates/sqlbuild-analysis/src/contracts/mod.rs index 4ca56896ce..5fb131efc2 100644 --- a/crates/sqlbuild-analysis/src/contracts/mod.rs +++ b/crates/sqlbuild-analysis/src/contracts/mod.rs @@ -1,7 +1,7 @@ //! Model column contracts and promotion conflicts. pub(crate) mod _helpers; -pub(crate) mod constants; +pub mod constants; pub mod main; pub mod models; diff --git a/crates/sqlbuild-analysis/src/fingerprint_storage/constants.rs b/crates/sqlbuild-analysis/src/fingerprint_storage/constants.rs index 17aad5e650..791dcc6822 100644 --- a/crates/sqlbuild-analysis/src/fingerprint_storage/constants.rs +++ b/crates/sqlbuild-analysis/src/fingerprint_storage/constants.rs @@ -31,3 +31,5 @@ pub const FINGERPRINT_COLUMNS: [&str; 12] = [ /// The node type that wins a node-name collision between latest rows. pub const NODE_TYPE_MODEL: &str = "model"; +/// The node type of seed fingerprints. +pub const NODE_TYPE_SEED: &str = "seed"; diff --git a/crates/sqlbuild-analysis/src/janitor_planning/constants.rs b/crates/sqlbuild-analysis/src/janitor_planning/constants.rs index 9a1087f740..6f0b096877 100644 --- a/crates/sqlbuild-analysis/src/janitor_planning/constants.rs +++ b/crates/sqlbuild-analysis/src/janitor_planning/constants.rs @@ -1,16 +1,22 @@ //! Janitor planning names, prefixes and skip reasons. +use sqlbuild_core::constants::COLUMN_MIGRATION_TABLE_NAME; +use sqlbuild_core::constants::FINGERPRINT_TABLE_NAME; +use sqlbuild_core::constants::MIGRATION_TABLE_NAME; +use sqlbuild_core::constants::OLD_NAME_VIEW_TABLE_NAME; +use sqlbuild_core::constants::SOURCE_FRESHNESS_TABLE_NAME; + /// SQLBuild state tables the janitor never acts on. pub const BUILT_IN_EXCLUDE_PATTERNS: [&str; 9] = [ - "_sqlbuild_fingerprints", + FINGERPRINT_TABLE_NAME, "_sqlbuild_microbatches", - "_sqlbuild_source_freshness", + SOURCE_FRESHNESS_TABLE_NAME, "_sqlbuild_node_results", "_sqlbuild_audit_results", "_sqlbuild_janitor_events", - "_sqlbuild_migrations", - "_sqlbuild_column_migrations", - "_sqlbuild_old_name_views", + MIGRATION_TABLE_NAME, + COLUMN_MIGRATION_TABLE_NAME, + OLD_NAME_VIEW_TABLE_NAME, ]; /// Prefix of every strict janitor archive name. pub const ARCHIVE_NAME_PREFIX: &str = "_SQB_ARCHIVE__"; diff --git a/crates/sqlbuild-analysis/src/planner_migrations/main/stored_migration_fingerprint.rs b/crates/sqlbuild-analysis/src/planner_migrations/main/stored_migration_fingerprint.rs index d9eab548d5..c6f07fd0d5 100644 --- a/crates/sqlbuild-analysis/src/planner_migrations/main/stored_migration_fingerprint.rs +++ b/crates/sqlbuild-analysis/src/planner_migrations/main/stored_migration_fingerprint.rs @@ -1,5 +1,7 @@ //! Read persisted migration hashes without changing their representation. +use crate::planner_changes::constants::MIGRATION_FINGERPRINT_METADATA_KEY; + /// Read only string migration fingerprints from object metadata. pub fn stored_migration_fingerprint(metadata: &str) -> Option { let payload: serde_json::Value = match serde_json::from_str(metadata) { @@ -7,7 +9,7 @@ pub fn stored_migration_fingerprint(metadata: &str) -> Option { Err(_) => return None, }; payload - .get("migration_fingerprint")? + .get(MIGRATION_FINGERPRINT_METADATA_KEY)? .as_str() .map(str::to_owned) } diff --git a/crates/sqlbuild-analysis/src/python_selection/_helpers/nodes.rs b/crates/sqlbuild-analysis/src/python_selection/_helpers/nodes.rs index 40a7abb782..e3d4e92112 100644 --- a/crates/sqlbuild-analysis/src/python_selection/_helpers/nodes.rs +++ b/crates/sqlbuild-analysis/src/python_selection/_helpers/nodes.rs @@ -10,12 +10,12 @@ use crate::graph::main::parse_selector::parse_selector; use crate::graph::models::{EdgeDirection, ParsedSelector, ProjectGraph}; use crate::python_selection::constants::{ ASSET_KIND, CHECK_KIND, INTERSECTION_SEPARATOR, LOADER_KIND, NAME_KIND, PATH_BETWEEN_CODE, - PATH_KIND, PATH_ROOT_CODE, PATH_ROOT_ERROR, PATTERN_CHARACTERS, PYTHON_NODE_PATH_ROOT, - SQL_MODEL_PATH_ROOT, TAG_KIND, TASK_KIND, TEST_KIND, UNIT_TEST_CODE, - UNIT_TEST_ONLY_TEST_AND_BUILD, UNKNOWN_NAME_CODE, UNKNOWN_PATH_CODE, UNKNOWN_TAG_CODE, - UNMAPPED_KIND_CODE, + PATH_KIND, PATH_ROOT_CODE, PATH_ROOT_ERROR, PATTERN_CHARACTERS, SQL_MODEL_PATH_ROOT, TAG_KIND, + TASK_KIND, TEST_KIND, UNIT_TEST_CODE, UNIT_TEST_ONLY_TEST_AND_BUILD, UNKNOWN_NAME_CODE, + UNKNOWN_PATH_CODE, UNKNOWN_TAG_CODE, UNMAPPED_KIND_CODE, }; use crate::python_selection::models::{NodeIndex, PythonSelectionGraph, PythonSelectionNode}; +use sqlbuild_core::constants::PYTHON_NODE_ROOT as PYTHON_NODE_PATH_ROOT; pub(crate) type Names = BTreeSet; type Resolved = Result; diff --git a/crates/sqlbuild-analysis/src/python_selection/_helpers/unified.rs b/crates/sqlbuild-analysis/src/python_selection/_helpers/unified.rs index 6357ed80b9..8d91e471af 100644 --- a/crates/sqlbuild-analysis/src/python_selection/_helpers/unified.rs +++ b/crates/sqlbuild-analysis/src/python_selection/_helpers/unified.rs @@ -19,11 +19,12 @@ use crate::python_selection::_helpers::validation::{ }; use crate::python_selection::constants::{ ASSET_KIND, CHECK_KIND, EXPANSION_MARKER, INTERSECTION_SEPARATOR, LOADER_KIND, NAME_KIND, - PATH_KIND, PATH_ROOT_CODE, PATH_ROOT_ERROR, PLANNER_DEFAULT_CODE, PYTHON_NODE_PATH_ROOT, - SEED_KIND, SOURCE_KIND, SOURCE_RESOURCE, SQL_MODEL_PATH_ROOT, TAG_KIND, TASK_KIND, TEST_KIND, - UNKNOWN_NAME_CODE, UNKNOWN_TAG_CODE, + PATH_KIND, PATH_ROOT_CODE, PATH_ROOT_ERROR, PLANNER_DEFAULT_CODE, SEED_KIND, SOURCE_KIND, + SOURCE_RESOURCE, SQL_MODEL_PATH_ROOT, TAG_KIND, TASK_KIND, TEST_KIND, UNKNOWN_NAME_CODE, + UNKNOWN_TAG_CODE, }; use crate::python_selection::models::{Atom, PythonSelectionNode, RunSelection, SelectionContext}; +use sqlbuild_core::constants::PYTHON_NODE_ROOT as PYTHON_NODE_PATH_ROOT; type Atoms = BTreeSet; type Resolved = Result; diff --git a/crates/sqlbuild-analysis/src/python_selection/constants.rs b/crates/sqlbuild-analysis/src/python_selection/constants.rs index 3bd1d1c28a..202a3b2169 100644 --- a/crates/sqlbuild-analysis/src/python_selection/constants.rs +++ b/crates/sqlbuild-analysis/src/python_selection/constants.rs @@ -19,7 +19,6 @@ pub(crate) const EXPANSION_MARKER: char = '+'; pub(crate) const INTERSECTION_SEPARATOR: char = ','; pub(crate) const PATTERN_CHARACTERS: [char; 3] = ['*', '?', '[']; pub(crate) const SQL_MODEL_PATH_ROOT: &str = "models"; -pub(crate) const PYTHON_NODE_PATH_ROOT: &str = "python"; pub(crate) const CURRENT_DIRECTORY: &str = "."; pub(crate) const ROOT_DIRECTORY: &str = "/"; pub(crate) const PLANNER_DEFAULT_CODE: &str = "S000"; diff --git a/crates/sqlbuild-analysis/src/semantic_checks/_helpers/explanation/opt_outs.rs b/crates/sqlbuild-analysis/src/semantic_checks/_helpers/explanation/opt_outs.rs index f1eccc28f2..cc4e726024 100644 --- a/crates/sqlbuild-analysis/src/semantic_checks/_helpers/explanation/opt_outs.rs +++ b/crates/sqlbuild-analysis/src/semantic_checks/_helpers/explanation/opt_outs.rs @@ -6,10 +6,11 @@ use regex::Regex; use crate::semantic_checks::_helpers::explanation::messages::{compiled, pattern}; use crate::semantic_checks::constants::{ - ADDITIONAL_HELP_SEPARATOR, FINDING_KINDS, PROJECT_CONFIG_FILENAME, REQUIRE_SQL_ANALYSIS_KEY, - SETTING_SNIPPET_INDENT, SETTINGS_SECTION, + ADDITIONAL_HELP_SEPARATOR, FINDING_KINDS, REQUIRE_SQL_ANALYSIS_KEY, SETTING_SNIPPET_INDENT, }; use crate::semantic_checks::models::{OptOutDiagnostic, SemanticFailure}; +use sqlbuild_core::constants::PROJECT_CONFIG_FILENAME; +use sqlbuild_core::constants::SETTINGS_SECTION; static FINDING_PATTERNS: LazyLock> = LazyLock::new(|| { let alternatives: Vec = FINDING_KINDS diff --git a/crates/sqlbuild-analysis/src/semantic_checks/constants.rs b/crates/sqlbuild-analysis/src/semantic_checks/constants.rs index 486b6f4bde..fc721ba5db 100644 --- a/crates/sqlbuild-analysis/src/semantic_checks/constants.rs +++ b/crates/sqlbuild-analysis/src/semantic_checks/constants.rs @@ -16,8 +16,6 @@ pub(crate) const TYPE_CODES: [&str; 10] = [ pub(crate) const MIN_ABBREVIATION_LENGTH: usize = 3; pub(crate) const MAX_EDIT_DISTANCE: usize = 2; pub(crate) const DISPLAY_LIMIT: usize = 10; -pub(crate) const PROJECT_CONFIG_FILENAME: &str = "sqlbuild_project.toml"; -pub(crate) const SETTINGS_SECTION: &str = "settings"; pub(crate) const REQUIRE_SQL_ANALYSIS_KEY: &str = "require_sql_analysis"; pub(crate) const SETTING_SNIPPET_INDENT: &str = " "; pub(crate) const ADDITIONAL_HELP_SEPARATOR: &str = "\n = help: "; diff --git a/crates/sqlbuild-analysis/src/snowflake_cost/constants.rs b/crates/sqlbuild-analysis/src/snowflake_cost/constants.rs index e941f13be5..67a32cb1f1 100644 --- a/crates/sqlbuild-analysis/src/snowflake_cost/constants.rs +++ b/crates/sqlbuild-analysis/src/snowflake_cost/constants.rs @@ -5,6 +5,6 @@ pub(crate) const DROP_QUERY_TYPE: &str = "DROP"; pub(crate) const CREATE_QUERY_TYPE: &str = "CREATE"; pub(crate) const DYNAMIC_TABLE_MODIFIER: &str = "DYNAMIC"; pub(crate) const CREATE_VARIANT_PREFIX: &str = "create"; -pub(crate) const QUERY_TAG_APP: &str = "sqlbuild"; +pub const QUERY_TAG_APP: &str = "sqlbuild"; pub(crate) const QUERY_TAG_APP_KEY: &str = "app"; pub(crate) const QUERY_TAG_VERSION_KEY: &str = "v"; diff --git a/crates/sqlbuild-analysis/src/snowflake_cost/mod.rs b/crates/sqlbuild-analysis/src/snowflake_cost/mod.rs index 1e64f3d116..f01181f853 100644 --- a/crates/sqlbuild-analysis/src/snowflake_cost/mod.rs +++ b/crates/sqlbuild-analysis/src/snowflake_cost/mod.rs @@ -1,5 +1,5 @@ pub(crate) mod _helpers; -pub(crate) mod constants; +pub mod constants; pub mod main; pub mod models; diff --git a/crates/sqlbuild-attachments/src/audits/_helpers/policies.rs b/crates/sqlbuild-attachments/src/audits/_helpers/policies.rs index c9df6e1bce..13750471b7 100644 --- a/crates/sqlbuild-attachments/src/audits/_helpers/policies.rs +++ b/crates/sqlbuild-attachments/src/audits/_helpers/policies.rs @@ -5,7 +5,7 @@ use crate::audits::models::PolicySource; const SEVERITIES: [&str; 2] = ["warn", "error"]; /// Python lists run scopes sorted. const RUN_SCOPES: [&str; 2] = ["delta_and_final", "final"]; -const SETTINGS_FILE: &str = "sqlbuild_project.toml"; +use sqlbuild_core::constants::PROJECT_CONFIG_FILENAME as SETTINGS_FILE; /// The severity text, or Python's error naming the unknown value. pub(crate) fn resolve_severity( diff --git a/crates/sqlbuild-config/src/command_connection/constants.rs b/crates/sqlbuild-config/src/command_connection/constants.rs index 969eeb4ed6..d8474dd099 100644 --- a/crates/sqlbuild-config/src/command_connection/constants.rs +++ b/crates/sqlbuild-config/src/command_connection/constants.rs @@ -9,7 +9,6 @@ pub const CONNECTION_SOURCE_KEY: &str = "source"; pub const DBT_PROFILE_CONNECTION_SOURCE: &str = "dbt_profile"; pub const DUCKDB_ADAPTER: &str = "duckdb"; pub const DUCKDB_MEMORY_DATABASE: &str = ":memory:"; -pub const PROJECT_CONFIG_FILENAME: &str = "sqlbuild_project.toml"; pub const LOCAL_CONFIG_FILENAME: &str = "sqlbuild_local.toml"; /// Connection keys a dbt profile connection routes on rather than connects with. pub const DBT_PROFILE_ROUTING_KEYS: [&str; 5] = diff --git a/crates/sqlbuild-config/src/command_connection/main/authored_warehouse.rs b/crates/sqlbuild-config/src/command_connection/main/authored_warehouse.rs index 450078ba96..2e88d58a4f 100644 --- a/crates/sqlbuild-config/src/command_connection/main/authored_warehouse.rs +++ b/crates/sqlbuild-config/src/command_connection/main/authored_warehouse.rs @@ -1,7 +1,8 @@ //! The authored warehouse a target sets for a command group. -use crate::command_connection::constants::{LOCAL_CONFIG_FILENAME, PROJECT_CONFIG_FILENAME}; +use crate::command_connection::constants::LOCAL_CONFIG_FILENAME; use crate::command_connection::models::AuthoredWarehouse; +use sqlbuild_core::constants::PROJECT_CONFIG_FILENAME; /// The local target's group warehouse, else the project target's, with the file that sets it. #[must_use] diff --git a/crates/sqlbuild-config/src/project/_helpers/defaults.rs b/crates/sqlbuild-config/src/project/_helpers/defaults.rs index fc20814c61..e6d118dc9c 100644 --- a/crates/sqlbuild-config/src/project/_helpers/defaults.rs +++ b/crates/sqlbuild-config/src/project/_helpers/defaults.rs @@ -13,12 +13,14 @@ use crate::project::_helpers::keys::{Place, reject_removed_virtual_keys, validat use crate::project::constants::{ BATCH_CONCURRENCY_KEY, CHANGE_POLICY_KEYS, CONTRACT_POLICIES, DEFAULTS_KEYS, DISABLED_VALUE, GLOB_SEGMENTS, INHERIT_VALUE, MATERIALIZATION_DEFAULT_KEYS, MAX_MICROBATCHES_KEY, - MODEL_HEADER_KEYS, MODELS_DIRECTORY_NAME, PROJECT_CONFIG_FILENAME, RETENTION_MATERIALIZATIONS, - TABLE_TYPES, TEMPLATE_OPEN_TOKEN, UNSUPPORTED_GLOB_MARKERS, ZERO_DAY_DURATION, + MODEL_HEADER_KEYS, MODELS_DIRECTORY_NAME, RETENTION_MATERIALIZATIONS, TABLE_TYPES, + TEMPLATE_OPEN_TOKEN, UNSUPPORTED_GLOB_MARKERS, }; use crate::project::errors::ConfigFailure; use crate::project::models::{Built, ConfigFile}; use crate::project::types::{Entries, Fields, Loaded}; +use sqlbuild_core::constants::PROJECT_CONFIG_FILENAME; +use sqlbuild_core::constants::ZERO_DAY_DURATION; /// A retention or table-type field: where it lives and whether `'inherit'` is accepted. #[derive(Clone, Copy, Debug, PartialEq, Eq)] diff --git a/crates/sqlbuild-config/src/project/_helpers/documents.rs b/crates/sqlbuild-config/src/project/_helpers/documents.rs index 827dfb0d08..80ff474882 100644 --- a/crates/sqlbuild-config/src/project/_helpers/documents.rs +++ b/crates/sqlbuild-config/src/project/_helpers/documents.rs @@ -24,13 +24,15 @@ use crate::project::_helpers::targets::{ }; use crate::project::_helpers::values::is_truthy; use crate::project::constants::{ - LEGACY_EVENT_EXPORTERS_KEY, LOCAL_CONFIG_FILENAME, PROJECT_CONFIG_FILENAME, - QUERY_CHANGE_TRACKING_KEY, REQUIRE_SQL_ANALYSIS_KEY, SETTINGS_SECTION, SQL_ANALYSIS_KEY, + LEGACY_EVENT_EXPORTERS_KEY, LOCAL_CONFIG_FILENAME, QUERY_CHANGE_TRACKING_KEY, + REQUIRE_SQL_ANALYSIS_KEY, SQL_ANALYSIS_KEY, }; use crate::project::errors::{ConfigFailure, FailureClass, LoadFailure, MessagePiece}; use crate::project::models::{Built, ConfigFile, ConfigPaths}; use crate::project::types::{Entries, Fields, Loaded}; use crate::toml::main::load_toml::load_toml; +use sqlbuild_core::constants::PROJECT_CONFIG_FILENAME; +use sqlbuild_core::constants::SETTINGS_SECTION; use std::path::Path; const PROJECT_FILE: ConfigFile = ConfigFile::Project; diff --git a/crates/sqlbuild-config/src/project/_helpers/sections.rs b/crates/sqlbuild-config/src/project/_helpers/sections.rs index 54de2a9029..e6d2fb5489 100644 --- a/crates/sqlbuild-config/src/project/_helpers/sections.rs +++ b/crates/sqlbuild-config/src/project/_helpers/sections.rs @@ -17,11 +17,12 @@ use crate::project::constants::{ REQUIRE_SQL_ANALYSIS_KEY, RUN_NAMESPACE_KEY, RUN_NAMESPACE_MAX_LENGTH, SCENARIO_KEYS, SETTINGS_KEYS, SNAPSHOT_FULL_REFRESH_POLICIES, SNAPSHOT_KEYS, SNAPSHOT_LIMIT_KEYS, SNAPSHOT_SCHEMA_CHANGE_POLICIES, SQL_ANALYSIS_KEY, UNACCOUNTED_PARTITION_POLICIES, - USD_PER_CREDIT_KEY, ZERO_DAY_DURATION, ZERO_FIXED_DURATION, ZERO_FLOAT, + USD_PER_CREDIT_KEY, ZERO_FIXED_DURATION, ZERO_FLOAT, }; use crate::project::errors::ConfigFailure; use crate::project::models::{Built, ConfigFile}; use crate::project::types::{Entries, Fields, Loaded}; +use sqlbuild_core::constants::ZERO_DAY_DURATION; fn place(file: ConfigFile, label: &str) -> Place<'_> { Place { file, label } diff --git a/crates/sqlbuild-config/src/project/constants.rs b/crates/sqlbuild-config/src/project/constants.rs index c932d0a782..373d44b4c7 100644 --- a/crates/sqlbuild-config/src/project/constants.rs +++ b/crates/sqlbuild-config/src/project/constants.rs @@ -1,8 +1,6 @@ //! Project configuration file names, keys and the vocabularies their values come from. -pub(crate) const PROJECT_CONFIG_FILENAME: &str = "sqlbuild_project.toml"; pub(crate) const LOCAL_CONFIG_FILENAME: &str = "sqlbuild_local.toml"; -pub(crate) const SETTINGS_SECTION: &str = "settings"; pub(crate) const SQL_ANALYSIS_KEY: &str = "sql_analysis"; pub(crate) const LEGACY_SQL_VALIDATION_KEY: &str = "sql_validation"; pub(crate) const REQUIRE_SQL_ANALYSIS_KEY: &str = "require_sql_analysis"; @@ -27,7 +25,6 @@ pub(crate) const CHANGE_POLICY_KEYS: [&str; 2] = [ON_SCHEMA_CHANGE_KEY, REPLAY_O pub(crate) const TEMPLATE_OPEN_TOKEN: &str = "${"; pub(crate) const MODELS_DIRECTORY_NAME: &str = "models"; pub(crate) const UNLIMITED_DIFF_FULL_ROWS: &str = "unlimited"; -pub(crate) const ZERO_DAY_DURATION: &str = "0d"; pub(crate) const ZERO_FIXED_DURATION: &str = "0s"; pub(crate) const DEFAULT_QUERY_ARTIFACT_TTL: &str = "24h"; pub(crate) const BOUNDED_REPLAY_PREFIX: &str = "bounded-"; diff --git a/crates/sqlbuild-config/src/project/main/explicit_references_help.rs b/crates/sqlbuild-config/src/project/main/explicit_references_help.rs index 43a8d02407..d87e6a3613 100644 --- a/crates/sqlbuild-config/src/project/main/explicit_references_help.rs +++ b/crates/sqlbuild-config/src/project/main/explicit_references_help.rs @@ -1,9 +1,8 @@ //! The help shown when explicit references reject an implicit reference. use crate::project::_helpers::help::{Setting, SettingValue, current_value_note, setting_help}; -use crate::project::constants::{ - ENFORCE_EXPLICIT_KEY, PROJECT_CONFIG_FILENAME, REFERENCES_SECTION, -}; +use crate::project::constants::{ENFORCE_EXPLICIT_KEY, REFERENCES_SECTION}; +use sqlbuild_core::constants::PROJECT_CONFIG_FILENAME; /// The current-value note, then help that relaxes the policy to allow `allowed`. #[must_use] diff --git a/crates/sqlbuild-config/src/project/main/microbatch_guidance.rs b/crates/sqlbuild-config/src/project/main/microbatch_guidance.rs index 56451a16b6..7125bb4c8f 100644 --- a/crates/sqlbuild-config/src/project/main/microbatch_guidance.rs +++ b/crates/sqlbuild-config/src/project/main/microbatch_guidance.rs @@ -1,10 +1,9 @@ //! The note and help shown when microbatch concurrency is off. use crate::project::_helpers::help::{Setting, SettingValue, current_value_note, setting_help}; -use crate::project::constants::{ - MICROBATCH_CONCURRENCY_KEY, MICROBATCH_CONCURRENCY_PURPOSE, PROJECT_CONFIG_FILENAME, - SETTINGS_SECTION, -}; +use crate::project::constants::{MICROBATCH_CONCURRENCY_KEY, MICROBATCH_CONCURRENCY_PURPOSE}; +use sqlbuild_core::constants::PROJECT_CONFIG_FILENAME; +use sqlbuild_core::constants::SETTINGS_SECTION; /// The current-value note and the help that enables microbatch concurrency. #[must_use] diff --git a/crates/sqlbuild-config/src/project/main/sql_analysis_off_guidance.rs b/crates/sqlbuild-config/src/project/main/sql_analysis_off_guidance.rs index b5169f0cd1..c369a30091 100644 --- a/crates/sqlbuild-config/src/project/main/sql_analysis_off_guidance.rs +++ b/crates/sqlbuild-config/src/project/main/sql_analysis_off_guidance.rs @@ -1,9 +1,9 @@ //! The note and help shown when SQL analysis is turned off. use crate::project::_helpers::help::{Setting, SettingValue, setting_help, setting_note}; -use crate::project::constants::{ - LOCAL_CONFIG_FILENAME, PROJECT_CONFIG_FILENAME, SETTINGS_SECTION, SQL_ANALYSIS_KEY, -}; +use crate::project::constants::{LOCAL_CONFIG_FILENAME, SQL_ANALYSIS_KEY}; +use sqlbuild_core::constants::PROJECT_CONFIG_FILENAME; +use sqlbuild_core::constants::SETTINGS_SECTION; /// The note naming the file that turns SQL analysis off and the help that turns it on for `purpose`. #[must_use] diff --git a/crates/sqlbuild-config/src/workspace/_helpers/skill_settings.rs b/crates/sqlbuild-config/src/workspace/_helpers/skill_settings.rs index 4c0961fa12..ebe7413f0b 100644 --- a/crates/sqlbuild-config/src/workspace/_helpers/skill_settings.rs +++ b/crates/sqlbuild-config/src/workspace/_helpers/skill_settings.rs @@ -6,10 +6,11 @@ use crate::models::ConfigValue; use crate::toml::main::load_toml::load_toml; use crate::workspace::_helpers::files::read_text; use crate::workspace::constants::{ - CLAUDE_TARGET, DEFAULT_SKILL_TARGETS, GIT_MARKER, OPENCODE_TARGET, PROJECT_CONFIG_FILENAME, - SKILL_ENTRY_FILE, SKILL_NAME, UTF8_BYTE_ORDER_MARK, VALID_SKILL_TARGETS, + CLAUDE_TARGET, DEFAULT_SKILL_TARGETS, GIT_MARKER, OPENCODE_TARGET, SKILL_ENTRY_FILE, + SKILL_NAME, UTF8_BYTE_ORDER_MARK, VALID_SKILL_TARGETS, }; use crate::workspace::errors::WorkspaceError; +use sqlbuild_core::constants::PROJECT_CONFIG_FILENAME; /// Project skill installation preferences (`SkillSettings`). pub(crate) struct SkillSettings { diff --git a/crates/sqlbuild-config/src/workspace/constants.rs b/crates/sqlbuild-config/src/workspace/constants.rs index 811834e0bf..3d328a8ef6 100644 --- a/crates/sqlbuild-config/src/workspace/constants.rs +++ b/crates/sqlbuild-config/src/workspace/constants.rs @@ -28,7 +28,6 @@ pub(crate) const VALID_SKILL_TARGETS: [&str; 3] = ["opencode", "claude", "agents pub(crate) const DEFAULT_SKILL_TARGETS: [&str; 2] = ["agents", "claude"]; pub(crate) const OPENCODE_TARGET: &str = "opencode"; pub(crate) const CLAUDE_TARGET: &str = "claude"; -pub(crate) const PROJECT_CONFIG_FILENAME: &str = "sqlbuild_project.toml"; pub(crate) const UTF8_BYTE_ORDER_MARK: char = '\u{feff}'; pub(crate) const GIT_MARKER: &str = ".git"; pub(crate) const PLAYGROUND_EXAMPLES: [(&str, &str); 4] = [ diff --git a/crates/sqlbuild-core/src/constants.rs b/crates/sqlbuild-core/src/constants.rs index cfb324eb8c..75a4f1cdc8 100644 --- a/crates/sqlbuild-core/src/constants.rs +++ b/crates/sqlbuild-core/src/constants.rs @@ -6,3 +6,29 @@ pub const CLOSE_PAREN: &str = ")"; pub const PANIC_MESSAGE: &str = "NativeCompilerError: native SQL compilation panicked"; /// The prefix every native internal failure carries, so the boundary raises `NativeCompilerError`. pub const NATIVE_FAILURE_PREFIX: &str = "NativeCompilerError: "; +/// File name of the project configuration at the project root. +pub const PROJECT_CONFIG_FILENAME: &str = "sqlbuild_project.toml"; +/// Project configuration section holding settings. +pub const SETTINGS_SECTION: &str = "settings"; +/// Settings key selecting how table builds are promoted. +pub const TABLE_PROMOTION_MODE_SETTING_KEY: &str = "table_promotion_mode"; +/// Table promotion mode that builds into a staging relation before swapping. +pub const STAGED_PROMOTION_MODE: &str = "staged"; +/// Project directory holding Python node modules. +pub const PYTHON_NODE_ROOT: &str = "python"; +/// Stem of a Python package's initializer module. +pub const PYTHON_INIT_MODULE_STEM: &str = "__init__"; +/// State table of model, seed and hook fingerprints. +pub const FINGERPRINT_TABLE_NAME: &str = "_sqlbuild_fingerprints"; +/// State table of observed source freshness. +pub const SOURCE_FRESHNESS_TABLE_NAME: &str = "_sqlbuild_source_freshness"; +/// State table of model migration events. +pub const MIGRATION_TABLE_NAME: &str = "_sqlbuild_migrations"; +/// State table of column migration events. +pub const COLUMN_MIGRATION_TABLE_NAME: &str = "_sqlbuild_column_migrations"; +/// State table of old-name view events. +pub const OLD_NAME_VIEW_TABLE_NAME: &str = "_sqlbuild_old_name_views"; +/// A zero-length cursor duration. +pub const ZERO_DAY_DURATION: &str = "0d"; +/// Node key of a common table expression in the polyglot SQL AST. +pub const CTE_AST_KIND: &str = "cte"; diff --git a/crates/sqlbuild-core/src/dbt_command/main/render_dbt_init_summary.rs b/crates/sqlbuild-core/src/dbt_command/main/render_dbt_init_summary.rs index 26d06a16a5..276a44ce2b 100644 --- a/crates/sqlbuild-core/src/dbt_command/main/render_dbt_init_summary.rs +++ b/crates/sqlbuild-core/src/dbt_command/main/render_dbt_init_summary.rs @@ -1,6 +1,7 @@ //! The summary written after `sqb dbt init` creates a twin project. use crate::cli_text::models::{CliDocument, CliStyle}; +use crate::constants::PROJECT_CONFIG_FILENAME; use crate::dbt_command::constants::FIELD_LABEL_WIDTH; use crate::dbt_command::models::DbtInitSummary; @@ -27,7 +28,7 @@ pub fn render_dbt_init_summary(summary: &DbtInitSummary, style: CliStyle) -> Str document.lines.push(format!( " 1. {} ({})", style.value("SQLBuild twin config"), - style.object_name("sqlbuild_project.toml") + style.object_name(PROJECT_CONFIG_FILENAME) )); document.lines.push( " This points SQLBuild at your dbt project and profile. Edit it if dbt paths" diff --git a/crates/sqlbuild-core/src/debug_report/constants.rs b/crates/sqlbuild-core/src/debug_report/constants.rs index 320f8a6ef8..b7afd7f981 100644 --- a/crates/sqlbuild-core/src/debug_report/constants.rs +++ b/crates/sqlbuild-core/src/debug_report/constants.rs @@ -1,6 +1,5 @@ //! `sqb debug` file names, masking and wording. -pub(crate) const PROJECT_CONFIG_FILENAME: &str = "sqlbuild_project.toml"; pub(crate) const LOCAL_CONFIG_FILENAME: &str = "sqlbuild_local.toml"; pub(crate) const LEGACY_PROJECT_CONFIG_FILENAME: &str = "sqlbuild_project.yml"; pub(crate) const LEGACY_LOCAL_CONFIG_FILENAME: &str = "sqlbuild_local.yml"; diff --git a/crates/sqlbuild-core/src/debug_report/main/debug_config_files.rs b/crates/sqlbuild-core/src/debug_report/main/debug_config_files.rs index 6ab96b510a..72a9c8e63c 100644 --- a/crates/sqlbuild-core/src/debug_report/main/debug_config_files.rs +++ b/crates/sqlbuild-core/src/debug_report/main/debug_config_files.rs @@ -2,9 +2,9 @@ use std::path::{Path, PathBuf}; +use crate::constants::PROJECT_CONFIG_FILENAME; use crate::debug_report::constants::{ LEGACY_LOCAL_CONFIG_FILENAME, LEGACY_PROJECT_CONFIG_FILENAME, LOCAL_CONFIG_FILENAME, - PROJECT_CONFIG_FILENAME, }; use crate::debug_report::models::DebugConfigFiles; diff --git a/crates/sqlbuild-core/src/init_command/constants.rs b/crates/sqlbuild-core/src/init_command/constants.rs index efd035371f..340fd783a4 100644 --- a/crates/sqlbuild-core/src/init_command/constants.rs +++ b/crates/sqlbuild-core/src/init_command/constants.rs @@ -19,9 +19,6 @@ pub const SCAFFOLD_DIRECTORIES: [&str; 15] = [ "audits/singular", ]; -/// The project configuration file name. -pub const PROJECT_FILE: &str = "sqlbuild_project.toml"; - /// The placeholder file kept in otherwise empty directories. pub const GITKEEP_FILE: &str = ".gitkeep"; diff --git a/crates/sqlbuild-core/src/init_command/main/render_init_summary.rs b/crates/sqlbuild-core/src/init_command/main/render_init_summary.rs index 300bf21589..6f40e2863f 100644 --- a/crates/sqlbuild-core/src/init_command/main/render_init_summary.rs +++ b/crates/sqlbuild-core/src/init_command/main/render_init_summary.rs @@ -1,6 +1,7 @@ //! The summary printed after a project is created. use crate::cli_text::models::{CliDocument, CliStyle}; +use crate::constants::PROJECT_CONFIG_FILENAME; use crate::init_command::constants::NEXT_STEPS; /// The created header, project and config fields, and the numbered next steps. @@ -10,7 +11,7 @@ pub fn render_init_summary(project_name: &str, style: CliStyle) -> String { document.header("SQLBuild project created", None); document.lines.push(String::new()); document.field("Project", (project_name, " ")); - document.field("Config", ("sqlbuild_project.toml", " ")); + document.field("Config", (PROJECT_CONFIG_FILENAME, " ")); document.lines.push(String::new()); document.section("Next steps"); document diff --git a/crates/sqlbuild-core/src/janitor_text/main/janitor_disabled_text.rs b/crates/sqlbuild-core/src/janitor_text/main/janitor_disabled_text.rs index 79fabfc34e..b87556f24f 100644 --- a/crates/sqlbuild-core/src/janitor_text/main/janitor_disabled_text.rs +++ b/crates/sqlbuild-core/src/janitor_text/main/janitor_disabled_text.rs @@ -1,6 +1,7 @@ //! Guidance when janitor is not enabled for the project. use crate::cli_text::models::CliStyle; +use crate::constants::PROJECT_CONFIG_FILENAME; use crate::janitor_text::constants::DISABLED_DETAIL; /// How to enable janitor and preview cleanup. @@ -11,7 +12,7 @@ pub fn janitor_disabled_text(style: CliStyle) -> String { After enabling, run janitor again to preview cleanup:\n {}\n", style.warning_strong("Janitor is disabled for this project."), style.warning(DISABLED_DETAIL), - style.object_name("sqlbuild_project.toml"), + style.object_name(PROJECT_CONFIG_FILENAME), style.accent("sqb janitor") ) } diff --git a/crates/sqlbuild-discovery/src/constants.rs b/crates/sqlbuild-discovery/src/constants.rs index 1c58b4febe..f1bb0e5c90 100644 --- a/crates/sqlbuild-discovery/src/constants.rs +++ b/crates/sqlbuild-discovery/src/constants.rs @@ -1,4 +1,4 @@ -//! Project layout names shared by native discovery, mirroring the Python discovery constants. +//! Project layout names shared by native discovery. /// Root of SQL model files. pub const MODELS_DIRECTORY: &str = "models"; @@ -41,8 +41,6 @@ pub const GROUPED_NAMED_DECLARATION_DIRECTORIES: [&str; 6] = [ pub const GLOBAL_NAMED_DECLARATION_DIRECTORIES: [&str; 3] = ["audits", "schemas", "hooks"]; /// Suffix of macro files. pub const PYTHON_FILE_SUFFIX: &str = ".py"; -/// The stem of package initialisers, which are not macro files. -pub const PYTHON_INIT_MODULE_STEM: &str = "__init__"; /// The project-wide audit role directory. pub const AUDIT_ROLE_DIRECTORY: &str = "audits"; /// The folder-local audit role directory inside a declaration group. diff --git a/crates/sqlbuild-discovery/src/declaration_files/_helpers/listing/module_paths.rs b/crates/sqlbuild-discovery/src/declaration_files/_helpers/listing/module_paths.rs index c2b151ceb5..c1c8242eb2 100644 --- a/crates/sqlbuild-discovery/src/declaration_files/_helpers/listing/module_paths.rs +++ b/crates/sqlbuild-discovery/src/declaration_files/_helpers/listing/module_paths.rs @@ -1,6 +1,7 @@ //! Which `*.py` paths below a root Python module discovery keeps. -use crate::declaration_files::constants::{INIT_MODULE_STEM, PRIVATE_PREFIX}; +use crate::declaration_files::constants::PRIVATE_PREFIX; +use sqlbuild_core::constants::PYTHON_INIT_MODULE_STEM as INIT_MODULE_STEM; /// Whether `relative_path` below `root` is a listed module: never `__init__`, and public if asked. pub(crate) fn is_listed_module(relative_path: &str, root: &str, public_only: bool) -> bool { diff --git a/crates/sqlbuild-discovery/src/declaration_files/constants.rs b/crates/sqlbuild-discovery/src/declaration_files/constants.rs index 35ae950b15..769949697b 100644 --- a/crates/sqlbuild-discovery/src/declaration_files/constants.rs +++ b/crates/sqlbuild-discovery/src/declaration_files/constants.rs @@ -2,7 +2,5 @@ /// The directory whose `*.py` files define warehouse Python functions. pub const PYTHON_FUNCTIONS_ROOT: &str = "functions/python"; -/// The stem of a package `__init__` module, which module discovery skips. -pub const INIT_MODULE_STEM: &str = "__init__"; /// The prefix of a path part that keeps a module out of public module discovery. pub const PRIVATE_PREFIX: char = '_'; diff --git a/crates/sqlbuild-discovery/src/declarations/_helpers/declaration_tree.rs b/crates/sqlbuild-discovery/src/declarations/_helpers/declaration_tree.rs index 48431079ca..191540fa06 100644 --- a/crates/sqlbuild-discovery/src/declarations/_helpers/declaration_tree.rs +++ b/crates/sqlbuild-discovery/src/declarations/_helpers/declaration_tree.rs @@ -2,8 +2,8 @@ use crate::constants::{ CANONICAL_AUTHORED_ROOTS, DECLARATION_GROUP_DIRECTORY, GLOBAL_DECLARATION_DIRECTORIES, - LOCAL_DECLARATION_DIRECTORIES, PYTHON_FILE_SUFFIX, PYTHON_INIT_MODULE_STEM, - SCOPED_DECLARATION_DIRECTORIES, SQL_FILE_SUFFIX, + LOCAL_DECLARATION_DIRECTORIES, PYTHON_FILE_SUFFIX, SCOPED_DECLARATION_DIRECTORIES, + SQL_FILE_SUFFIX, }; use crate::declarations::_helpers::declaration_groups::validate_declaration_groups; use crate::declarations::_helpers::paths::{directory_facts, name, parent, stem}; @@ -14,6 +14,7 @@ use crate::tree::main::directories::directories; use crate::tree::main::path_order::compare_posix_text; use crate::tree::main::rglob::rglob; use crate::tree::models::ProjectTree; +use sqlbuild_core::constants::PYTHON_INIT_MODULE_STEM; /// Where a declaration root sits and who owns it. struct DeclarationRoot<'a> { diff --git a/crates/sqlbuild-model-config/src/model_validation/_helpers/incremental.rs b/crates/sqlbuild-model-config/src/model_validation/_helpers/incremental.rs index 53c23c6e17..28c2055977 100644 --- a/crates/sqlbuild-model-config/src/model_validation/_helpers/incremental.rs +++ b/crates/sqlbuild-model-config/src/model_validation/_helpers/incremental.rs @@ -16,11 +16,12 @@ use crate::model_validation::constants::{ ENFORCED_CONTRACT, FUTURE_CURSOR_ACTIONS, INCREMENTAL_MATERIALIZATION, INCREMENTAL_STRATEGIES, MERGE_STRATEGY, ON_SCHEMA_CHANGE_EXAMPLE, ON_SCHEMA_CHANGE_POLICIES, REPLAY_ON_CHANGE_ACTIONS, REPLAY_ON_CHANGE_EXAMPLE, REPLAY_ON_CHANGE_VALID_VALUES, TABLE_MATERIALIZATION, - TIMESTAMP_CURSOR, VIEW_MATERIALIZATION, ZERO_DAY_DURATION, + TIMESTAMP_CURSOR, VIEW_MATERIALIZATION, }; use crate::model_validation::models::{ModelValidationFacts, ValidationStop}; use crate::model_validation::types::Check; use crate::types::{AuthoredNode, NodeKind}; +use sqlbuild_core::constants::ZERO_DAY_DURATION; const ON_SCHEMA_CHANGE_KEY: &str = "on_schema_change"; const REPLAY_ON_CHANGE_KEY: &str = "replay_on_change"; diff --git a/crates/sqlbuild-model-config/src/model_validation/constants.rs b/crates/sqlbuild-model-config/src/model_validation/constants.rs index 0249e5c8c0..341d1f0cf4 100644 --- a/crates/sqlbuild-model-config/src/model_validation/constants.rs +++ b/crates/sqlbuild-model-config/src/model_validation/constants.rs @@ -113,8 +113,6 @@ sqlbuild_project.toml:\n [settings]\n microbatch_concurren pub const SETTING_SNIPPET_INDENT: &str = " "; /// The duration-disabled cursor policy value. pub const CURSOR_POLICY_DISABLED: &str = "disabled"; -/// The zero-day cursor duration. -pub const ZERO_DAY_DURATION: &str = "0d"; /// The `batch_size` token that follows the cursor grain. pub const EFFECTIVE_BATCH_SIZE: &str = "effective"; /// The SQL wildcard column. diff --git a/crates/sqlbuild-model-config/src/model_validation/main/retention_override.rs b/crates/sqlbuild-model-config/src/model_validation/main/retention_override.rs index 44b20d041c..e60ee1765b 100644 --- a/crates/sqlbuild-model-config/src/model_validation/main/retention_override.rs +++ b/crates/sqlbuild-model-config/src/model_validation/main/retention_override.rs @@ -4,12 +4,11 @@ use crate::errors::ConfigError; use crate::model_validation::_helpers::config::model_header_help; use crate::model_validation::_helpers::config::text_if_string; use crate::model_validation::_helpers::durations::parse_duration; -use crate::model_validation::constants::{ - DISABLED_RETENTION, INHERIT_POLICY, RETENTION_KEY, ZERO_DAY_DURATION, -}; +use crate::model_validation::constants::{DISABLED_RETENTION, INHERIT_POLICY, RETENTION_KEY}; use crate::model_validation::errors::DurationNumberError; use crate::model_validation::models::RetentionOverride; use crate::types::{AuthoredNode, NodeKind}; +use sqlbuild_core::constants::ZERO_DAY_DURATION; /// Return the header retention override, or the error it raises. pub fn retention_override( diff --git a/crates/sqlbuild-pysource/src/rule_evidence/_helpers/paths.rs b/crates/sqlbuild-pysource/src/rule_evidence/_helpers/paths.rs index 2b87ef6f22..c929921954 100644 --- a/crates/sqlbuild-pysource/src/rule_evidence/_helpers/paths.rs +++ b/crates/sqlbuild-pysource/src/rule_evidence/_helpers/paths.rs @@ -7,8 +7,9 @@ use ruff_python_ast::Stmt; use crate::cpython_tree::models::CpythonNode; use crate::rule_evidence::constants::{ - INIT_FILE_NAME, INIT_MODULE_NAME, PYTHON_SUFFIX, RULES_DIRECTORY_NAME, STAR_IMPORT_NAME, + INIT_FILE_NAME, PYTHON_SUFFIX, RULES_DIRECTORY_NAME, STAR_IMPORT_NAME, }; +use sqlbuild_core::constants::PYTHON_INIT_MODULE_STEM as INIT_MODULE_NAME; /// `Path.resolve()`: symlinks resolved through the existing prefix, the rest kept lexically. pub(crate) fn resolve(path: &Path) -> PathBuf { diff --git a/crates/sqlbuild-pysource/src/rule_evidence/constants.rs b/crates/sqlbuild-pysource/src/rule_evidence/constants.rs index 831fdd337a..0b0a3ded5d 100644 --- a/crates/sqlbuild-pysource/src/rule_evidence/constants.rs +++ b/crates/sqlbuild-pysource/src/rule_evidence/constants.rs @@ -6,7 +6,6 @@ pub(crate) const RULE_CASE: &str = "RuleCase"; pub(crate) const RULE_KEYWORD: &str = "rule"; pub(crate) const TEST_CASE_KEYWORD: &str = "test_case"; pub(crate) const CONDITION_KEYWORD: &str = "condition"; -pub(crate) const INIT_MODULE_NAME: &str = "__init__"; pub(crate) const INIT_FILE_NAME: &str = "__init__.py"; pub(crate) const RULES_DIRECTORY_NAME: &str = "rules"; pub(crate) const TESTS_DIRECTORY_NAME: &str = "tests"; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/analysis_session/model_analyses.rs b/crates/sqlbuild-python/src/bindings/_helpers/analysis_session/model_analyses.rs index 69ce5af6c6..73b4c55997 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/analysis_session/model_analyses.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/analysis_session/model_analyses.rs @@ -13,6 +13,7 @@ use crate::bindings::_helpers::attachments::cursor_intrinsics::analysis_cursor_s use crate::bindings::_helpers::boundary::keyword_request::Request; use crate::bindings::_helpers::cache::native_store::current_code_identity; use crate::bindings::_helpers::compile_driver::analysis_cache::record_analysis_cache_metrics; +use crate::bindings::_helpers::semantic_shapes::shapes::{VALID_FROM_DEFAULT, VALID_TO_DEFAULT}; const NATIVE_MODULE: &str = "sqlbuild._native"; const ANALYSIS_MODELS_MODULE: &str = "sqlbuild._native"; @@ -308,8 +309,8 @@ fn model_rows<'py>( || (!entry.is_none() && entry.getattr("type_enforcement")?.is_truthy()?); let snapshot_columns: Option<(String, String)> = if get("materialized")?.eq(&snapshot)? { Some(( - or_default(&get("valid_from_column")?, "valid_from")?, - or_default(&get("valid_to_column")?, "valid_to")?, + or_default(&get("valid_from_column")?, VALID_FROM_DEFAULT)?, + or_default(&get("valid_to_column")?, VALID_TO_DEFAULT)?, )) } else { None diff --git a/crates/sqlbuild-python/src/bindings/_helpers/attachments/cursor_intrinsics.rs b/crates/sqlbuild-python/src/bindings/_helpers/attachments/cursor_intrinsics.rs index ad6064e7bc..262ad5285a 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/attachments/cursor_intrinsics.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/attachments/cursor_intrinsics.rs @@ -20,7 +20,11 @@ const INTEGER_ANALYSIS_BOUND: &str = "CAST(0 AS BIGINT)"; const TIMESTAMP_ANALYSIS_BOUND: &str = "CAST('2000-01-01 00:00:00' AS TIMESTAMP)"; const MODEL_SQL_CONTEXT: &str = "Model SQL"; const SQL_CONTEXT: &str = "SQL"; -const RESERVED_CURSOR_MARKERS: [&str; 2] = ["__SQB_CURSOR_START__", "__SQB_CURSOR_END__"]; +/// Executor placeholder replaced with a microbatch's start bound. +pub(crate) const MICROBATCH_START_SENTINEL: &str = "__SQB_CURSOR_START__"; +/// Executor placeholder replaced with a microbatch's end bound. +pub(crate) const MICROBATCH_END_SENTINEL: &str = "__SQB_CURSOR_END__"; +const RESERVED_CURSOR_MARKERS: [&str; 2] = [MICROBATCH_START_SENTINEL, MICROBATCH_END_SENTINEL]; /// The reserved executor bound markers no authored SQL may contain. pub(crate) fn reserved_cursor_markers() -> Vec { diff --git a/crates/sqlbuild-python/src/bindings/_helpers/command_runtime/connection.rs b/crates/sqlbuild-python/src/bindings/_helpers/command_runtime/connection.rs index 094d94104d..0530f4ec01 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/command_runtime/connection.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/command_runtime/connection.rs @@ -7,10 +7,11 @@ use pyo3::types::{PyDict, PyDictMethods, PyString}; use pyo3::{pyfunction, wrap_pyfunction}; use sqlbuild_config::command_connection::constants::{ CONNECTION_DATABASE_KEY, CONNECTION_SOURCE_KEY, DBT_PROFILE_CONNECTION_SOURCE, - DBT_PROFILE_ROUTING_KEYS, DUCKDB_ADAPTER, DUCKDB_MEMORY_DATABASE, PROJECT_CONFIG_FILENAME, + DBT_PROFILE_ROUTING_KEYS, DUCKDB_ADAPTER, DUCKDB_MEMORY_DATABASE, }; use sqlbuild_config::command_connection::main::missing_connection_error::missing_connection_error; use sqlbuild_config::command_connection::main::snowflake_key_warning::snowflake_key_warning; +use sqlbuild_core::constants::PROJECT_CONFIG_FILENAME; use crate::bindings::_helpers::command_runtime::runtime_errors::raised; use crate::bindings::_helpers::command_runtime::warehouses::{WarehouseInputs, apply_warehouse}; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compile_attachment/model_inputs.rs b/crates/sqlbuild-python/src/bindings/_helpers/compile_attachment/model_inputs.rs index 0bf6e22f21..a8a4f3bac9 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compile_attachment/model_inputs.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compile_attachment/model_inputs.rs @@ -22,7 +22,7 @@ const NATIVE_MODULE: &str = "sqlbuild._native"; const COMPILE_MODELS_MODULE: &str = "sqlbuild._native"; const COMPILE_ERRORS_MODULE: &str = "sqlbuild._native"; const DISCOVERY_MODELS_MODULE: &str = "sqlbuild._native"; -const PROJECT_CONFIG_FILENAME: &str = "sqlbuild_project.toml"; +use sqlbuild_core::constants::PROJECT_CONFIG_FILENAME; const SCOPE_MODELS_MODULE: &str = "sqlbuild._native"; const SCOPE_TYPES_MODULE: &str = "sqlbuild._native"; const COLLECTOR_MODULE: &str = "sqlbuild._native"; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compile_command/constants.rs b/crates/sqlbuild-python/src/bindings/_helpers/compile_command/constants.rs index 5d1a7f85fc..649bcd6d24 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compile_command/constants.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compile_command/constants.rs @@ -1,5 +1,9 @@ //! Compile reuse storage, fingerprint, environment and notice constants. +use sqlbuild_core::constants::PROJECT_CONFIG_FILENAME; + +use crate::bindings::_helpers::frontier::stage_capture::CAPTURE_DIR_ENV_VAR; + /// The directory under the compiler cache directory holding the native reuse store. pub(crate) const NATIVE_REUSE_DIRECTORY_NAME: &str = "project-reuse-v2"; /// The reuse directory an older release kept stored compiles in. @@ -11,14 +15,12 @@ pub(crate) const REUSE_DISABLE_VALUE: &str = "1"; pub(crate) const REUSE_LOGGER_NAME: &str = "sqlbuild.compile.reuse"; pub(crate) const TRACKED_ENVIRONMENT_PREFIXES: [&str; 2] = ["SQLBUILD_", "SQB_"]; /// Sorted; tracked-prefix variables that never invalidate a stored compile. -pub(crate) const UNTRACKED_ENVIRONMENT_NAMES: [&str; 2] = [ - "SQLBUILD_COMPILER_STAGE_CAPTURE_DIR", - "SQLBUILD_DISABLE_COMPILE_REUSE", -]; +pub(crate) const UNTRACKED_ENVIRONMENT_NAMES: [&str; 2] = + [CAPTURE_DIR_ENV_VAR, REUSE_DISABLE_ENV_VAR]; pub(crate) const MISSING_ENVIRONMENT_VALUE: &str = "\0missing"; pub(crate) const MISSING_FILE_DIGEST: &str = "\0missing"; pub(crate) const PROJECT_CONFIG_FILENAMES: [&str; 2] = - ["sqlbuild_project.toml", "sqlbuild_project.yml"]; + [PROJECT_CONFIG_FILENAME, "sqlbuild_project.yml"]; /// The `sqlbuild._native` attribute the reuse check reads its racy window from. pub(crate) const RACY_WINDOW_ATTRIBUTE: &str = "COMPILE_REUSE_RACY_WINDOW_NS"; /// The `sqlbuild._native` attribute the reuse check reads its excluded root directories from. diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compile_command/reuse_attempt.rs b/crates/sqlbuild-python/src/bindings/_helpers/compile_command/reuse_attempt.rs index c8271b4702..028e51ef7b 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compile_command/reuse_attempt.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compile_command/reuse_attempt.rs @@ -9,6 +9,7 @@ use pyo3::prelude::{ use pyo3::types::{PyDict, PyDictMethods, PyFrozenSet, PyList, PyString}; use pyo3::{IntoPyObjectExt, pyfunction, wrap_pyfunction}; use sqlbuild_cli::replay::main::write_launch_record::write_launch_record as write_record_file; +use sqlbuild_core::command_output::constants::INVOCATION_CONTEXT_ENV; use sqlbuild_cli::replay::constants::{LAUNCH_RECORD_DIRECTORY, LAUNCH_RECORD_FORMAT}; @@ -37,7 +38,6 @@ const REUSE_REQUEST: (&str, &str) = ("sqlbuild.cli.compile_reuse.models", "Compi const RUNTIME_IDENTITY_MODULE: &str = "sqlbuild.cli.compile_reuse._helpers.runtime_identity"; const STARTUP_INPUTS_MODULE: &str = "sqlbuild.cli.compile_reuse._helpers.startup_inputs"; const COMPUTE_LOG_TEE_MODULE: &str = "sqlbuild.runtime.compute_logs.classes.text_tee"; -const OUTPUT_CAPTURE_CONSTANTS: &str = "sqlbuild.runtime.output_capture.constants"; const HIT: &str = "hit"; const CHECKED_HIT: &str = "hit"; const CHECKED_BYPASS: &str = "bypass"; @@ -210,11 +210,7 @@ pub(crate) fn write_launch_record(attempt: &Bound<'_, PyAny>) -> PyResult<()> { return Ok(()); } let environment: Bound<'_, PyAny> = imported(py, "os", "environ")?; - if environment.contains(imported( - py, - OUTPUT_CAPTURE_CONSTANTS, - "INVOCATION_CONTEXT_ENV", - )?)? { + if environment.contains(INVOCATION_CONTEXT_ENV)? { return Ok(()); } let mut directory: Bound<'_, PyAny> = attempt.getattr("project_dir")?; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/config_validation.rs b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/config_validation.rs index 42557c8924..35a99e92ff 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/config_validation.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/config_validation.rs @@ -12,7 +12,7 @@ const SPEC_ERRORS_MODULE: &str = "sqlbuild._native"; const TARGET_CONFIG_MODULE: &str = "sqlbuild._native"; const ADAPTER_NAME_MODULE: &str = "sqlbuild._native"; const SESSION_WAREHOUSE_MODULE: &str = "sqlbuild.adapter.discovery.main._session_warehouse_support"; -const PROJECT_CONFIG_FILENAME: &str = "sqlbuild_project.toml"; +use sqlbuild_core::constants::PROJECT_CONFIG_FILENAME; const LOCAL_CONFIG_FILENAME: &str = "sqlbuild_local.toml"; const SQL_ANALYSIS_KEY: &str = "sql_analysis"; /// `WarehouseGroup` members in declaration order. diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/project.rs b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/project.rs index 5e566a6b29..b82f324eb0 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/project.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/project.rs @@ -26,8 +26,8 @@ const BUILD_INPUTS_MODULE: &str = "sqlbuild._native"; const ASSEMBLE_MODULE: &str = "sqlbuild._native"; const PROJECT_WITH_MODULE: &str = "sqlbuild._native"; const PROMOTION_MODULE: &str = "sqlbuild._native"; -const PROJECT_CONFIG_FILENAME: &str = "sqlbuild_project.toml"; -const TABLE_PROMOTION_MODE_SETTING_KEY: &str = "table_promotion_mode"; +use sqlbuild_core::constants::PROJECT_CONFIG_FILENAME; +use sqlbuild_core::constants::TABLE_PROMOTION_MODE_SETTING_KEY; const LOCAL_CONFIG_FILENAME: &str = "sqlbuild_local.toml"; const ANALYSIS_SELECTION_MODULE: &str = "sqlbuild._native"; const ADAPTER_NAME_MODULE: &str = "sqlbuild._native"; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/targets.rs b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/targets.rs index f0b8d9983b..3713daa5ec 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/targets.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/targets.rs @@ -45,7 +45,7 @@ const CONNECTION_IDENTITY_KEYS: [&str; 7] = [ const EXPLICIT_TARGET_ADAPTERS: [&str; 3] = ["snowflake", "bigquery", "databricks"]; const LOADER_NAMESPACE_LABEL: &str = "managed loader target namespace"; const DEFAULTS_HELP_PURPOSE: &str = "to set project-wide defaults"; -const PROJECT_CONFIG_FILENAME: &str = "sqlbuild_project.toml"; +use sqlbuild_core::constants::PROJECT_CONFIG_FILENAME; const DEFAULTS_SECTION: &str = "defaults"; /// One optional `(database, schema)` namespace. diff --git a/crates/sqlbuild-python/src/bindings/_helpers/contracts/project_config.rs b/crates/sqlbuild-python/src/bindings/_helpers/contracts/project_config.rs index c08e048f41..b2a1074e97 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/contracts/project_config.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/contracts/project_config.rs @@ -16,12 +16,13 @@ use sqlbuild_config::project::main::resolve_config_file::resolve_config_file; use sqlbuild_config::project::models::{Built, ConfigFile, ConfigPaths}; use crate::bindings::_helpers::boundary::config_values::config_value_to_python; +use sqlbuild_core::constants::PROJECT_CONFIG_FILENAME; const MODELS_MODULE: &str = "sqlbuild._native"; const ERRORS_MODULE: &str = "sqlbuild._native"; const PROJECT_CONFIG_ERROR: &str = "ProjectConfigError"; const FILE_NAMES: [&str; 4] = [ - "sqlbuild_project.toml", + PROJECT_CONFIG_FILENAME, "sqlbuild_project.yml", "sqlbuild_local.toml", "sqlbuild_local.yml", diff --git a/crates/sqlbuild-python/src/bindings/_helpers/cursor/conversion.rs b/crates/sqlbuild-python/src/bindings/_helpers/cursor/conversion.rs index 0bbace1b4c..6d4b909315 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/cursor/conversion.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/cursor/conversion.rs @@ -22,7 +22,8 @@ use crate::bindings::_helpers::cursor::values::{ pub(crate) const MODELS_MODULE: &str = "sqlbuild._native"; const EXCEPTIONS_MODULE: &str = "sqlbuild.cursor_algebra.exceptions"; const PLANNER_TYPES_MODULE: &str = "sqlbuild._native"; -const GRAIN_NAMES: [(&str, Grain); 6] = [ +/// Cursor grain names, finest first. +pub(crate) const GRAIN_NAMES: [(&str, Grain); 6] = [ ("second", Grain::Second), ("minute", Grain::Minute), ("hour", Grain::Hour), diff --git a/crates/sqlbuild-python/src/bindings/_helpers/dbt_command/auto_init.rs b/crates/sqlbuild-python/src/bindings/_helpers/dbt_command/auto_init.rs index 8aa8a8520d..6d67f1d171 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/dbt_command/auto_init.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/dbt_command/auto_init.rs @@ -17,7 +17,7 @@ use crate::bindings::_helpers::dbt_command::support::{color_enabled, flag, path, const NATIVE_RECORDS_MODULE: &str = "sqlbuild._native"; const NATIVE_MODULE: &str = "sqlbuild._native"; -const PROJECT_CONFIG_FILENAME: &str = "sqlbuild_project.toml"; +use sqlbuild_core::constants::PROJECT_CONFIG_FILENAME; const PROGRESS_REPORTER_MODULE: &str = "sqlbuild._native"; fn project_config_found(directory: &Bound<'_, PyAny>) -> PyResult { diff --git a/crates/sqlbuild-python/src/bindings/_helpers/dbt_command/init_project.rs b/crates/sqlbuild-python/src/bindings/_helpers/dbt_command/init_project.rs index ae69b05319..da84c3fb43 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/dbt_command/init_project.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/dbt_command/init_project.rs @@ -25,7 +25,7 @@ const NATIVE_RECORDS_MODULE: &str = "sqlbuild._native"; const NATIVE_MODULE: &str = "sqlbuild._native"; const DBT_EXCEPTIONS: &str = "sqlbuild.integrations.dbt.exceptions"; const PROFILE_RENDER_MODULE: &str = "sqlbuild.integrations.dbt._helpers.profile.render"; -const PROJECT_FILE: &str = "sqlbuild_project.toml"; +use sqlbuild_core::constants::PROJECT_CONFIG_FILENAME as PROJECT_FILE; const DEFAULT_OUTPUT_DIR: &str = "sqlbuild_project"; const PROFILES_FILE: &str = "profiles.yml"; const PROFILES_DIR_ENV_VAR: &str = "DBT_PROFILES_DIR"; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/discovery/imported_python_nodes.rs b/crates/sqlbuild-python/src/bindings/_helpers/discovery/imported_python_nodes.rs index b6f1b12efc..b2fb22082c 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/discovery/imported_python_nodes.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/discovery/imported_python_nodes.rs @@ -14,7 +14,7 @@ use crate::bindings::_helpers::discovery::imported_modules::{ }; use crate::bindings::_helpers::discovery::imported_providers; -const PYTHON_NODE_ROOT: &str = "python"; +use sqlbuild_core::constants::PYTHON_NODE_ROOT; const NODE_ERROR: &str = "PythonNodeDiscoveryError"; const AUDIT_CASE: (&str, &str) = ("sqlbuild.python_nodes.models", "AuditCase"); const READ_FACTORY: (&str, &str) = ( diff --git a/crates/sqlbuild-python/src/bindings/_helpers/discovery/scope_snapshot.rs b/crates/sqlbuild-python/src/bindings/_helpers/discovery/scope_snapshot.rs index a9edcb12ad..cf869ed15b 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/discovery/scope_snapshot.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/discovery/scope_snapshot.rs @@ -28,7 +28,7 @@ const NATIVE_RECORDS_MODULE: &str = "sqlbuild._native"; const MODELS_MODULE: &str = "sqlbuild._native"; const CONTRACT_MODELS_MODULE: &str = "sqlbuild._native"; -const PROJECT_CONFIG_FILENAME: &str = "sqlbuild_project.toml"; +use sqlbuild_core::constants::PROJECT_CONFIG_FILENAME; const LOCAL_CONFIG_FILENAME: &str = "sqlbuild_local.toml"; const FALLBACK_PROJECT_NAME: &str = "project"; const FALLBACK_ADAPTER: &str = "duckdb"; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/exported_constants/constants.rs b/crates/sqlbuild-python/src/bindings/_helpers/exported_constants/constants.rs new file mode 100644 index 0000000000..6cb8ba52a8 --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/exported_constants/constants.rs @@ -0,0 +1,135 @@ +//! Native constants that stay-Python callers read from `sqlbuild._native`, each defined once in Rust. + +use fensu_policy::lifecycle::constants::CUSTOM_HOST_PROTOCOL_VERSION; +use sqlbuild_analysis::contracts::constants::CONTRACT_ENFORCED; +use sqlbuild_analysis::fingerprint_storage::constants::{ + COLUMN_DEFINITION_B64, COLUMN_DEFINITION_HASH, COLUMN_METADATA_JSON_B64, COLUMN_NODE_NAME, + COLUMN_NODE_TYPE, COLUMN_RUN_ID, COLUMN_SCHEMA_FINGERPRINT, COLUMN_TARGET_DATABASE, + COLUMN_TARGET_NAME, COLUMN_TARGET_SCHEMA, COLUMN_TIMESTAMP, COLUMN_VERSION_HASH, + NODE_TYPE_MODEL, NODE_TYPE_SEED, +}; +use sqlbuild_analysis::planner_changes::constants::{ + AUDIT_GATE_METADATA_KEY, MIGRATION_FINGERPRINT_METADATA_KEY, +}; +use sqlbuild_analysis::snowflake_cost::constants::QUERY_TAG_APP; +use sqlbuild_cli::compute_logs::constants::{ + COMPUTE_LOG_FORMAT_VERSION, DEFAULT_RETENTION_COUNT, MAX_READ_BYTES, +}; +use sqlbuild_cli::launcher::constants::LAUNCH_RECORD_OPTION; +use sqlbuild_core::command_output::constants::{ + COMMAND_OUTPUT_LOSS_RECORD_TYPE, COMMAND_OUTPUT_RECORD_TYPE, CURRENT_SCHEMA_VERSION, + INVOCATION_CONTEXT_ENV, +}; +use sqlbuild_core::constants::{ + COLUMN_MIGRATION_TABLE_NAME, CTE_AST_KIND, FINGERPRINT_TABLE_NAME, MIGRATION_TABLE_NAME, + OLD_NAME_VIEW_TABLE_NAME, PROJECT_CONFIG_FILENAME, PYTHON_INIT_MODULE_STEM, PYTHON_NODE_ROOT, + SETTINGS_SECTION, SOURCE_FRESHNESS_TABLE_NAME, STAGED_PROMOTION_MODE, + TABLE_PROMOTION_MODE_SETTING_KEY, ZERO_DAY_DURATION, +}; +use sqlbuild_rules::constants::MIN_CUSTOM_RULE_TEST_CASES_KEY; +use sqlbuild_rules::custom_host_pool::constants::CUSTOM_RULE_PROJECT_SUBJECT; +use sqlbuild_state::execution_history::constants::{ + DEFAULT_PAGE_LIMIT, MAX_PAGE_LIMIT, SCHEMA_VERSION, +}; + +use crate::bindings::_helpers::attachments::cursor_intrinsics::{ + MICROBATCH_END_SENTINEL, MICROBATCH_START_SENTINEL, +}; +use crate::bindings::_helpers::compile_command::constants::{ + REUSE_DISABLE_ENV_VAR, TRACKED_ENVIRONMENT_PREFIXES, UNTRACKED_ENVIRONMENT_NAMES, +}; +use crate::bindings::_helpers::frontier::stage_capture::CAPTURE_DIR_ENV_VAR; +use crate::bindings::_helpers::result_output::event_writer::RESULT_PATH_ENV; +use crate::bindings::_helpers::rules::constants::{ + CUSTOM_HOST_LAUNCH_MODULE, CUSTOM_HOST_RUNTIME_VERSION, +}; +use crate::bindings::_helpers::semantic_shapes::shapes::{VALID_FROM_DEFAULT, VALID_TO_DEFAULT}; +use crate::bindings::_helpers::state_rows::constants::MIGRATION_WRITE_ATTEMPTS; + +/// Text constants exported under their Python names. +pub(crate) const EXPORTED_TEXTS: &[(&str, &str)] = &[ + ("MICROBATCH_START_SENTINEL", MICROBATCH_START_SENTINEL), + ("MICROBATCH_END_SENTINEL", MICROBATCH_END_SENTINEL), + ( + "MIGRATION_FINGERPRINT_METADATA_KEY", + MIGRATION_FINGERPRINT_METADATA_KEY, + ), + ("SNAPSHOT_DEFAULT_VALID_FROM_COLUMN", VALID_FROM_DEFAULT), + ("SNAPSHOT_DEFAULT_VALID_TO_COLUMN", VALID_TO_DEFAULT), + ("PROJECT_CONFIG_FILENAME", PROJECT_CONFIG_FILENAME), + ("PYTHON_INIT_MODULE_STEM", PYTHON_INIT_MODULE_STEM), + ("PYTHON_NODE_ROOT", PYTHON_NODE_ROOT), + ("SETTINGS_SECTION", SETTINGS_SECTION), + ( + "TABLE_PROMOTION_MODE_SETTING_KEY", + TABLE_PROMOTION_MODE_SETTING_KEY, + ), + ("STAGED_PROMOTION_MODE", STAGED_PROMOTION_MODE), + ("STAGE_CAPTURE_DIR_ENV_VAR", CAPTURE_DIR_ENV_VAR), + ("COMPILE_REUSE_DISABLE_ENV_VAR", REUSE_DISABLE_ENV_VAR), + ("SQLBUILD_QUERY_TAG_APP", QUERY_TAG_APP), + ("COMMAND_OUTPUT_RECORD_TYPE", COMMAND_OUTPUT_RECORD_TYPE), + ( + "COMMAND_OUTPUT_LOSS_RECORD_TYPE", + COMMAND_OUTPUT_LOSS_RECORD_TYPE, + ), + ("INVOCATION_CONTEXT_ENV", INVOCATION_CONTEXT_ENV), + ("INTEGRATION_RESULT_PATH_ENV", RESULT_PATH_ENV), + ("FINGERPRINT_TABLE_NAME", FINGERPRINT_TABLE_NAME), + ("SOURCE_FRESHNESS_TABLE_NAME", SOURCE_FRESHNESS_TABLE_NAME), + ("MIGRATION_TABLE_NAME", MIGRATION_TABLE_NAME), + ("COLUMN_MIGRATION_TABLE_NAME", COLUMN_MIGRATION_TABLE_NAME), + ("OLD_NAME_VIEW_TABLE_NAME", OLD_NAME_VIEW_TABLE_NAME), + ("COLUMN_NODE_TYPE", COLUMN_NODE_TYPE), + ("COLUMN_NODE_NAME", COLUMN_NODE_NAME), + ("COLUMN_TARGET_DATABASE", COLUMN_TARGET_DATABASE), + ("COLUMN_TARGET_SCHEMA", COLUMN_TARGET_SCHEMA), + ("COLUMN_TARGET_NAME", COLUMN_TARGET_NAME), + ("COLUMN_RUN_ID", COLUMN_RUN_ID), + ("COLUMN_DEFINITION_HASH", COLUMN_DEFINITION_HASH), + ("COLUMN_VERSION_HASH", COLUMN_VERSION_HASH), + ("COLUMN_SCHEMA_FINGERPRINT", COLUMN_SCHEMA_FINGERPRINT), + ("COLUMN_DEFINITION_B64", COLUMN_DEFINITION_B64), + ("COLUMN_METADATA_JSON_B64", COLUMN_METADATA_JSON_B64), + ("COLUMN_TIMESTAMP", COLUMN_TIMESTAMP), + ("NODE_TYPE_MODEL", NODE_TYPE_MODEL), + ("NODE_TYPE_SEED", NODE_TYPE_SEED), + ("AUDIT_GATE_METADATA_KEY", AUDIT_GATE_METADATA_KEY), + ("CUSTOM_HOST_RUNTIME_VERSION", CUSTOM_HOST_RUNTIME_VERSION), + ("CUSTOM_HOST_LAUNCH_MODULE", CUSTOM_HOST_LAUNCH_MODULE), + ("CUSTOM_RULE_PROJECT_SUBJECT", CUSTOM_RULE_PROJECT_SUBJECT), + ("CONTRACT_ENFORCED", CONTRACT_ENFORCED), + ("LAUNCH_RECORD_OPTION", LAUNCH_RECORD_OPTION), + ("ZERO_DAY_CURSOR_DURATION", ZERO_DAY_DURATION), + ("AST_CTE_KIND", CTE_AST_KIND), + ("MIN_CUSTOM_RULE_TEST_CASES", MIN_CUSTOM_RULE_TEST_CASES_KEY), +]; +/// Integer constants exported under their Python names. +pub(crate) const EXPORTED_INTEGERS: &[(&str, i64)] = &[ + ("DEFAULT_PAGE_LIMIT", DEFAULT_PAGE_LIMIT), + ("MAX_PAGE_LIMIT", MAX_PAGE_LIMIT), + ("CURRENT_EVENT_LOG_SCHEMA_VERSION", SCHEMA_VERSION), + ("CURRENT_RUN_STORAGE_SCHEMA_VERSION", SCHEMA_VERSION), + ("COMPUTE_LOG_FORMAT_VERSION", COMPUTE_LOG_FORMAT_VERSION), + ("DEFAULT_RETENTION_COUNT", DEFAULT_RETENTION_COUNT as i64), + ("MAX_READ_BYTES", MAX_READ_BYTES as i64), + ( + "CURRENT_COMMAND_OUTPUT_SCHEMA_VERSION", + CURRENT_SCHEMA_VERSION, + ), + ("MIGRATION_WRITE_ATTEMPTS", MIGRATION_WRITE_ATTEMPTS), + ( + "CUSTOM_HOST_PROTOCOL_VERSION", + CUSTOM_HOST_PROTOCOL_VERSION as i64, + ), +]; +/// Text tuples exported under their Python names. +pub(crate) const EXPORTED_TEXT_TUPLES: &[(&str, &[&str])] = &[( + "STORE_TRACKED_ENVIRONMENT_PREFIXES", + &TRACKED_ENVIRONMENT_PREFIXES, +)]; +/// Text sets exported as frozensets under their Python names. +pub(crate) const EXPORTED_TEXT_SETS: &[(&str, &[&str])] = &[( + "STORE_UNTRACKED_ENVIRONMENT_NAMES", + &UNTRACKED_ENVIRONMENT_NAMES, +)]; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/exported_constants/mod.rs b/crates/sqlbuild-python/src/bindings/_helpers/exported_constants/mod.rs new file mode 100644 index 0000000000..e9c88e14e1 --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/exported_constants/mod.rs @@ -0,0 +1,2 @@ +pub(crate) mod constants; +pub(crate) mod registration; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/exported_constants/registration.rs b/crates/sqlbuild-python/src/bindings/_helpers/exported_constants/registration.rs new file mode 100644 index 0000000000..bd788edf76 --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/exported_constants/registration.rs @@ -0,0 +1,32 @@ +//! Register the exported native constants and the cursor grain order on `sqlbuild._native`. + +use pyo3::prelude::{Bound, PyAnyMethods, PyModule, PyModuleMethods, PyResult}; +use pyo3::types::{PyDict, PyDictMethods, PyFrozenSet, PyTuple}; + +use crate::bindings::_helpers::cursor::conversion::GRAIN_NAMES; +use crate::bindings::_helpers::exported_constants::constants::{ + EXPORTED_INTEGERS, EXPORTED_TEXT_SETS, EXPORTED_TEXT_TUPLES, EXPORTED_TEXTS, +}; + +/// Registers after the native enums, since `GRAIN_ORDER` is keyed by `CursorGrain` members. +pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { + let py = module.py(); + for (name, value) in EXPORTED_TEXTS { + module.add(*name, *value)?; + } + for (name, value) in EXPORTED_INTEGERS { + module.add(*name, *value)?; + } + for (name, values) in EXPORTED_TEXT_TUPLES { + module.add(*name, PyTuple::new(py, values.iter())?)?; + } + for (name, values) in EXPORTED_TEXT_SETS { + module.add(*name, PyFrozenSet::new(py, values.iter())?)?; + } + let grain = module.getattr("CursorGrain")?; + let order = PyDict::new(py); + for (name, rank) in GRAIN_NAMES { + order.set_item(grain.call1((name,))?, rank as usize)?; + } + module.add("GRAIN_ORDER", order) +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/fingerprints/storage_io.rs b/crates/sqlbuild-python/src/bindings/_helpers/fingerprints/storage_io.rs index 05ce3603dc..b4b8774651 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/fingerprints/storage_io.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/fingerprints/storage_io.rs @@ -20,7 +20,7 @@ use crate::bindings::_helpers::fingerprints::storage_sql::{ fingerprint_insert_batches, fingerprint_insert_sql, }; -const FINGERPRINT_TABLE_NAME: &str = "_sqlbuild_fingerprints"; +use sqlbuild_core::constants::FINGERPRINT_TABLE_NAME; const MODELS_MODULE: &str = "sqlbuild._native"; const ADAPTER_TYPES_MODULE: &str = "sqlbuild.adapter.contract.types"; const TIME_MODULE: &str = "time"; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/freshness/freshness_command.rs b/crates/sqlbuild-python/src/bindings/_helpers/freshness/freshness_command.rs index bfaff6a081..cb2627250b 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/freshness/freshness_command.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/freshness/freshness_command.rs @@ -29,7 +29,7 @@ const GRAPH_SELECTION_MODULE: &str = "sqlbuild._native"; const UPSTREAM_MODULE: &str = "sqlbuild._native"; const COMPILE_TYPES_MODULE: &str = "sqlbuild.compiler.compile.types"; const RELATION_LOOKUP_MODULE: &str = "sqlbuild.adapter.relations.main.relation_lookup"; -const STATE_TABLE_NAME: &str = "_sqlbuild_source_freshness"; +use sqlbuild_core::constants::SOURCE_FRESHNESS_TABLE_NAME as STATE_TABLE_NAME; const STALE_REQUIRES_STATE_MESSAGE: &str = "freshness --fail-on-stale requires --state"; const STALE_REQUIRES_STATE_CODE: &str = "C238"; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/freshness/state_io.rs b/crates/sqlbuild-python/src/bindings/_helpers/freshness/state_io.rs index 9ebc788dde..e52cb787e4 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/freshness/state_io.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/freshness/state_io.rs @@ -3,7 +3,8 @@ use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyModule, PyModuleMethods, PyResult, Python}; use pyo3::types::{PyDict, PyDictMethods, PyFrozenSet}; use pyo3::{pyfunction, wrap_pyfunction}; -use sqlbuild_state::freshness::constants::{REQUIRED_STATE_COLUMNS, SOURCE_FRESHNESS_TABLE_NAME}; +use sqlbuild_core::constants::SOURCE_FRESHNESS_TABLE_NAME; +use sqlbuild_state::freshness::constants::REQUIRED_STATE_COLUMNS; use sqlbuild_state::freshness::main::read_latest_sql::read_latest_sql; use sqlbuild_state::freshness::main::relevant_sources_sql::relevant_sources_sql; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/frontier/stage_capture.rs b/crates/sqlbuild-python/src/bindings/_helpers/frontier/stage_capture.rs index 65274bb9de..ab4f013520 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/frontier/stage_capture.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/frontier/stage_capture.rs @@ -13,7 +13,7 @@ use crate::bindings::_helpers::compile_driver::python_calls::{ }; use crate::bindings::_helpers::frontier::encoder::{CaptureEncoder, emit_error}; -const CAPTURE_DIR_ENV_VAR: &str = "SQLBUILD_COMPILER_STAGE_CAPTURE_DIR"; +pub(crate) const CAPTURE_DIR_ENV_VAR: &str = "SQLBUILD_COMPILER_STAGE_CAPTURE_DIR"; const CAPTURE_SUFFIX: &str = ".json"; const PLAN_CAPTURE_DIR_ENV_VAR: &str = "SQLBUILD_PLANNER_CAPTURE_DIR"; const PLAN_CAPTURE_NAME: &str = "plan_output"; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/init_command/command.rs b/crates/sqlbuild-python/src/bindings/_helpers/init_command/command.rs index 4b276a57b5..9b1c3c8348 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/init_command/command.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/init_command/command.rs @@ -8,7 +8,7 @@ use pyo3::prelude::{ use pyo3::types::{PyDict, PyDictMethods, PyString}; use pyo3::{pyfunction, wrap_pyfunction}; use sqlbuild_core::cli_text::models::CliStyle; -use sqlbuild_core::init_command::constants::PROJECT_FILE; +use sqlbuild_core::constants::PROJECT_CONFIG_FILENAME as PROJECT_FILE; use sqlbuild_core::init_command::main::init_project_name::init_project_name; use sqlbuild_core::init_command::main::init_project_toml::init_project_toml; use sqlbuild_core::init_command::main::render_init_summary::render_init_summary; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/init_command/scaffold.rs b/crates/sqlbuild-python/src/bindings/_helpers/init_command/scaffold.rs index 4a80dff91d..5aaaaab000 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/init_command/scaffold.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/init_command/scaffold.rs @@ -5,7 +5,8 @@ use std::io::{self, ErrorKind}; use std::path::Path; use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyErr, PyResult, Python}; -use sqlbuild_core::init_command::constants::{GITKEEP_FILE, PROJECT_FILE, SCAFFOLD_DIRECTORIES}; +use sqlbuild_core::constants::PROJECT_CONFIG_FILENAME as PROJECT_FILE; +use sqlbuild_core::init_command::constants::{GITKEEP_FILE, SCAFFOLD_DIRECTORIES}; /// `OSError(errno, strerror, filename)`, which Python narrows to its errno subclass. fn os_error(py: Python<'_>, error: &io::Error, path: &Path) -> PyErr { diff --git a/crates/sqlbuild-python/src/bindings/_helpers/janitor/janitor_facts.rs b/crates/sqlbuild-python/src/bindings/_helpers/janitor/janitor_facts.rs index c3af25ab1a..2b6a791f75 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/janitor/janitor_facts.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/janitor/janitor_facts.rs @@ -27,8 +27,8 @@ const LOG_DEBUG_EVENT: (&str, &str) = ( ); const READ_LATEST_FINGERPRINTS: (&str, &str) = ("sqlbuild._native", "read_latest_fingerprints"); const EXECUTION_LOGGER: &str = "sqlbuild.execution"; -const FINGERPRINT_TABLE_NAME: &str = "_sqlbuild_fingerprints"; -const SOURCE_FRESHNESS_TABLE_NAME: &str = "_sqlbuild_source_freshness"; +use sqlbuild_core::constants::FINGERPRINT_TABLE_NAME; +use sqlbuild_core::constants::SOURCE_FRESHNESS_TABLE_NAME; /// The adapter and its open connection. pub(crate) struct Warehouse<'py> { diff --git a/crates/sqlbuild-python/src/bindings/_helpers/janitor/janitor_old_names.rs b/crates/sqlbuild-python/src/bindings/_helpers/janitor/janitor_old_names.rs index 3017017b5d..083173283f 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/janitor/janitor_old_names.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/janitor/janitor_old_names.rs @@ -22,8 +22,8 @@ const IS_COMPATIBILITY_VIEW: (&str, &str) = ( ); const READ_HISTORIES: (&str, &str) = ("sqlbuild._native", "read_old_name_view_histories"); const STORED_COLUMNS: (&str, &str) = ("sqlbuild._native", "stored_old_name_view_columns"); -const OLD_NAME_VIEW_TABLE_NAME: &str = "_sqlbuild_old_name_views"; -const MIGRATION_TABLE_NAME: &str = "_sqlbuild_migrations"; +use sqlbuild_core::constants::MIGRATION_TABLE_NAME; +use sqlbuild_core::constants::OLD_NAME_VIEW_TABLE_NAME; const MIGRATE_FROM_CONFIG_KEY: &str = "migrate_from"; /// Old-name view planning and the old names it owns. diff --git a/crates/sqlbuild-python/src/bindings/_helpers/mod.rs b/crates/sqlbuild-python/src/bindings/_helpers/mod.rs index 85851a32d6..7163558ad6 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/mod.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/mod.rs @@ -40,6 +40,7 @@ pub(crate) mod diff_report; pub(crate) mod discovery; pub(crate) mod event_exporting; pub(crate) mod execution_history; +pub(crate) mod exported_constants; pub(crate) mod fingerprints; pub(crate) mod format_command; pub(crate) mod freshness; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/plan_output/cursor_details.rs b/crates/sqlbuild-python/src/bindings/_helpers/plan_output/cursor_details.rs index 1f5672ff48..fbf071b610 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/plan_output/cursor_details.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/plan_output/cursor_details.rs @@ -3,6 +3,7 @@ use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult, Python}; use crate::bindings::_helpers::compile_driver::python_calls::imported; +use crate::bindings::_helpers::cursor::conversion::GRAIN_NAMES; use crate::bindings::_helpers::plan_output::display::{ aligned_timestamp_bounds, resolve_effective_microbatch_batch_size, }; @@ -15,8 +16,6 @@ const TIMESTAMP_CURSOR: &str = "timestamp"; const INTEGER_CURSOR: &str = "integer"; const WATERMARK_STRATEGY: &str = "watermark"; const SECOND_GRAIN: &str = "second"; -/// `GRAIN_ORDER`, finest first. -const GRAIN_ORDER: [&str; 6] = ["second", "minute", "hour", "day", "month", "year"]; /// `CursorBoundsOwner` and `CursorResolutionStatus` of one model, as their values. pub(crate) const RUNTIME_OWNER: &str = "runtime"; @@ -44,9 +43,9 @@ pub(crate) struct CursorDetails<'py> { } fn grain_rank(grain: &str) -> PyResult { - GRAIN_ORDER + GRAIN_NAMES .iter() - .position(|known| *known == grain) + .position(|(known, _)| *known == grain) .ok_or_else(|| { pyo3::exceptions::PyValueError::new_err(format!("'{grain}' is not a valid CursorGrain")) }) diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_entries/command_plans.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_entries/command_plans.rs index 2086ff2412..c3402cd2e6 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_entries/command_plans.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_entries/command_plans.rs @@ -1,6 +1,6 @@ //! Static plans for commands that do not build models, and direct seed identity reads. -use crate::bindings::_helpers::compile_driver::python_calls::{KeywordArguments, imported}; +use crate::bindings::_helpers::compile_driver::python_calls::KeywordArguments; use crate::bindings::_helpers::planner_entries::plan_output::{ function_entry, selected_test_entries, test_selected, }; @@ -11,10 +11,10 @@ use crate::bindings::_helpers::planner_entries::values::{ use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyModule, PyModuleMethods, PyResult, Python}; use pyo3::types::{PyDict, PyDictMethods, PyFrozenSet, PyList, PyListMethods, PyString, PyTuple}; use pyo3::{pyfunction, wrap_pyfunction}; +use sqlbuild_analysis::fingerprint_storage::constants::NODE_TYPE_SEED; +use sqlbuild_core::constants::FINGERPRINT_TABLE_NAME; use std::collections::{BTreeMap, BTreeSet}; -const FINGERPRINT_CONSTANTS: &str = "sqlbuild.compiler.fingerprints.constants"; - /// Optional fields of a static command plan; absent ones take `PlanOutput` defaults. #[derive(Default)] struct StaticPlanParts<'py> { @@ -370,9 +370,7 @@ fn read_selected_seed_fingerprints<'py>( seeds: &Bound<'py, PyAny>, ) -> PyResult> { let py = adapter.py(); - let table_name: String = - imported(py, FINGERPRINT_CONSTANTS, "FINGERPRINT_TABLE_NAME")?.extract()?; - let seed_node_type = imported(py, FINGERPRINT_CONSTANTS, "NODE_TYPE_SEED")?; + let table_name: &str = FINGERPRINT_TABLE_NAME; let mut names_by_target: BTreeMap<(String, String), (Target, BTreeSet)> = BTreeMap::new(); for seed in seeds.try_iter()? { @@ -393,7 +391,7 @@ fn read_selected_seed_fingerprints<'py>( let fingerprints = PyDict::new(py); let mut relations_by_target: Vec> = Vec::new(); for ((database, schema), _) in names_by_target.values() { - let names = PyTuple::new(py, [table_name.as_str()])?; + let names = PyTuple::new(py, [table_name])?; let schemas = PyTuple::new(py, [schema.as_str()])?; let kwargs = PyDict::new(py); kwargs.set_item("connection", connection)?; @@ -449,7 +447,7 @@ fn read_selected_seed_fingerprints<'py>( )?; for item in items.iter() { let (name, fingerprint) = (item.get_item(0)?, item.get_item(1)?); - if fingerprint.getattr("node_type")?.eq(&seed_node_type)? + if fingerprint.getattr("node_type")?.eq(NODE_TYPE_SEED)? && names.contains(&name.extract::()?) { fingerprints.set_item(name, fingerprint)?; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/constants.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/constants.rs index 89636a1534..5a37210972 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/constants.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/constants.rs @@ -5,8 +5,6 @@ pub(super) const DONE: &str = "done"; pub(super) const PENDING_ARCHIVE: &str = "pending_archive"; pub(super) const PENDING_VIEW: &str = "pending_view"; pub(super) const NATIVE_MODULE: &str = "sqlbuild._native"; -pub(super) const FINGERPRINT_TABLE_NAME: &str = "_sqlbuild_fingerprints"; -pub(super) const NODE_TYPE_MODEL: &str = "model"; pub(super) const LIST_RELATIONS_FOR_INSPECTION: (&str, &str) = ( "sqlbuild.adapter.relations.main._list_relations_for_inspection", "list_relations_for_inspection", diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/state_inspection.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/state_inspection.rs index 05ea22662c..ad53285986 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/state_inspection.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/state_inspection.rs @@ -6,22 +6,23 @@ use pyo3::prelude::{ Bound, Py, PyAny, PyAnyMethods, PyRef, PyRefMut, PyResult, Python, pyclass, pymethods, }; use pyo3::types::{PyBool, PyDict, PyString, PyTuple, PyTupleMethods}; -use sqlbuild_state::events::constants::{ - COLUMN_MIGRATION_TABLE_NAME, MIGRATION_TABLE_NAME, OLD_NAME_VIEW_TABLE_NAME, -}; +use sqlbuild_core::constants::COLUMN_MIGRATION_TABLE_NAME; +use sqlbuild_core::constants::MIGRATION_TABLE_NAME; +use sqlbuild_core::constants::OLD_NAME_VIEW_TABLE_NAME; use sqlbuild_state::events::main::newest_fingerprint::newest_fingerprint; use sqlbuild_state::events::models::{EventRelation, FingerprintKey, FingerprintOrder}; use crate::bindings::_helpers::compile_driver::python_calls::{call_with, imported}; use crate::bindings::_helpers::planner_migrations::constants::{ - FINGERPRINT_TABLE_NAME, GET_COLUMNS_FOR_INSPECTION, LIST_RELATIONS_FOR_INSPECTION, - NATIVE_MODULE, NODE_TYPE_MODEL, + GET_COLUMNS_FOR_INSPECTION, LIST_RELATIONS_FOR_INSPECTION, NATIVE_MODULE, }; use crate::bindings::_helpers::state_rows::constants::{COLUMN_KIND, MODEL_KIND, OLD_NAME_KIND}; use crate::bindings::_helpers::state_rows::migration_io::{read_events, stored_old_name_columns}; use crate::bindings::_helpers::state_rows::migration_objects::layout; use crate::bindings::_helpers::state_rows::migration_projections::old_name_views; use crate::bindings::_helpers::state_rows::models::StateAccess; +use sqlbuild_analysis::fingerprint_storage::constants::NODE_TYPE_MODEL; +use sqlbuild_core::constants::FINGERPRINT_TABLE_NAME; /// A lowercased schema and relation name. type RelationKey = (String, String); diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_warehouse/constants.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_warehouse/constants.rs index 3ff81e8ff3..49765eee0a 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_warehouse/constants.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_warehouse/constants.rs @@ -47,13 +47,7 @@ pub(crate) const CURSOR_ALGEBRA_ERROR: (&str, &str) = ("sqlbuild.cursor_algebra.exceptions", "CursorAlgebraError"); pub(crate) const DATE_VALUE: (&str, &str) = ("sqlbuild._native", "DateValue"); pub(crate) const PLANNER_LOGGER: &str = "sqlbuild.planner"; -pub(crate) const FINGERPRINT_TABLE_NAME: &str = "_sqlbuild_fingerprints"; -pub(crate) const SOURCE_FRESHNESS_TABLE_NAME: &str = "_sqlbuild_source_freshness"; -pub(crate) const OLD_NAME_VIEW_TABLE_NAME: &str = "_sqlbuild_old_name_views"; -pub(crate) const MIGRATION_TABLE_NAME: &str = "_sqlbuild_migrations"; pub(crate) const METADATA_NAME_FILTER_LIMIT: usize = 250; -pub(crate) const NODE_TYPE_MODEL: &str = "model"; -pub(crate) const NODE_TYPE_SEED: &str = "seed"; pub(crate) const FUNCTION_NODE_TYPES: [&str; 2] = ["udf", "table_fn"]; pub(crate) const WILDCARD_TABLE_CHARACTER: &str = "*"; pub(crate) const MIGRATE_FROM_CONFIG_KEY: &str = "migrate_from"; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_warehouse/snapshot.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_warehouse/snapshot.rs index f46a99e78a..81872bf21a 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_warehouse/snapshot.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_warehouse/snapshot.rs @@ -7,10 +7,8 @@ use crate::bindings::_helpers::planner_phases::objects::{ by_name, frozenset, model_key, native, text, tuple, }; use crate::bindings::_helpers::planner_warehouse::constants::{ - ACTIVE_CATALOG, FINGERPRINT_TABLE_NAME, FUNCTION_NODE_TYPES, METADATA_NAME_FILTER_LIMIT, - MIGRATE_FROM_CONFIG_KEY, MIGRATION_TABLE_NAME, NODE_TYPE_MODEL, NODE_TYPE_SEED, - OLD_NAME_VIEW_TABLE_NAME, RELATIONS_MAIN_COLUMNS, RELATIONS_MAIN_LIST, - SOURCE_FRESHNESS_TABLE_NAME, + ACTIVE_CATALOG, FUNCTION_NODE_TYPES, METADATA_NAME_FILTER_LIMIT, MIGRATE_FROM_CONFIG_KEY, + RELATIONS_MAIN_COLUMNS, RELATIONS_MAIN_LIST, }; use crate::bindings::_helpers::planner_warehouse::cursors::{CursorInputs, cursor_snapshots}; use pyo3::IntoPyObject; @@ -18,6 +16,12 @@ use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult, Python}; use pyo3::pyfunction; use pyo3::types::PyDict as KeywordDict; use pyo3::types::{PyDict, PyDictMethods, PyString, PyTuple}; +use sqlbuild_analysis::fingerprint_storage::constants::NODE_TYPE_MODEL; +use sqlbuild_analysis::fingerprint_storage::constants::NODE_TYPE_SEED; +use sqlbuild_core::constants::FINGERPRINT_TABLE_NAME; +use sqlbuild_core::constants::MIGRATION_TABLE_NAME; +use sqlbuild_core::constants::OLD_NAME_VIEW_TABLE_NAME; +use sqlbuild_core::constants::SOURCE_FRESHNESS_TABLE_NAME; use std::collections::{BTreeSet, HashSet}; /// Everything one snapshot gather needs. diff --git a/crates/sqlbuild-python/src/bindings/_helpers/result_output/event_writer.rs b/crates/sqlbuild-python/src/bindings/_helpers/result_output/event_writer.rs index 41c6b55dab..6362c065ae 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/result_output/event_writer.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/result_output/event_writer.rs @@ -19,7 +19,7 @@ use crate::bindings::_helpers::result_output::terminal_index::{ TerminalEventClaim, TerminalEventIndex, current_index_object, }; -const RESULT_PATH_ENV: &str = "SQLBUILD_INTEGRATION_RESULT_PATH"; +pub(crate) const RESULT_PATH_ENV: &str = "SQLBUILD_INTEGRATION_RESULT_PATH"; const BUILD_OPTIONS: (&[&str], &[&str]) = (&["result", "plan"], &["command"]); /// Thread-safe JSON Lines execution event writer. diff --git a/crates/sqlbuild-python/src/bindings/_helpers/rules/constants.rs b/crates/sqlbuild-python/src/bindings/_helpers/rules/constants.rs index 1a0e8670cf..613e955c91 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/rules/constants.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/rules/constants.rs @@ -24,3 +24,8 @@ pub(crate) const CUSTOM_HOST_ABANDONED_SUFFIX: &str = ".abandoned"; pub(crate) const CUSTOM_HOST_CANCELLED_MARKER: &str = "cancelled"; /// Rules that need proven column types and are skipped without SQL analysis. pub(crate) const TYPE_PROOF_RULE_CODES: [&str; 1] = ["SQBRCONTRACT105"]; +/// Runtime identity custom-rule hosts and their cache entries record. +pub(crate) const CUSTOM_HOST_RUNTIME_VERSION: &str = "sqlbuild-rules-custom-v2"; +/// Module a custom-rule host process runs. +pub(crate) const CUSTOM_HOST_LAUNCH_MODULE: &str = + "sqlbuild.rule_engine._helpers.host.custom_host_launch"; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/rules/custom_evaluation.rs b/crates/sqlbuild-python/src/bindings/_helpers/rules/custom_evaluation.rs index 5b7dd1d2a5..6908890048 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/rules/custom_evaluation.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/rules/custom_evaluation.rs @@ -19,7 +19,9 @@ use sqlbuild_rules::custom_rule_cache::types::ReadSet; use crate::bindings::_helpers::cache::native_store::current_code_identity; use crate::bindings::_helpers::compile_driver::python_calls::{call_with, imported}; -use crate::bindings::_helpers::rules::constants::CUSTOM_RULES_CACHE_FILE; +use crate::bindings::_helpers::rules::constants::{ + CUSTOM_HOST_RUNTIME_VERSION, CUSTOM_RULES_CACHE_FILE, +}; use crate::bindings::_helpers::rules::custom_hosts::{ HostCancel, HostRequest, host_payload_project, run_custom_hosts, }; @@ -28,7 +30,6 @@ use crate::bindings::_helpers::rules::custom_rule_cache::CustomRuleCache; const NATIVE_RECORDS_MODULE: &str = "sqlbuild._native"; const NATIVE_MODULE: &str = "sqlbuild._native"; -const RULES_CONSTANTS_MODULE: &str = "sqlbuild.rule_engine.constants"; const RULES_ERRORS_MODULE: &str = "sqlbuild._native"; const RULE_CONTEXT_MODULE: &str = "sqlbuild.rule_engine.classes.rule_context"; const FACT_REPLAY_MODULE: &str = "sqlbuild.rule_engine._helpers.engine.fact_replay"; @@ -473,7 +474,7 @@ fn rule_identity( ), ( "runtime_version", - imported(py, RULES_CONSTANTS_MODULE, "CUSTOM_HOST_RUNTIME_VERSION")?, + pyo3::types::PyString::new(py, CUSTOM_HOST_RUNTIME_VERSION).into_any(), ), ("code", rule.getattr("code")?), ("family", rule.getattr("family")?), diff --git a/crates/sqlbuild-python/src/bindings/_helpers/rules/custom_hosts.rs b/crates/sqlbuild-python/src/bindings/_helpers/rules/custom_hosts.rs index 59ddcd9a3c..a72344175d 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/rules/custom_hosts.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/rules/custom_hosts.rs @@ -27,14 +27,14 @@ use sqlbuild_rules::engine::main::custom_host::run_custom_host_json; use crate::bindings::_helpers::compile_driver::python_calls::imported; use crate::bindings::_helpers::rules::constants::{ - CUSTOM_HOST_ABANDONED_SUFFIX, CUSTOM_HOST_CANCELLED_MARKER, + CUSTOM_HOST_ABANDONED_SUFFIX, CUSTOM_HOST_CANCELLED_MARKER, CUSTOM_HOST_LAUNCH_MODULE, + CUSTOM_HOST_RUNTIME_VERSION, }; use crate::bindings::_helpers::rules::custom_host_observation::{ LaunchLog, ObservedHosts, before_publish, observed_hosts, record_launch, }; const NATIVE_MODULE: &str = "sqlbuild._native"; -const RULES_CONSTANTS_MODULE: &str = "sqlbuild.rule_engine.constants"; const RULES_ERRORS_MODULE: &str = "sqlbuild._native"; const COMPILE_MODELS_MODULE: &str = "sqlbuild._native"; const HOST_INPUTS: [&str; 3] = ["target", "rules-cache", "host-inputs"]; @@ -169,8 +169,8 @@ impl Launch { ) -> PyResult { Ok(Self { program: imported(py, "sys", "executable")?.extract()?, - arguments: vec!["-m".to_owned(), constant(py, "CUSTOM_HOST_LAUNCH_MODULE")?], - runtime_version: constant(py, "CUSTOM_HOST_RUNTIME_VERSION")?, + arguments: vec!["-m".to_owned(), CUSTOM_HOST_LAUNCH_MODULE.to_owned()], + runtime_version: CUSTOM_HOST_RUNTIME_VERSION.to_owned(), payload, launches, }) @@ -432,10 +432,6 @@ fn cores(py: Python<'_>) -> PyResult { )) } -fn constant<'py, T: pyo3::FromPyObject<'py>>(py: Python<'py>, name: &str) -> PyResult { - imported(py, RULES_CONSTANTS_MODULE, name)?.extract() -} - fn uuid_hex(py: Python<'_>) -> PyResult { imported(py, "uuid", "uuid4")? .call0()? diff --git a/crates/sqlbuild-python/src/bindings/_helpers/semantic_shapes/shapes.rs b/crates/sqlbuild-python/src/bindings/_helpers/semantic_shapes/shapes.rs index 2f493570e6..468a2995fe 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/semantic_shapes/shapes.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/semantic_shapes/shapes.rs @@ -15,8 +15,10 @@ const NATIVE_MODULE: &str = "sqlbuild._native"; const DATACLASSES_MODULE: &str = "dataclasses"; const UNKNOWN: &str = "UNKNOWN"; const GENERIC_DIALECT: &str = "generic"; -const VALID_FROM_DEFAULT: &str = "valid_from"; -const VALID_TO_DEFAULT: &str = "valid_to"; +/// Snapshot validity start column when the model names none. +pub(crate) const VALID_FROM_DEFAULT: &str = "valid_from"; +/// Snapshot validity end column when the model names none. +pub(crate) const VALID_TO_DEFAULT: &str = "valid_to"; const TABLE_FUNCTION_PREFIX: &str = "__sqlbuild_table_function_"; const UDF_KIND: &str = "udf"; const TABLE_FUNCTION_KIND: &str = "table_fn"; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/state_rows/migration_io.rs b/crates/sqlbuild-python/src/bindings/_helpers/state_rows/migration_io.rs index 732a2326b3..4116afb30d 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/state_rows/migration_io.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/state_rows/migration_io.rs @@ -6,7 +6,7 @@ use pyo3::prelude::{ }; use pyo3::types::{PyDict, PyString, PyTuple, PyTupleMethods}; use pyo3::{pyfunction, wrap_pyfunction}; -use sqlbuild_state::events::constants::OLD_NAME_VIEW_TABLE_NAME; +use sqlbuild_core::constants::OLD_NAME_VIEW_TABLE_NAME; use sqlbuild_state::events::main::recorded_renames::recorded_renames; use sqlbuild_state::events::models::RecordedRenameKey; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/state_rows/migration_objects.rs b/crates/sqlbuild-python/src/bindings/_helpers/state_rows/migration_objects.rs index 43cec94da4..5487870783 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/state_rows/migration_objects.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/state_rows/migration_objects.rs @@ -3,9 +3,11 @@ use pyo3::exceptions::PyValueError; use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyErr, PyResult, Python}; use pyo3::types::PyString; +use sqlbuild_core::constants::COLUMN_MIGRATION_TABLE_NAME; +use sqlbuild_core::constants::MIGRATION_TABLE_NAME; +use sqlbuild_core::constants::OLD_NAME_VIEW_TABLE_NAME; use sqlbuild_state::events::constants::{ - COLUMN_MIGRATION_COLUMNS, COLUMN_MIGRATION_TABLE_NAME, MIGRATION_COLUMNS, MIGRATION_TABLE_NAME, - OLD_NAME_VIEW_COLUMNS, OLD_NAME_VIEW_TABLE_NAME, + COLUMN_MIGRATION_COLUMNS, MIGRATION_COLUMNS, OLD_NAME_VIEW_COLUMNS, }; use sqlbuild_state::events::models::EventOrder; diff --git a/crates/sqlbuild-python/src/bindings/main/register.rs b/crates/sqlbuild-python/src/bindings/main/register.rs index 141ff1ece5..aa81acaf02 100644 --- a/crates/sqlbuild-python/src/bindings/main/register.rs +++ b/crates/sqlbuild-python/src/bindings/main/register.rs @@ -6,10 +6,10 @@ use crate::bindings::_helpers::boundary::panics::NativeCompilerError; use crate::bindings::_helpers::{ analysis, analysis_session, attachments, auditing, authored_values, boundary, cache, cli, compile_attachment, compile_command, compile_driver, compile_outputs, compiled_project, - contracts, cursor, dag_artifact, diagnostics, discovery, fingerprints, frontier, graph, - lineage, macro_bridge, manifest, model_config, plan_output, planner_registration, - project_assembly, records, refactor, references, render, rules, scopes, semantic_checks, - semantic_shapes, sql_test_glue, sqltext, state_rows, type_system, + contracts, cursor, dag_artifact, diagnostics, discovery, exported_constants, fingerprints, + frontier, graph, lineage, macro_bridge, manifest, model_config, plan_output, + planner_registration, project_assembly, records, refactor, references, render, rules, scopes, + semantic_checks, semantic_shapes, sql_test_glue, sqltext, state_rows, type_system, }; use crate::bindings::models; @@ -52,6 +52,7 @@ fn register_project_stages(module: &Bound<'_, PyModule>) -> PyResult<()> { authored_values::registration::register(module)?; auditing::registration::register(module)?; cursor::registration::register(module)?; + exported_constants::registration::register(module)?; state_rows::registration::register(module)?; diagnostics::redaction::register(module)?; diagnostics::trackers::register(module)?; diff --git a/crates/sqlbuild-rules/src/configuration/_helpers/loading.rs b/crates/sqlbuild-rules/src/configuration/_helpers/loading.rs index a7d43d7096..ed6d6fa9af 100644 --- a/crates/sqlbuild-rules/src/configuration/_helpers/loading.rs +++ b/crates/sqlbuild-rules/src/configuration/_helpers/loading.rs @@ -1,9 +1,11 @@ +use crate::constants::MIN_CUSTOM_RULE_TEST_CASES_KEY; use crate::models::RulesConfig; use fensu_policy::policy::types::RuleCodeGrammar; +use sqlbuild_core::constants::PROJECT_CONFIG_FILENAME; use std::path::Path; pub(crate) fn load(project_dir: &Path) -> Result { - let path = project_dir.join("sqlbuild_project.toml"); + let path = project_dir.join(PROJECT_CONFIG_FILENAME); let source = std::fs::read_to_string(&path).map_err(|error| { format!( "could not load rules config from {}: {error}", @@ -100,7 +102,7 @@ pub(crate) fn validate(config: &RulesConfig) -> Result<(), String> { "max_ranking_order_by", "min_audits_per_model", "min_tests_per_model", - "min_custom_rule_test_cases", + MIN_CUSTOM_RULE_TEST_CASES_KEY, "max_subdomain_depth", "min_shared_owner_prefix_directories", "max_role_container_depth", diff --git a/crates/sqlbuild-rules/src/configuration/main/forbidden_exceptions.rs b/crates/sqlbuild-rules/src/configuration/main/forbidden_exceptions.rs index 7108ee1055..690d165255 100644 --- a/crates/sqlbuild-rules/src/configuration/main/forbidden_exceptions.rs +++ b/crates/sqlbuild-rules/src/configuration/main/forbidden_exceptions.rs @@ -5,8 +5,8 @@ use sqlbuild_core::cli_text::main::format_count_noun::format_count_noun; use crate::configuration::errors::ConfigPolicyError; use crate::configuration::main::setting_help::setting_help; -use crate::constants::PROJECT_CONFIG_FILE; use crate::models::RulesConfig; +use sqlbuild_core::constants::PROJECT_CONFIG_FILENAME as PROJECT_CONFIG_FILE; /// The policy error when `allow_exceptions = false` and exception or ignore entries exist. #[must_use] diff --git a/crates/sqlbuild-rules/src/configuration/main/setting_help.rs b/crates/sqlbuild-rules/src/configuration/main/setting_help.rs index 4cf8eabbf2..567bf96acd 100644 --- a/crates/sqlbuild-rules/src/configuration/main/setting_help.rs +++ b/crates/sqlbuild-rules/src/configuration/main/setting_help.rs @@ -1,6 +1,7 @@ //! Remediation text that shows the exact `sqlbuild_project.toml` setting to write. -use crate::constants::{PROJECT_CONFIG_FILE, SETTING_SNIPPET_INDENT}; +use crate::constants::SETTING_SNIPPET_INDENT; +use sqlbuild_core::constants::PROJECT_CONFIG_FILENAME as PROJECT_CONFIG_FILE; /// `, set this in sqlbuild_project.toml:` followed by the section and `key = value`. pub(crate) fn setting_help(purpose: &str, section: &str, key: &str, value: &str) -> String { diff --git a/crates/sqlbuild-rules/src/constants.rs b/crates/sqlbuild-rules/src/constants.rs index f2b6b08109..c8e55bb51f 100644 --- a/crates/sqlbuild-rules/src/constants.rs +++ b/crates/sqlbuild-rules/src/constants.rs @@ -29,7 +29,8 @@ pub(crate) const DECLARATION_DOMAIN_COMPONENTS: usize = 3; pub(crate) const RULE_CODE_NUMBER_LENGTH: usize = 3; pub(crate) const EMPTY_FIXTURE_QUERY_TOKEN_COUNT: usize = 6; pub(crate) const ENCLOSING_CTE_TOKEN_COUNT: usize = 3; -pub(crate) const PROJECT_CONFIG_FILE: &str = "sqlbuild_project.toml"; pub(crate) const SETTING_SNIPPET_INDENT: &str = " "; pub(crate) const NATIVE_ROWS_MEMO_VERSION: &str = "native-rules-rows-response-v1"; pub(crate) const SQL_RULE_PREFIX: &str = "SQBRSQL"; +/// Threshold key setting the fewest test cases a custom rule must ship. +pub const MIN_CUSTOM_RULE_TEST_CASES_KEY: &str = "min_custom_rule_test_cases"; diff --git a/crates/sqlbuild-rules/src/lint_text/constants.rs b/crates/sqlbuild-rules/src/lint_text/constants.rs index b713e61363..fbc6ff9d4b 100644 --- a/crates/sqlbuild-rules/src/lint_text/constants.rs +++ b/crates/sqlbuild-rules/src/lint_text/constants.rs @@ -121,7 +121,6 @@ pub(crate) const NON_CONTIGUOUS_FIX: &str = "fix overlaps generated SQL or a non-contiguous authored region"; pub(crate) const GENERATED_FIX: &str = "fix overlaps generated SQL"; pub(crate) const GENERATED_SQL_SUFFIX: &str = "(in generated SQL)"; -pub(crate) const PROJECT_CONFIG_FILENAME: &str = "sqlbuild_project.toml"; pub(crate) const LOCAL_CONFIG_FILENAME: &str = "sqlbuild_local.toml"; pub(crate) const DEFAULT_LINE_WIDTH: i64 = 100; pub(crate) const DEFAULT_MAX_DESCRIPTION_LINES: i64 = 10; diff --git a/crates/sqlbuild-rules/src/lint_text/main/resolve_format_settings.rs b/crates/sqlbuild-rules/src/lint_text/main/resolve_format_settings.rs index f017d05a27..d9cf308333 100644 --- a/crates/sqlbuild-rules/src/lint_text/main/resolve_format_settings.rs +++ b/crates/sqlbuild-rules/src/lint_text/main/resolve_format_settings.rs @@ -5,9 +5,10 @@ use std::path::Path; use crate::lint_text::_helpers::settings::{adapter_dialect, positive_int, read_table}; use crate::lint_text::constants::{ DEFAULT_DIALECT, DEFAULT_LINE_WIDTH, DEFAULT_MAX_DESCRIPTION_LINES, DEFAULT_MAX_LITERAL_LENGTH, - DEFAULT_MAX_RANKING_ORDER_BY, LOCAL_CONFIG_FILENAME, PROJECT_CONFIG_FILENAME, + DEFAULT_MAX_RANKING_ORDER_BY, LOCAL_CONFIG_FILENAME, }; use crate::lint_text::models::FormatSettings; +use sqlbuild_core::constants::PROJECT_CONFIG_FILENAME; /// `[format]` widths, rule thresholds and the adapter dialect, the local adapter winning. pub fn resolve_format_settings(project_dir: &Path) -> Result { diff --git a/crates/sqlbuild-rules/src/rule_catalogue/constants.rs b/crates/sqlbuild-rules/src/rule_catalogue/constants.rs index 927e472f7c..6f6a815944 100644 --- a/crates/sqlbuild-rules/src/rule_catalogue/constants.rs +++ b/crates/sqlbuild-rules/src/rule_catalogue/constants.rs @@ -1,5 +1,7 @@ //! Threshold defaults, guidance wording and the rule codes guidance sections depend on. +use crate::constants::MIN_CUSTOM_RULE_TEST_CASES_KEY; + /// Effective threshold defaults, sorted by name as guidance renders them. pub(crate) const THRESHOLD_DEFAULTS: [(&str, u32); 10] = [ ("max_constant_container_files", 10), @@ -8,7 +10,7 @@ pub(crate) const THRESHOLD_DEFAULTS: [(&str, u32); 10] = [ ("max_role_container_depth", 1), ("max_subdomain_depth", 1), ("min_audits_per_model", 1), - ("min_custom_rule_test_cases", 1), + (MIN_CUSTOM_RULE_TEST_CASES_KEY, 1), ("min_shared_container_prefix_files", 2), ("min_shared_owner_prefix_directories", 2), ("min_tests_per_model", 1), diff --git a/crates/sqlbuild-rules/src/rules/_helpers/evaluation.rs b/crates/sqlbuild-rules/src/rules/_helpers/evaluation.rs index e27f63926b..50729ded5e 100644 --- a/crates/sqlbuild-rules/src/rules/_helpers/evaluation.rs +++ b/crates/sqlbuild-rules/src/rules/_helpers/evaluation.rs @@ -1,5 +1,6 @@ use crate::configuration::main::setting_help::setting_help; use crate::configuration::main::toml_string_array::toml_string_array; +use crate::constants::MIN_CUSTOM_RULE_TEST_CASES_KEY; use crate::constants::{ DECLARATION_DOMAIN_COMPONENTS, ENFORCED_CONTRACT, NEGATION_OPERATOR, REFERENCE_KIND, SOURCE_REFERENCE_KIND, VIEW_MATERIALIZATION, @@ -1492,7 +1493,7 @@ fn custom_rule_test_coverage( let (selected, rule) = policy; let minimum = config .thresholds - .get("min_custom_rule_test_cases") + .get(MIN_CUSTOM_RULE_TEST_CASES_KEY) .copied() .unwrap_or(1); if minimum == 0 { diff --git a/crates/sqlbuild-rules/src/rules/_helpers/model_layers.rs b/crates/sqlbuild-rules/src/rules/_helpers/model_layers.rs index 8a9bf4039f..ce66e8fcf6 100644 --- a/crates/sqlbuild-rules/src/rules/_helpers/model_layers.rs +++ b/crates/sqlbuild-rules/src/rules/_helpers/model_layers.rs @@ -7,6 +7,7 @@ use crate::constants::{ use crate::models::{Fault, Model, RuleMetadata}; use crate::rules::_helpers::evaluation::parse_name; use crate::rules::models::{FaultCollector, ProjectEvaluationRequest}; +use sqlbuild_core::constants::PROJECT_CONFIG_FILENAME; const NO_CANONICAL_LAYER_FOLDER: &str = "no canonical layer folder"; @@ -85,7 +86,7 @@ pub(crate) fn internal_forward_refs( faults.push(Fault { unevaluated: false, code: rule.code.clone(), - path: "sqlbuild_project.toml".into(), + path: PROJECT_CONFIG_FILENAME.into(), line: 1, column: 1, message: format!( diff --git a/crates/sqlbuild-rules/src/rules/_helpers/sql_test_rules.rs b/crates/sqlbuild-rules/src/rules/_helpers/sql_test_rules.rs index 4c9fe1d5ed..6b5419602d 100644 --- a/crates/sqlbuild-rules/src/rules/_helpers/sql_test_rules.rs +++ b/crates/sqlbuild-rules/src/rules/_helpers/sql_test_rules.rs @@ -20,7 +20,7 @@ const UNIT_ROOT: &str = "tests/unit"; const SCENARIO_ROOT: &str = "tests/scenarios"; const EMPTY_INPUT_RULE_CODE: &str = "SQBRTEST203"; const ALLOWED_TESTS_OPTION: &str = "allowed_tests"; -const PROJECT_CONFIG_PATH: &str = "sqlbuild_project.toml"; +use sqlbuild_core::constants::PROJECT_CONFIG_FILENAME as PROJECT_CONFIG_PATH; pub(crate) fn evaluate_project( evaluation: ProjectEvaluationRequest<'_>, diff --git a/crates/sqlbuild-scopes/src/scope_cache/constants.rs b/crates/sqlbuild-scopes/src/scope_cache/constants.rs index f30b5208ef..5f261c2754 100644 --- a/crates/sqlbuild-scopes/src/scope_cache/constants.rs +++ b/crates/sqlbuild-scopes/src/scope_cache/constants.rs @@ -1,5 +1,7 @@ //! Cache schema and fingerprint versions, keys and the scope-relevant source facts. +use sqlbuild_core::constants::PROJECT_CONFIG_FILENAME; + pub(crate) const CACHE_SCHEMA_VERSION: i64 = 3; pub(crate) const FINGERPRINT_ALGORITHM_VERSION: i64 = 2; /// The largest cache file read; bigger files are misses. @@ -38,7 +40,7 @@ pub(crate) const COMPLETENESS_KEYS: [&str; 6] = [ ]; /// Config files the project fingerprint hashes when present. pub(crate) const FINGERPRINT_CONFIG_FILES: [&str; 4] = [ - "sqlbuild_project.toml", + PROJECT_CONFIG_FILENAME, "sqlbuild_local.toml", ".sqlbuildignore", ".gitignore", diff --git a/crates/sqlbuild-state/src/events/constants.rs b/crates/sqlbuild-state/src/events/constants.rs index f0cf958db1..c89814977d 100644 --- a/crates/sqlbuild-state/src/events/constants.rs +++ b/crates/sqlbuild-state/src/events/constants.rs @@ -15,9 +15,6 @@ pub(crate) const ALIAS_KEY_SEPARATOR: &str = ":"; pub(crate) const PYTHON_TRUE: &str = "True"; pub(crate) const PYTHON_FALSE: &str = "False"; pub(crate) const PYTHON_NONE: &str = "None"; -pub const MIGRATION_TABLE_NAME: &str = "_sqlbuild_migrations"; -pub const COLUMN_MIGRATION_TABLE_NAME: &str = "_sqlbuild_column_migrations"; -pub const OLD_NAME_VIEW_TABLE_NAME: &str = "_sqlbuild_old_name_views"; /// Event columns stored as text timestamps; every other event column is a string. pub const TIMESTAMP_COLUMNS: [&str; 2] = ["expires_at", "created_at"]; pub const MIGRATION_COLUMNS: [&str; 15] = [ diff --git a/crates/sqlbuild-state/src/freshness/constants.rs b/crates/sqlbuild-state/src/freshness/constants.rs index 19e375d039..6cc649bec7 100644 --- a/crates/sqlbuild-state/src/freshness/constants.rs +++ b/crates/sqlbuild-state/src/freshness/constants.rs @@ -28,7 +28,6 @@ pub(crate) const MARCH_BASED_MONTH_DAYS: i128 = 153; pub(crate) const MONTH_LENGTH_DIVISOR: i128 = 5; pub(crate) const MONTHS_FROM_MARCH: i128 = 9; pub(crate) const FEBRUARY: u32 = 2; -pub const SOURCE_FRESHNESS_TABLE_NAME: &str = "_sqlbuild_source_freshness"; /// Columns a source freshness state table must have, in Python's declaration order. pub const REQUIRED_STATE_COLUMNS: [&str; 6] = [ "source_name", diff --git a/python-boundary.toml b/python-boundary.toml index d79f6e5b03..e066008ffd 100644 --- a/python-boundary.toml +++ b/python-boundary.toml @@ -244,11 +244,6 @@ path = "runtime/execution_history/exceptions.py" reason = "public-api" note = "Rule 3 shapes: public storage errors re-exported by sqlbuild.execution_history and raised by the native backends." -[[stays]] -path = "runtime/execution_history/constants.py" -reason = "public-api" -note = "Rule 3 shapes: page limit and schema version constants re-exported by sqlbuild.execution_history." - [[stays]] path = "runtime/execution_limits" reason = "executor" @@ -292,7 +287,7 @@ note = "Rule 3 shapes: public storage errors re-exported by sqlbuild.compute_log [[stays]] path = "runtime/compute_logs/constants.py" reason = "public-api" -note = "Rule 3 shapes: format version, retention and read-limit constants re-exported by sqlbuild.compute_logs." +note = "Rule 1 capture shim constants: the SQL log record field and redaction message the Python diagnostic log handler uses; format version, retention and read limits are native (sqlbuild._native)." [[stays]] path = "runtime/event_exporting/__init__.py" @@ -387,7 +382,7 @@ note = "Rule 3 shapes: CommandOutputValidationError re-exported by sqlbuild.sink [[stays]] path = "runtime/output_capture/constants.py" reason = "public-api" -note = "Rule 3 shapes: the invocation-context variable and wire constants used by the models and integrations." +note = "Rule 3 shapes: the command-output sink parameter name and producer version; INVOCATION_CONTEXT_ENV is re-exported from sqlbuild._native for integrations." [[stays]] path = "integrations/dagster" @@ -682,7 +677,7 @@ note = "Shape rule: data records only (CompileReuseRequest, SettingsEnvironmentI [[stays]] path = "compiler/source_freshness/constants.py" reason = "adapter" -note = "Shape rule: data constants only (SOURCE_FRESHNESS_TABLE_NAME, COLUMN_SOURCE_NAME, COLUMN_TARGET_DATABASE, COLUMN_TARGET_SCHEMA, ...), no logic; consumed by adapter/contract/classes/duckdb_backed_adapter.py, adapter/state_sql/_helpers/source_freshness.py, adapters/bigquery/classes/bigquery_adapter.py and 6 more." +note = "Shape rule: source freshness column names and adapter column types, no logic; SOURCE_FRESHNESS_TABLE_NAME is re-exported from sqlbuild._native." [[stays]] path = "compiler/auditing/types.py" @@ -717,7 +712,7 @@ note = "Shape rule: data exceptions only (SharedInputError, ExecutorInputError), [[stays]] path = "compiler/fingerprints/constants.py" reason = "executor" -note = "Shape rule: data constants only (FINGERPRINT_TABLE_NAME, COLUMN_NODE_TYPE, COLUMN_NODE_NAME, COLUMN_TARGET_DATABASE, ...), no logic; consumed by adapter/contract/classes/duckdb_backed_adapter.py, adapters/bigquery/classes/bigquery_adapter.py, adapters/databricks/classes/databricks_adapter.py and 13 more." +note = "Shape rule: Python-only node type names (NODE_TYPE_HOOK, NODE_TYPE_QUERY_DIFF_ARTIFACT), no logic; table and column names are native (sqlbuild._native)." [[stays]] path = "diagnostics/models.py" @@ -759,11 +754,6 @@ path = "integrations/dbt/exceptions.py" reason = "user-code" note = "Shape rule: data exceptions only (DbtProfileError), no logic; consumed by integrations/dbt/_helpers/profile/load.py, integrations/dbt/_helpers/profile/render.py." -[[stays]] -path = "compiler/migrations/constants.py" -reason = "executor" -note = "Shape rule: data constants only (MIGRATION_TABLE_NAME, MIGRATION_WRITE_ATTEMPTS, COLUMN_MIGRATION_TABLE_NAME, OLD_NAME_VIEW_TABLE_NAME), no logic; consumed by adapters/sqlserver/classes/sqlserver_adapter.py, executor/migrations/_helpers/columns.py, executor/migrations/_helpers/old_name_facts.py and 2 more." - [[stays]] path = "diagnostics/constants.py" reason = "public-api" @@ -777,7 +767,7 @@ note = "Shape rule: data exceptions only (CursorAlgebraError), no logic; consume [[stays]] path = "spec/contracts/constants.py" reason = "adapter" -note = "Shape rule: data constants only (DEFAULT_SEED_CSV_SETTINGS, ZERO_DAY_CURSOR_DURATION), no logic; consumed by adapter/contract/classes/base_adapter.py, adapter/contract/classes/duckdb_backed_adapter.py, adapter/contract/classes/materialization.py and 6 more." +note = "Shape rule: data constants only (DEFAULT_SEED_CSV_SETTINGS), no logic; consumed by adapter contract classes." [[stays]] path = "cli/entry/constants.py" @@ -797,7 +787,7 @@ note = "Shape rule: data exceptions only (SqlValueRenderingError), no logic; con [[stays]] path = "cli/output/constants.py" reason = "public-api" -note = "Shape rule: data constants only (INTEGRATION_RESULT_PATH_ENV), no logic; consumed by integrations/dagster/_helpers/invocation_factory.py." +note = "Shape rule: re-export only (INTEGRATION_RESULT_PATH_ENV, from sqlbuild._native), no logic; consumed by integrations/dagster/_helpers/invocation_factory.py." [[stays]] path = "compiler/compile/types.py" @@ -812,42 +802,17 @@ note = "Shape rule: data protocol and aliases only (RuleContext, RuleCheck, Rule [[stays]] path = "rule_engine/constants.py" reason = "user-code" -note = "Shape rule: data constants only (CUSTOM_HOST_PROTOCOL_VERSION, CUSTOM_HOST_RUNTIME_VERSION, CUSTOM_RULE_IMPORT_ROOTS, FACT_PROJECT_MODELS, ...), no logic; consumed by rule_engine/_helpers/host/custom_host.py, rule_engine/_helpers/host/tracked_facts.py, rule_engine/_helpers/engine/fact_replay.py and 10 more." - -[[stays]] -path = "compiler/discovery/constants.py" -reason = "executor" -note = "Shape rule: data constants only (PROJECT_CONFIG_FILENAME, SETTINGS_SECTION, TABLE_PROMOTION_MODE_SETTING_KEY, PYTHON_NODE_ROOT, ...), no logic; consumed by executor/run/main/_execute.py, compiler/discovery/_helpers/filesystem/project_modules.py." +note = "Shape rule: custom-rule host and fact-name constants (CUSTOM_HOST_MAX_TRACKED_READS, CUSTOM_RULE_IMPORT_ROOTS, FACT_PROJECT_MODELS, ...) used by the Python custom-rule host, no logic; versions, launch module and shared keys are native (sqlbuild._native)." [[stays]] path = "compiler/compile/exceptions.py" reason = "user-code" note = "Shape rule: data exceptions only (MacroDeclarationLookupError, a CompileInputError that is also a KeyError, which a native error class with one parent cannot express), no logic; consumed by compiler/compile/_helpers/render/user_macros.py." -[[stays]] -path = "cursor_algebra/constants.py" -reason = "executor" -note = "Shape rule: data constants only (GRAIN_ORDER), no logic; consumed by executor/run/_helpers/validation/cursor_bounds.py." - [[stays]] path = "compiler/sql_analysis/constants.py" reason = "public-api" -note = "Shape rule: data constants only (POLYGLOT_ANALYZE_QUERY_ENTRY_POINT, POLYGLOT_COMPLEXITY_GUARD_OPTION, POLYGLOT_MAX_FUNCTION_CALL_DEPTH, ...), no logic; consumed by compiler/sql_analysis/_helpers/polyglot.py." - -[[stays]] -path = "compiler/planner/constants.py" -reason = "executor" -note = "Shape rule: data constants only (MICROBATCH_START_SENTINEL, MICROBATCH_END_SENTINEL, MIGRATION_FINGERPRINT_METADATA_KEY, SNAPSHOT_DEFAULT_VALID_FROM_COLUMN, ...), no logic; consumed by executor/build/_helpers/scheduler.py, executor/run/_helpers/validation/cursor_bounds.py, executor/run/_helpers/reuse/fingerprint_metadata.py and 1 more." - -[[stays]] -path = "compiler/frontier/constants.py" -reason = "user-code" -note = "Shape rule: data constants only (STAGE_CAPTURE_DIR_ENV_VAR, COMPILE_REUSE_DISABLE_ENV_VAR), no logic; consumed by compiler/macro_bridge/constants.py." - -[[stays]] -path = "cost/constants.py" -reason = "adapter" -note = "Shape rule: data constants only (SQLBUILD_QUERY_TAG_APP), no logic; consumed by adapters/snowflake/classes/snowflake_cursor.py." +note = "Rule 4: polyglot_sql glue only; the entry point, call-shape and option names (POLYGLOT_ANALYZE_QUERY_ENTRY_POINT, POLYGLOT_COMPLEXITY_GUARD_OPTION, POLYGLOT_MAX_FUNCTION_CALL_DEPTH, ...) that compiler/sql_analysis/_helpers/polyglot.py uses to wrap the polyglot_sql Python package." [[stays]] path = "compiler/compile/models.py" diff --git a/src/sqlbuild/_native.pyi b/src/sqlbuild/_native.pyi index f0004ab238..6f1ad030d1 100644 --- a/src/sqlbuild/_native.pyi +++ b/src/sqlbuild/_native.pyi @@ -159,6 +159,65 @@ from sqlbuild.runtime.output_capture.models import ( from sqlbuild.runtime.output_capture.types import CommandOutputStream BUILD_IDENTITY: str +MICROBATCH_START_SENTINEL: str +MICROBATCH_END_SENTINEL: str +MIGRATION_FINGERPRINT_METADATA_KEY: str +SNAPSHOT_DEFAULT_VALID_FROM_COLUMN: str +SNAPSHOT_DEFAULT_VALID_TO_COLUMN: str +PROJECT_CONFIG_FILENAME: str +PYTHON_INIT_MODULE_STEM: str +PYTHON_NODE_ROOT: str +SETTINGS_SECTION: str +TABLE_PROMOTION_MODE_SETTING_KEY: str +STAGED_PROMOTION_MODE: str +STAGE_CAPTURE_DIR_ENV_VAR: str +COMPILE_REUSE_DISABLE_ENV_VAR: str +SQLBUILD_QUERY_TAG_APP: str +STORE_TRACKED_ENVIRONMENT_PREFIXES: tuple[str, ...] +STORE_UNTRACKED_ENVIRONMENT_NAMES: frozenset[str] +GRAIN_ORDER: dict[CursorGrain, int] +INVOCATION_CONTEXT_ENV: str +COMMAND_OUTPUT_RECORD_TYPE: str +COMMAND_OUTPUT_LOSS_RECORD_TYPE: str +INTEGRATION_RESULT_PATH_ENV: str +FINGERPRINT_TABLE_NAME: str +SOURCE_FRESHNESS_TABLE_NAME: str +MIGRATION_TABLE_NAME: str +COLUMN_MIGRATION_TABLE_NAME: str +OLD_NAME_VIEW_TABLE_NAME: str +COLUMN_NODE_TYPE: str +COLUMN_NODE_NAME: str +COLUMN_TARGET_DATABASE: str +COLUMN_TARGET_SCHEMA: str +COLUMN_TARGET_NAME: str +COLUMN_RUN_ID: str +COLUMN_DEFINITION_HASH: str +COLUMN_VERSION_HASH: str +COLUMN_SCHEMA_FINGERPRINT: str +COLUMN_DEFINITION_B64: str +COLUMN_METADATA_JSON_B64: str +COLUMN_TIMESTAMP: str +NODE_TYPE_MODEL: str +NODE_TYPE_SEED: str +AUDIT_GATE_METADATA_KEY: str +CUSTOM_HOST_RUNTIME_VERSION: str +CUSTOM_HOST_LAUNCH_MODULE: str +CUSTOM_RULE_PROJECT_SUBJECT: str +CONTRACT_ENFORCED: str +LAUNCH_RECORD_OPTION: str +ZERO_DAY_CURSOR_DURATION: str +AST_CTE_KIND: str +MIN_CUSTOM_RULE_TEST_CASES: str +DEFAULT_PAGE_LIMIT: int +MAX_PAGE_LIMIT: int +CURRENT_EVENT_LOG_SCHEMA_VERSION: int +CURRENT_RUN_STORAGE_SCHEMA_VERSION: int +COMPUTE_LOG_FORMAT_VERSION: int +DEFAULT_RETENTION_COUNT: int +MAX_READ_BYTES: int +CURRENT_COMMAND_OUTPUT_SCHEMA_VERSION: int +MIGRATION_WRITE_ATTEMPTS: int +CUSTOM_HOST_PROTOCOL_VERSION: int def plan_sql_tests( *, diff --git a/src/sqlbuild/adapter/contract/classes/duckdb_backed_adapter.py b/src/sqlbuild/adapter/contract/classes/duckdb_backed_adapter.py index 447c10131c..934de58d39 100644 --- a/src/sqlbuild/adapter/contract/classes/duckdb_backed_adapter.py +++ b/src/sqlbuild/adapter/contract/classes/duckdb_backed_adapter.py @@ -945,7 +945,7 @@ def render_prune_fingerprint_history_sql( schema: str, retain_versions: int, ) -> str: - from sqlbuild.compiler.fingerprints.constants import FINGERPRINT_TABLE_NAME + from sqlbuild._native import FINGERPRINT_TABLE_NAME table_name: str | None = self.render_qualified_name( database=database, @@ -973,7 +973,7 @@ def render_prune_source_freshness_history_sql( schema: str, retain_versions: int, ) -> str: - from sqlbuild.compiler.source_freshness.constants import SOURCE_FRESHNESS_TABLE_NAME + from sqlbuild._native import SOURCE_FRESHNESS_TABLE_NAME table_name: str | None = self.render_qualified_name( database=database, diff --git a/src/sqlbuild/adapters/bigquery/classes/bigquery_adapter.py b/src/sqlbuild/adapters/bigquery/classes/bigquery_adapter.py index 32ed39021c..27d1c538aa 100644 --- a/src/sqlbuild/adapters/bigquery/classes/bigquery_adapter.py +++ b/src/sqlbuild/adapters/bigquery/classes/bigquery_adapter.py @@ -404,7 +404,7 @@ def render_prune_fingerprint_history_sql( schema: str, retain_versions: int, ) -> str: - from sqlbuild.compiler.fingerprints.constants import FINGERPRINT_TABLE_NAME + from sqlbuild._native import FINGERPRINT_TABLE_NAME table_name: str | None = self.render_qualified_name( database=database, @@ -436,7 +436,7 @@ def render_prune_source_freshness_history_sql( schema: str, retain_versions: int, ) -> str: - from sqlbuild.compiler.source_freshness.constants import SOURCE_FRESHNESS_TABLE_NAME + from sqlbuild._native import SOURCE_FRESHNESS_TABLE_NAME table_name: str | None = self.render_qualified_name( database=database, diff --git a/src/sqlbuild/adapters/databricks/classes/databricks_adapter.py b/src/sqlbuild/adapters/databricks/classes/databricks_adapter.py index 68ead70ade..5b18d976db 100644 --- a/src/sqlbuild/adapters/databricks/classes/databricks_adapter.py +++ b/src/sqlbuild/adapters/databricks/classes/databricks_adapter.py @@ -495,7 +495,7 @@ def render_prune_fingerprint_history_sql( schema: str, retain_versions: int, ) -> str: - from sqlbuild.compiler.fingerprints.constants import FINGERPRINT_TABLE_NAME + from sqlbuild._native import FINGERPRINT_TABLE_NAME table_name: str | None = self.render_qualified_name( database=database, @@ -527,7 +527,7 @@ def render_prune_source_freshness_history_sql( schema: str, retain_versions: int, ) -> str: - from sqlbuild.compiler.source_freshness.constants import SOURCE_FRESHNESS_TABLE_NAME + from sqlbuild._native import SOURCE_FRESHNESS_TABLE_NAME table_name: str | None = self.render_qualified_name( database=database, diff --git a/src/sqlbuild/adapters/postgres/classes/postgres_adapter.py b/src/sqlbuild/adapters/postgres/classes/postgres_adapter.py index 740c59410c..06f5ca83fb 100644 --- a/src/sqlbuild/adapters/postgres/classes/postgres_adapter.py +++ b/src/sqlbuild/adapters/postgres/classes/postgres_adapter.py @@ -1518,7 +1518,7 @@ def render_create_fingerprint_index_sqls( database: str | None, schema: str, ) -> tuple[str, ...]: - from sqlbuild.compiler.fingerprints.constants import FINGERPRINT_TABLE_NAME + from sqlbuild._native import FINGERPRINT_TABLE_NAME table_name: str | None = self.render_qualified_name( database=database, @@ -1568,7 +1568,7 @@ def render_prune_fingerprint_history_sql( schema: str, retain_versions: int, ) -> str: - from sqlbuild.compiler.fingerprints.constants import FINGERPRINT_TABLE_NAME + from sqlbuild._native import FINGERPRINT_TABLE_NAME table_name: str | None = self.render_qualified_name( database=database, @@ -1605,7 +1605,7 @@ def render_create_source_freshness_index_sqls( database: str | None, schema: str, ) -> tuple[str, ...]: - from sqlbuild.compiler.source_freshness.constants import SOURCE_FRESHNESS_TABLE_NAME + from sqlbuild._native import SOURCE_FRESHNESS_TABLE_NAME table_name: str | None = self.render_qualified_name( database=database, @@ -1780,7 +1780,7 @@ def render_prune_source_freshness_history_sql( schema: str, retain_versions: int, ) -> str: - from sqlbuild.compiler.source_freshness.constants import SOURCE_FRESHNESS_TABLE_NAME + from sqlbuild._native import SOURCE_FRESHNESS_TABLE_NAME table_name: str | None = self.render_qualified_name( database=database, diff --git a/src/sqlbuild/adapters/snowflake/classes/snowflake_adapter.py b/src/sqlbuild/adapters/snowflake/classes/snowflake_adapter.py index 2b6e20dad3..229155db69 100644 --- a/src/sqlbuild/adapters/snowflake/classes/snowflake_adapter.py +++ b/src/sqlbuild/adapters/snowflake/classes/snowflake_adapter.py @@ -732,7 +732,7 @@ def render_prune_fingerprint_history_sql( schema: str, retain_versions: int, ) -> str: - from sqlbuild.compiler.fingerprints.constants import FINGERPRINT_TABLE_NAME + from sqlbuild._native import FINGERPRINT_TABLE_NAME table_name: str | None = self.render_qualified_name( database=database, @@ -764,7 +764,7 @@ def render_prune_source_freshness_history_sql( schema: str, retain_versions: int, ) -> str: - from sqlbuild.compiler.source_freshness.constants import SOURCE_FRESHNESS_TABLE_NAME + from sqlbuild._native import SOURCE_FRESHNESS_TABLE_NAME table_name: str | None = self.render_qualified_name( database=database, diff --git a/src/sqlbuild/adapters/snowflake/classes/snowflake_cursor.py b/src/sqlbuild/adapters/snowflake/classes/snowflake_cursor.py index 8b5d0fd682..94d6b96685 100644 --- a/src/sqlbuild/adapters/snowflake/classes/snowflake_cursor.py +++ b/src/sqlbuild/adapters/snowflake/classes/snowflake_cursor.py @@ -12,8 +12,7 @@ from typing import Any from uuid import uuid4 -from sqlbuild._native import CostContext, record_statement -from sqlbuild.cost.constants import SQLBUILD_QUERY_TAG_APP +from sqlbuild._native import SQLBUILD_QUERY_TAG_APP, CostContext, record_statement from sqlbuild.cost.models import CostResourceContext, StatementExecutionTelemetry from sqlbuild.runtime.execution_limits.main.current_execution_deadline import ( current_execution_deadline, diff --git a/src/sqlbuild/adapters/sqlserver/classes/sqlserver_adapter.py b/src/sqlbuild/adapters/sqlserver/classes/sqlserver_adapter.py index a6c30bf6e4..4527779538 100644 --- a/src/sqlbuild/adapters/sqlserver/classes/sqlserver_adapter.py +++ b/src/sqlbuild/adapters/sqlserver/classes/sqlserver_adapter.py @@ -316,7 +316,7 @@ def render_create_fingerprint_table_sql( database: str | None, schema: str, ) -> str: - from sqlbuild.compiler.fingerprints.constants import ( + from sqlbuild._native import ( COLUMN_DEFINITION_B64, COLUMN_DEFINITION_HASH, COLUMN_METADATA_JSON_B64, @@ -385,7 +385,7 @@ def render_create_fingerprint_index_sqls( database: str | None, schema: str, ) -> tuple[str, ...]: - from sqlbuild.compiler.fingerprints.constants import FINGERPRINT_TABLE_NAME + from sqlbuild._native import FINGERPRINT_TABLE_NAME table_name: str | None = self.render_qualified_name( database=database, @@ -430,7 +430,7 @@ def render_prune_fingerprint_history_sql( schema: str, retain_versions: int, ) -> str: - from sqlbuild.compiler.fingerprints.constants import FINGERPRINT_TABLE_NAME + from sqlbuild._native import FINGERPRINT_TABLE_NAME table_name: str | None = self.render_qualified_name( database=database, @@ -465,7 +465,7 @@ def render_create_source_freshness_index_sqls( database: str | None, schema: str, ) -> tuple[str, ...]: - from sqlbuild.compiler.source_freshness.constants import SOURCE_FRESHNESS_TABLE_NAME + from sqlbuild._native import SOURCE_FRESHNESS_TABLE_NAME table_name: str | None = self.render_qualified_name( database=database, @@ -657,8 +657,7 @@ def render_create_janitor_event_table_sql(self, *, database: str | None, schema: ) def render_create_migration_state_table_sql(self, *, database: str | None, schema: str) -> str: - from sqlbuild._native import render_migration_state_create_sql - from sqlbuild.compiler.migrations.constants import MIGRATION_TABLE_NAME + from sqlbuild._native import MIGRATION_TABLE_NAME, render_migration_state_create_sql create_sql: str = render_migration_state_create_sql( kind="model", @@ -677,8 +676,7 @@ def render_create_migration_state_table_sql(self, *, database: str | None, schem def render_create_old_name_view_state_table_sql( self, *, database: str | None, schema: str ) -> str: - from sqlbuild._native import render_migration_state_create_sql - from sqlbuild.compiler.migrations.constants import OLD_NAME_VIEW_TABLE_NAME + from sqlbuild._native import OLD_NAME_VIEW_TABLE_NAME, render_migration_state_create_sql create_sql: str = render_migration_state_create_sql( kind="old_name", @@ -697,8 +695,7 @@ def render_create_old_name_view_state_table_sql( def render_create_column_migration_state_table_sql( self, *, database: str | None, schema: str ) -> str: - from sqlbuild._native import render_migration_state_create_sql - from sqlbuild.compiler.migrations.constants import COLUMN_MIGRATION_TABLE_NAME + from sqlbuild._native import COLUMN_MIGRATION_TABLE_NAME, render_migration_state_create_sql create_sql: str = render_migration_state_create_sql( kind="column", @@ -735,7 +732,7 @@ def render_prune_source_freshness_history_sql( schema: str, retain_versions: int, ) -> str: - from sqlbuild.compiler.source_freshness.constants import SOURCE_FRESHNESS_TABLE_NAME + from sqlbuild._native import SOURCE_FRESHNESS_TABLE_NAME table_name: str | None = self.render_qualified_name( database=database, diff --git a/src/sqlbuild/cli/commands/main/entrypoint/_dispatch_with_compute_logs.py b/src/sqlbuild/cli/commands/main/entrypoint/_dispatch_with_compute_logs.py index a8cc32b3bd..926b51d22c 100644 --- a/src/sqlbuild/cli/commands/main/entrypoint/_dispatch_with_compute_logs.py +++ b/src/sqlbuild/cli/commands/main/entrypoint/_dispatch_with_compute_logs.py @@ -6,14 +6,18 @@ from datetime import UTC, datetime from pathlib import Path -from sqlbuild._native import CliNamespace, LocalFilesystemComputeLogStorage, supports_color +from sqlbuild._native import ( + COMPUTE_LOG_FORMAT_VERSION, + CliNamespace, + LocalFilesystemComputeLogStorage, + supports_color, +) from sqlbuild.cli.commands._helpers.entry.compute_log_diagnostics import ( log_compute_capture_failure, ) from sqlbuild.diagnostics.classes.invocation_diagnostic_routing import InvocationDiagnosticRouting from sqlbuild.diagnostics.main.configure import configure_diagnostics from sqlbuild.diagnostics.models import DiagnosticRoutingOptions -from sqlbuild.runtime.compute_logs.constants import COMPUTE_LOG_FORMAT_VERSION from sqlbuild.runtime.compute_logs.main.run_with_compute_log_capture import ( run_with_compute_log_capture, ) diff --git a/src/sqlbuild/cli/entry/constants.py b/src/sqlbuild/cli/entry/constants.py index 12e543d6ee..3bff31b705 100644 --- a/src/sqlbuild/cli/entry/constants.py +++ b/src/sqlbuild/cli/entry/constants.py @@ -4,4 +4,3 @@ NATIVE_ALLOCATOR_PURGE_DELAY_MILLISECONDS: str = "10" OUTPUT_STREAM_ENCODING: str = "utf-8" OUTPUT_STREAM_ERRORS: str = "backslashreplace" -LAUNCH_RECORD_OPTION: str = "--sqlbuild-launch-record=" diff --git a/src/sqlbuild/cli/entry/main/launched.py b/src/sqlbuild/cli/entry/main/launched.py index 69729dc592..8632729d51 100644 --- a/src/sqlbuild/cli/entry/main/launched.py +++ b/src/sqlbuild/cli/entry/main/launched.py @@ -2,7 +2,7 @@ import sys -from sqlbuild.cli.entry.constants import LAUNCH_RECORD_OPTION +from sqlbuild._native import LAUNCH_RECORD_OPTION from sqlbuild.cli.entry.main.entry import main as run_cli diff --git a/src/sqlbuild/cli/output/constants.py b/src/sqlbuild/cli/output/constants.py index dd8a0acf62..3e27072c58 100644 --- a/src/sqlbuild/cli/output/constants.py +++ b/src/sqlbuild/cli/output/constants.py @@ -1,3 +1,3 @@ """CLI execution output constants.""" -INTEGRATION_RESULT_PATH_ENV: str = "SQLBUILD_INTEGRATION_RESULT_PATH" +from sqlbuild._native import INTEGRATION_RESULT_PATH_ENV as INTEGRATION_RESULT_PATH_ENV diff --git a/src/sqlbuild/compiler/discovery/_helpers/filesystem/project_modules.py b/src/sqlbuild/compiler/discovery/_helpers/filesystem/project_modules.py index 76b7eca129..51216b3abe 100644 --- a/src/sqlbuild/compiler/discovery/_helpers/filesystem/project_modules.py +++ b/src/sqlbuild/compiler/discovery/_helpers/filesystem/project_modules.py @@ -13,6 +13,8 @@ from typing import get_type_hints from sqlbuild._native import ( + PYTHON_INIT_MODULE_STEM, + PYTHON_NODE_ROOT, DiscoveredProvider, DiscoveredProviderUsage, EventExporterDiscoveryError, @@ -20,10 +22,6 @@ PythonNodeDiscoveryError, project_relative_path, ) -from sqlbuild.compiler.discovery.constants import ( - PYTHON_INIT_MODULE_STEM, - PYTHON_NODE_ROOT, -) from sqlbuild.runtime.event_exporting.constants import EVENT_EXPORTER_EVENT_PARAMETER_NAME from sqlbuild.runtime.observability.models import LifecycleEvent diff --git a/src/sqlbuild/compiler/discovery/constants.py b/src/sqlbuild/compiler/discovery/constants.py deleted file mode 100644 index cb4fcc00e1..0000000000 --- a/src/sqlbuild/compiler/discovery/constants.py +++ /dev/null @@ -1,13 +0,0 @@ -"""Public discovery constants.""" - -from __future__ import annotations - -PROJECT_CONFIG_FILENAME: str = "sqlbuild_project.toml" - -PYTHON_INIT_MODULE_STEM: str = "__init__" -PYTHON_NODE_ROOT: str = "python" - - -SETTINGS_SECTION: str = "settings" -TABLE_PROMOTION_MODE_SETTING_KEY: str = "table_promotion_mode" -STAGED_PROMOTION_MODE: str = "staged" diff --git a/src/sqlbuild/compiler/fingerprints/constants.py b/src/sqlbuild/compiler/fingerprints/constants.py index 586c4174a5..5a2aed808a 100644 --- a/src/sqlbuild/compiler/fingerprints/constants.py +++ b/src/sqlbuild/compiler/fingerprints/constants.py @@ -2,24 +2,5 @@ from __future__ import annotations -FINGERPRINT_TABLE_NAME: str = "_sqlbuild_fingerprints" - -COLUMN_NODE_TYPE: str = "node_type" -COLUMN_NODE_NAME: str = "node_name" -COLUMN_TARGET_DATABASE: str = "target_database" -COLUMN_TARGET_SCHEMA: str = "target_schema" -COLUMN_TARGET_NAME: str = "target_name" -COLUMN_RUN_ID: str = "run_id" -COLUMN_DEFINITION_HASH: str = "definition_hash" -COLUMN_VERSION_HASH: str = "version_hash" -COLUMN_SCHEMA_FINGERPRINT: str = "schema_fingerprint" -COLUMN_DEFINITION_B64: str = "definition_b64" -COLUMN_METADATA_JSON_B64: str = "metadata_json_b64" -COLUMN_TIMESTAMP: str = "ts" - -NODE_TYPE_MODEL: str = "model" -NODE_TYPE_SEED: str = "seed" NODE_TYPE_HOOK: str = "hook" NODE_TYPE_QUERY_DIFF_ARTIFACT: str = "query_diff_artifact" - -AUDIT_GATE_METADATA_KEY: str = "audit_gate" diff --git a/src/sqlbuild/compiler/frontier/constants.py b/src/sqlbuild/compiler/frontier/constants.py deleted file mode 100644 index 791af21afd..0000000000 --- a/src/sqlbuild/compiler/frontier/constants.py +++ /dev/null @@ -1,6 +0,0 @@ -"""Compiler cache namespace and stage capture constants.""" - -from __future__ import annotations - -STAGE_CAPTURE_DIR_ENV_VAR: str = "SQLBUILD_COMPILER_STAGE_CAPTURE_DIR" -COMPILE_REUSE_DISABLE_ENV_VAR: str = "SQLBUILD_DISABLE_COMPILE_REUSE" diff --git a/src/sqlbuild/compiler/macro_bridge/_helpers/store_environment.py b/src/sqlbuild/compiler/macro_bridge/_helpers/store_environment.py index a22b1804b0..4b53025cbe 100644 --- a/src/sqlbuild/compiler/macro_bridge/_helpers/store_environment.py +++ b/src/sqlbuild/compiler/macro_bridge/_helpers/store_environment.py @@ -13,6 +13,7 @@ from pathlib import Path import sqlbuild._native as _native +from sqlbuild._native import STORE_TRACKED_ENVIRONMENT_PREFIXES, STORE_UNTRACKED_ENVIRONMENT_NAMES from sqlbuild.compiler.macro_bridge.constants import ( INSTALLED_DISTRIBUTION_SUFFIX, INSTALLED_PACKAGE_DIRECTORY_NAMES, @@ -22,8 +23,6 @@ MAIN_MODULE_NAME, MISSING_SEARCH_PATH_STAMP, PROJECT_ROOT_SEARCH_PATH_STAMP, - STORE_TRACKED_ENVIRONMENT_PREFIXES, - STORE_UNTRACKED_ENVIRONMENT_NAMES, ) from sqlbuild.compiler.macro_bridge.models import ModuleSources diff --git a/src/sqlbuild/compiler/macro_bridge/constants.py b/src/sqlbuild/compiler/macro_bridge/constants.py index f70fcd3439..0226aab70f 100644 --- a/src/sqlbuild/compiler/macro_bridge/constants.py +++ b/src/sqlbuild/compiler/macro_bridge/constants.py @@ -5,11 +5,6 @@ import itertools from contextvars import ContextVar -from sqlbuild.compiler.frontier.constants import ( - COMPILE_REUSE_DISABLE_ENV_VAR, - STAGE_CAPTURE_DIR_ENV_VAR, -) - ACTIVE_MACRO_BRIDGE: ContextVar[object | None] = ContextVar( "sqlbuild_active_macro_bridge", default=None ) @@ -24,10 +19,3 @@ INSTALLED_RECORD_FILE_NAME: str = "RECORD" INTERPRETER_MODULE_ORIGINS: frozenset[str] = frozenset({"built-in", "frozen"}) MAIN_MODULE_NAME: str = "__main__" -STORE_TRACKED_ENVIRONMENT_PREFIXES: tuple[str, ...] = ("SQLBUILD_", "SQB_") -STORE_UNTRACKED_ENVIRONMENT_NAMES: frozenset[str] = frozenset( - { - COMPILE_REUSE_DISABLE_ENV_VAR, - STAGE_CAPTURE_DIR_ENV_VAR, - } -) diff --git a/src/sqlbuild/compiler/migrations/constants.py b/src/sqlbuild/compiler/migrations/constants.py deleted file mode 100644 index 07187e2a3c..0000000000 --- a/src/sqlbuild/compiler/migrations/constants.py +++ /dev/null @@ -1,10 +0,0 @@ -"""Stable constants for append-only model migration state.""" - -from __future__ import annotations - -MIGRATION_TABLE_NAME: str = "_sqlbuild_migrations" -MIGRATION_WRITE_ATTEMPTS: int = 3 - -COLUMN_MIGRATION_TABLE_NAME: str = "_sqlbuild_column_migrations" - -OLD_NAME_VIEW_TABLE_NAME: str = "_sqlbuild_old_name_views" diff --git a/src/sqlbuild/compiler/planner/constants.py b/src/sqlbuild/compiler/planner/constants.py deleted file mode 100644 index fe462093f5..0000000000 --- a/src/sqlbuild/compiler/planner/constants.py +++ /dev/null @@ -1,9 +0,0 @@ -"""Planner domain constants.""" - -from __future__ import annotations - -MICROBATCH_START_SENTINEL: str = "__SQB_CURSOR_START__" -MICROBATCH_END_SENTINEL: str = "__SQB_CURSOR_END__" -MIGRATION_FINGERPRINT_METADATA_KEY: str = "migration_fingerprint" -SNAPSHOT_DEFAULT_VALID_FROM_COLUMN: str = "valid_from" -SNAPSHOT_DEFAULT_VALID_TO_COLUMN: str = "valid_to" diff --git a/src/sqlbuild/compiler/source_freshness/constants.py b/src/sqlbuild/compiler/source_freshness/constants.py index 971545a6c7..dc199c72ff 100644 --- a/src/sqlbuild/compiler/source_freshness/constants.py +++ b/src/sqlbuild/compiler/source_freshness/constants.py @@ -2,10 +2,9 @@ from __future__ import annotations +from sqlbuild._native import SOURCE_FRESHNESS_TABLE_NAME as SOURCE_FRESHNESS_TABLE_NAME from sqlbuild._native import StateSqlValueType -SOURCE_FRESHNESS_TABLE_NAME: str = "_sqlbuild_source_freshness" - COLUMN_SOURCE_NAME: str = "source_name" COLUMN_TARGET_DATABASE: str = "target_database" COLUMN_TARGET_SCHEMA: str = "target_schema" diff --git a/src/sqlbuild/compute_logs.py b/src/sqlbuild/compute_logs.py index f9581b03b2..bac1d9b9cf 100644 --- a/src/sqlbuild/compute_logs.py +++ b/src/sqlbuild/compute_logs.py @@ -2,11 +2,11 @@ from pathlib import Path -from sqlbuild._native import LocalFilesystemComputeLogStorage -from sqlbuild.runtime.compute_logs.constants import ( +from sqlbuild._native import ( COMPUTE_LOG_FORMAT_VERSION, DEFAULT_RETENTION_COUNT, MAX_READ_BYTES, + LocalFilesystemComputeLogStorage, ) from sqlbuild.runtime.compute_logs.exceptions import ( CaptureAlreadyExistsError, diff --git a/src/sqlbuild/cost/constants.py b/src/sqlbuild/cost/constants.py deleted file mode 100644 index 5e618485cd..0000000000 --- a/src/sqlbuild/cost/constants.py +++ /dev/null @@ -1,3 +0,0 @@ -"""Cost observability defaults.""" - -SQLBUILD_QUERY_TAG_APP: str = "sqlbuild" diff --git a/src/sqlbuild/cursor_algebra/constants.py b/src/sqlbuild/cursor_algebra/constants.py deleted file mode 100644 index 684a9e6354..0000000000 --- a/src/sqlbuild/cursor_algebra/constants.py +++ /dev/null @@ -1,12 +0,0 @@ -"""Canonical cursor grain metadata.""" - -from sqlbuild._native import CursorGrain - -GRAIN_ORDER: dict[CursorGrain, int] = { - CursorGrain.SECOND: 0, - CursorGrain.MINUTE: 1, - CursorGrain.HOUR: 2, - CursorGrain.DAY: 3, - CursorGrain.MONTH: 4, - CursorGrain.YEAR: 5, -} diff --git a/src/sqlbuild/execution_history.py b/src/sqlbuild/execution_history.py index a3674fdda6..f95f4f08cb 100644 --- a/src/sqlbuild/execution_history.py +++ b/src/sqlbuild/execution_history.py @@ -2,12 +2,12 @@ from collections.abc import Iterable +from sqlbuild._native import DEFAULT_PAGE_LIMIT, MAX_PAGE_LIMIT from sqlbuild._native import append_and_project as _append_and_project from sqlbuild._native import canonical_event_content as _canonical_event_content from sqlbuild._native import canonical_event_id as _canonical_event_id from sqlbuild._native import project_execution_runs as _project_execution_runs from sqlbuild._native import validate_history_page_limit as _validate_history_page_limit -from sqlbuild.runtime.execution_history.constants import DEFAULT_PAGE_LIMIT, MAX_PAGE_LIMIT from sqlbuild.runtime.execution_history.exceptions import ( ExecutionHistoryStorageError, IntegrityConflictError, diff --git a/src/sqlbuild/executor/build/_helpers/scheduler.py b/src/sqlbuild/executor/build/_helpers/scheduler.py index ecb87b2763..fcf165264f 100644 --- a/src/sqlbuild/executor/build/_helpers/scheduler.py +++ b/src/sqlbuild/executor/build/_helpers/scheduler.py @@ -5,6 +5,8 @@ from collections.abc import Callable, Mapping from sqlbuild._native import ( + MICROBATCH_END_SENTINEL, + MICROBATCH_START_SENTINEL, CompiledObjectKey, IncrementalMode, MaterializationType, @@ -18,7 +20,6 @@ ) from sqlbuild.adapter.contract.models import RelationInfo from sqlbuild.adapter.contract.types import TablePromotionMode -from sqlbuild.compiler.planner.constants import MICROBATCH_END_SENTINEL, MICROBATCH_START_SENTINEL from sqlbuild.errors.contracts.exceptions import ExecutorInputError from sqlbuild.executor.build.constants import ( BUILD_CUSTOM_MATERIALIZATION_MISSING_CODE, diff --git a/src/sqlbuild/executor/clone/_helpers/fingerprinting.py b/src/sqlbuild/executor/clone/_helpers/fingerprinting.py index 278329ffbb..5c2abd241e 100644 --- a/src/sqlbuild/executor/clone/_helpers/fingerprinting.py +++ b/src/sqlbuild/executor/clone/_helpers/fingerprinting.py @@ -9,6 +9,9 @@ from typing import Any from sqlbuild._native import ( + FINGERPRINT_TABLE_NAME, + NODE_TYPE_MODEL, + NODE_TYPE_SEED, Fingerprint, FingerprintSet, MaterializationType, @@ -20,11 +23,6 @@ from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter from sqlbuild.adapter.contract.models import RelationLookup from sqlbuild.adapter.relations.main.relation_lookup import build_relation_lookup -from sqlbuild.compiler.fingerprints.constants import ( - FINGERPRINT_TABLE_NAME, - NODE_TYPE_MODEL, - NODE_TYPE_SEED, -) from sqlbuild.executor.clone.models import CloneExecutionResult from sqlbuild.executor.clone.types import ( CloneAction, diff --git a/src/sqlbuild/executor/diff/_helpers/query_artifacts.py b/src/sqlbuild/executor/diff/_helpers/query_artifacts.py index 2d88fa29dc..c22d58f774 100644 --- a/src/sqlbuild/executor/diff/_helpers/query_artifacts.py +++ b/src/sqlbuild/executor/diff/_helpers/query_artifacts.py @@ -8,6 +8,7 @@ from typing import Any from sqlbuild._native import ( + FINGERPRINT_TABLE_NAME, Fingerprint, FingerprintSet, build_query_diff_artifact_name, @@ -20,10 +21,7 @@ from sqlbuild.adapter.contract.classes.statement_recorder import StatementRecorder from sqlbuild.adapter.contract.models import ColumnInfo, RelationInfo, RelationLookup from sqlbuild.adapter.relations.main.relation_lookup import build_relation_lookup -from sqlbuild.compiler.fingerprints.constants import ( - FINGERPRINT_TABLE_NAME, - NODE_TYPE_QUERY_DIFF_ARTIFACT, -) +from sqlbuild.compiler.fingerprints.constants import NODE_TYPE_QUERY_DIFF_ARTIFACT from sqlbuild.errors.contracts.exceptions import ExecutorInputError from sqlbuild.executor.diff.models import ( QueryDiffArtifact, diff --git a/src/sqlbuild/executor/migrations/_helpers/columns.py b/src/sqlbuild/executor/migrations/_helpers/columns.py index 57284c5fb2..612f484223 100644 --- a/src/sqlbuild/executor/migrations/_helpers/columns.py +++ b/src/sqlbuild/executor/migrations/_helpers/columns.py @@ -6,6 +6,7 @@ from typing import Any from sqlbuild._native import ( + MIGRATION_WRITE_ATTEMPTS, ColumnMigrationDecision, ColumnMigrationEvent, ColumnMigrationPlanEntry, @@ -16,7 +17,6 @@ from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter from sqlbuild.adapter.contract.classes.statement_recorder import StatementRecorder from sqlbuild.adapter.contract.models import ColumnInfo -from sqlbuild.compiler.migrations.constants import MIGRATION_WRITE_ATTEMPTS from sqlbuild.executor.migrations._helpers.relation_for_location import ( migration_relation_for_location, ) diff --git a/src/sqlbuild/executor/migrations/_helpers/old_name_facts.py b/src/sqlbuild/executor/migrations/_helpers/old_name_facts.py index 12186bec4d..4f71d0f5b3 100644 --- a/src/sqlbuild/executor/migrations/_helpers/old_name_facts.py +++ b/src/sqlbuild/executor/migrations/_helpers/old_name_facts.py @@ -6,6 +6,7 @@ from typing import Any from sqlbuild._native import ( + MIGRATION_WRITE_ATTEMPTS, MigrationRelation, OldNameViewEvent, OldNameViewEventType, @@ -16,7 +17,6 @@ write_migration_state_event, ) from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter -from sqlbuild.compiler.migrations.constants import MIGRATION_WRITE_ATTEMPTS from sqlbuild.executor.migrations._helpers.relation_for_location import ( migration_relation_for_location, ) diff --git a/src/sqlbuild/executor/migrations/_helpers/old_name_views.py b/src/sqlbuild/executor/migrations/_helpers/old_name_views.py index 996d1072af..5d11472a84 100644 --- a/src/sqlbuild/executor/migrations/_helpers/old_name_views.py +++ b/src/sqlbuild/executor/migrations/_helpers/old_name_views.py @@ -9,6 +9,7 @@ from typing import Any from sqlbuild._native import ( + COLUMN_MIGRATION_TABLE_NAME, ColumnMigrationEvent, CompiledRelationLocation, Duration, @@ -31,7 +32,6 @@ resolve_qualified_name_parts, ) from sqlbuild.adapter.type_system.main.normalize_relation_type import normalize_relation_type -from sqlbuild.compiler.migrations.constants import COLUMN_MIGRATION_TABLE_NAME from sqlbuild.errors.contracts.exceptions import ExecutorInputError from sqlbuild.executor.migrations._helpers.old_name_facts import ( old_name_fact, diff --git a/src/sqlbuild/executor/migrations/_helpers/promotion.py b/src/sqlbuild/executor/migrations/_helpers/promotion.py index 033bdc4c82..5c30332937 100644 --- a/src/sqlbuild/executor/migrations/_helpers/promotion.py +++ b/src/sqlbuild/executor/migrations/_helpers/promotion.py @@ -6,6 +6,7 @@ from typing import Any from sqlbuild._native import ( + MIGRATION_WRITE_ATTEMPTS, MigrationDecision, MigrationEvent, MigrationRelation, @@ -18,7 +19,6 @@ ) from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter from sqlbuild.adapter.contract.classes.statement_recorder import StatementRecorder -from sqlbuild.compiler.migrations.constants import MIGRATION_WRITE_ATTEMPTS from sqlbuild.executor.migrations._helpers.old_name_facts import ( old_name_fact, old_name_state_table_sql, diff --git a/src/sqlbuild/executor/run/_helpers/materializations/snapshot.py b/src/sqlbuild/executor/run/_helpers/materializations/snapshot.py index 97b93b887a..0dcf20cd6d 100644 --- a/src/sqlbuild/executor/run/_helpers/materializations/snapshot.py +++ b/src/sqlbuild/executor/run/_helpers/materializations/snapshot.py @@ -5,6 +5,8 @@ from typing import Any from sqlbuild._native import ( + SNAPSHOT_DEFAULT_VALID_FROM_COLUMN, + SNAPSHOT_DEFAULT_VALID_TO_COLUMN, HistoricalInput, InitialValidFrom, ModelPlanEntry, @@ -31,10 +33,6 @@ ) from sqlbuild.adapter.type_system.main.normalize_type import normalize_type from sqlbuild.adapter.type_system.main.types_equal import types_equal -from sqlbuild.compiler.planner.constants import ( - SNAPSHOT_DEFAULT_VALID_FROM_COLUMN, - SNAPSHOT_DEFAULT_VALID_TO_COLUMN, -) from sqlbuild.diagnostics.main.diagnostics_context import diagnostics_context from sqlbuild.errors.contracts.exceptions import ExecutorInputError from sqlbuild.executor.auditing.models import AuditExecutionResult diff --git a/src/sqlbuild/executor/run/_helpers/reuse/fingerprint_metadata.py b/src/sqlbuild/executor/run/_helpers/reuse/fingerprint_metadata.py index 66b600ba3a..e9dc878139 100644 --- a/src/sqlbuild/executor/run/_helpers/reuse/fingerprint_metadata.py +++ b/src/sqlbuild/executor/run/_helpers/reuse/fingerprint_metadata.py @@ -8,14 +8,14 @@ from typing import cast from sqlbuild._native import ( + AUDIT_GATE_METADATA_KEY, + MIGRATION_FINGERPRINT_METADATA_KEY, AuditGateIdentity, AuditIdentity, AuditPlanEntry, build_audit_gate_identity, ) from sqlbuild.compiler.auditing.types import AuditOutcome, AuditSeverity -from sqlbuild.compiler.fingerprints.constants import AUDIT_GATE_METADATA_KEY -from sqlbuild.compiler.planner.constants import MIGRATION_FINGERPRINT_METADATA_KEY from sqlbuild.executor.auditing.models import AuditExecutionResult from sqlbuild.executor.run.models import ( AuditGateMetadata, diff --git a/src/sqlbuild/executor/run/_helpers/reuse/fingerprinting.py b/src/sqlbuild/executor/run/_helpers/reuse/fingerprinting.py index 7eec2be047..b32c74fb52 100644 --- a/src/sqlbuild/executor/run/_helpers/reuse/fingerprinting.py +++ b/src/sqlbuild/executor/run/_helpers/reuse/fingerprinting.py @@ -7,6 +7,7 @@ from typing import Any from sqlbuild._native import ( + NODE_TYPE_MODEL, AuditPlanEntry, Fingerprint, ModelPlanEntry, @@ -14,7 +15,6 @@ write_fingerprint, ) from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter -from sqlbuild.compiler.fingerprints.constants import NODE_TYPE_MODEL from sqlbuild.executor.auditing.models import AuditExecutionResult from sqlbuild.executor.run._helpers.reuse.fingerprint_metadata import ( model_fingerprint_metadata_with_audit_gate, diff --git a/src/sqlbuild/executor/run/_helpers/validation/cursor_bounds.py b/src/sqlbuild/executor/run/_helpers/validation/cursor_bounds.py index e0e68cbaa4..5879375918 100644 --- a/src/sqlbuild/executor/run/_helpers/validation/cursor_bounds.py +++ b/src/sqlbuild/executor/run/_helpers/validation/cursor_bounds.py @@ -9,6 +9,10 @@ from typing import Any from sqlbuild._native import ( + GRAIN_ORDER, + MICROBATCH_END_SENTINEL, + MICROBATCH_START_SENTINEL, + ZERO_DAY_CURSOR_DURATION, AlignedInterval, BackfillAction, CursorBounds, @@ -50,14 +54,11 @@ from sqlbuild._native import resolve_future_cursor_safety as apply_future_cursor_safety from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter from sqlbuild.adapter.relations.main.cached_relation_exists import cached_relation_exists -from sqlbuild.compiler.planner.constants import MICROBATCH_END_SENTINEL, MICROBATCH_START_SENTINEL -from sqlbuild.cursor_algebra.constants import GRAIN_ORDER from sqlbuild.cursor_algebra.types import BoundSentinel, CursorScalar from sqlbuild.errors.contracts.exceptions import ExecutorInputError from sqlbuild.executor.run.exceptions import EmptyCursorInputsError from sqlbuild.executor.run.models import RuntimeCursorInputRelation, RuntimeCursorSpec from sqlbuild.executor.run.types import RuntimeCursorWatermarkMode, WatermarkResolver -from sqlbuild.spec.contracts.constants import ZERO_DAY_CURSOR_DURATION def has_runtime_owned_cursor_watermarks( diff --git a/src/sqlbuild/executor/run/main/_execute.py b/src/sqlbuild/executor/run/main/_execute.py index f7545193b5..8562c2145c 100644 --- a/src/sqlbuild/executor/run/main/_execute.py +++ b/src/sqlbuild/executor/run/main/_execute.py @@ -5,17 +5,19 @@ from dataclasses import replace from typing import Any -from sqlbuild._native import Fingerprint, ModelPlanEntry, setting_help -from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter -from sqlbuild.adapter.contract.classes.statement_recorder import StatementRecorder -from sqlbuild.adapter.contract.models import ColumnInfo -from sqlbuild.adapter.contract.types import TablePromotionMode -from sqlbuild.compiler.discovery.constants import ( +from sqlbuild._native import ( PROJECT_CONFIG_FILENAME, SETTINGS_SECTION, STAGED_PROMOTION_MODE, TABLE_PROMOTION_MODE_SETTING_KEY, + Fingerprint, + ModelPlanEntry, + setting_help, ) +from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter +from sqlbuild.adapter.contract.classes.statement_recorder import StatementRecorder +from sqlbuild.adapter.contract.models import ColumnInfo +from sqlbuild.adapter.contract.types import TablePromotionMode from sqlbuild.diagnostics.main.diagnostics_context import diagnostics_context from sqlbuild.errors.contracts.exceptions import ExecutorInputError from sqlbuild.executor.auditing.models import AuditExecutionResult diff --git a/src/sqlbuild/executor/seed/_helpers/fingerprinting.py b/src/sqlbuild/executor/seed/_helpers/fingerprinting.py index 0172752a18..3bb9f4842d 100644 --- a/src/sqlbuild/executor/seed/_helpers/fingerprinting.py +++ b/src/sqlbuild/executor/seed/_helpers/fingerprinting.py @@ -6,9 +6,8 @@ from datetime import UTC, datetime from typing import Any -from sqlbuild._native import Fingerprint, SeedPlanEntry, write_fingerprint +from sqlbuild._native import NODE_TYPE_SEED, Fingerprint, SeedPlanEntry, write_fingerprint from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter -from sqlbuild.compiler.fingerprints.constants import NODE_TYPE_SEED def try_write_seed_fingerprint( diff --git a/src/sqlbuild/rule_engine/_helpers/engine/_custom_catalogue.py b/src/sqlbuild/rule_engine/_helpers/engine/_custom_catalogue.py index c60ca2c777..58cd57f4b7 100644 --- a/src/sqlbuild/rule_engine/_helpers/engine/_custom_catalogue.py +++ b/src/sqlbuild/rule_engine/_helpers/engine/_custom_catalogue.py @@ -9,12 +9,12 @@ from pathlib import Path from types import ModuleType -from sqlbuild._native import Rule, RulesError +from sqlbuild._native import PYTHON_INIT_MODULE_STEM, Rule, RulesError from sqlbuild.rule_engine._helpers.engine.definition import ( resolve_rule_signature, rule_from_value, ) -from sqlbuild.rule_engine.constants import INIT_MODULE_NAME, RULE_DECORATOR_TOKEN +from sqlbuild.rule_engine.constants import RULE_DECORATOR_TOKEN def load_custom_rules(*, project_dir: Path) -> tuple[Rule, ...]: @@ -50,7 +50,7 @@ def load_custom_rules(*, project_dir: Path) -> tuple[Rule, ...]: def _rule_module_name(*, file_path: Path, project_dir: Path) -> str: relative: Path = file_path.relative_to(project_dir.resolve()).with_suffix("") parts: tuple[str, ...] = relative.parts - if parts[-1] == INIT_MODULE_NAME: + if parts[-1] == PYTHON_INIT_MODULE_STEM: parts = parts[:-1] return ".".join(parts) diff --git a/src/sqlbuild/rule_engine/_helpers/engine/rule_harness.py b/src/sqlbuild/rule_engine/_helpers/engine/rule_harness.py index affcc7f3d4..e1662febb2 100644 --- a/src/sqlbuild/rule_engine/_helpers/engine/rule_harness.py +++ b/src/sqlbuild/rule_engine/_helpers/engine/rule_harness.py @@ -7,6 +7,7 @@ from tempfile import TemporaryDirectory from sqlbuild._native import ( + MIN_CUSTOM_RULE_TEST_CASES, DiscoveredProjectInputs, Rule, RuleAssertionError, @@ -26,7 +27,6 @@ resolve_rule_signature, rule_from_value, ) -from sqlbuild.rule_engine.constants import MIN_CUSTOM_RULE_TEST_CASES from sqlbuild.rule_engine.types import RuleCheck diff --git a/src/sqlbuild/rule_engine/_helpers/host/custom_evaluation.py b/src/sqlbuild/rule_engine/_helpers/host/custom_evaluation.py index 389d46c6b9..1345c0b19d 100644 --- a/src/sqlbuild/rule_engine/_helpers/host/custom_evaluation.py +++ b/src/sqlbuild/rule_engine/_helpers/host/custom_evaluation.py @@ -10,6 +10,7 @@ from pathlib import Path from sqlbuild._native import ( + CUSTOM_RULE_PROJECT_SUBJECT, CustomHostPartition, CustomRuleEvaluation, CustomRuleRun, @@ -35,10 +36,7 @@ build_rule_fact_views, ) from sqlbuild.rule_engine.classes.runtime_guard import RuntimeGuard -from sqlbuild.rule_engine.constants import ( - CUSTOM_HOST_CANCELLED_MESSAGE, - CUSTOM_RULE_PROJECT_SUBJECT, -) +from sqlbuild.rule_engine.constants import CUSTOM_HOST_CANCELLED_MESSAGE from sqlbuild.rule_engine.types import CustomRulePlan type _FindingKey = tuple[str, str, int, int, str, str] diff --git a/src/sqlbuild/rule_engine/_helpers/host/custom_host.py b/src/sqlbuild/rule_engine/_helpers/host/custom_host.py index 9f6cad703d..b00c6ecb87 100644 --- a/src/sqlbuild/rule_engine/_helpers/host/custom_host.py +++ b/src/sqlbuild/rule_engine/_helpers/host/custom_host.py @@ -15,6 +15,9 @@ from typing import Any, BinaryIO from sqlbuild._native import ( + CUSTOM_HOST_LAUNCH_MODULE, + CUSTOM_HOST_PROTOCOL_VERSION, + CUSTOM_HOST_RUNTIME_VERSION, CompiledProject, CustomHostPartition, CustomRuleRun, @@ -37,10 +40,7 @@ CUSTOM_HOST_INPUT_POLL_SECONDS, CUSTOM_HOST_INPUT_TUPLE_SIZE, CUSTOM_HOST_INPUT_WAIT_SECONDS, - CUSTOM_HOST_LAUNCH_MODULE, CUSTOM_HOST_MAX_TRACKED_READS, - CUSTOM_HOST_PROTOCOL_VERSION, - CUSTOM_HOST_RUNTIME_VERSION, ) from sqlbuild.rule_engine.types import CustomRulePlan diff --git a/src/sqlbuild/rule_engine/classes/contract_facts.py b/src/sqlbuild/rule_engine/classes/contract_facts.py index b4b3a35abd..454bc611e4 100644 --- a/src/sqlbuild/rule_engine/classes/contract_facts.py +++ b/src/sqlbuild/rule_engine/classes/contract_facts.py @@ -1,8 +1,7 @@ """Contract facts for custom rules.""" -from sqlbuild._native import Model +from sqlbuild._native import CONTRACT_ENFORCED, Model from sqlbuild.compiler.compile.types import CompiledModelView, CompiledProjectView -from sqlbuild.rule_engine.constants import CONTRACT_ENFORCED class ContractFacts: diff --git a/src/sqlbuild/rule_engine/classes/sql_document.py b/src/sqlbuild/rule_engine/classes/sql_document.py index 774b3c9c6e..e609dfa8a7 100644 --- a/src/sqlbuild/rule_engine/classes/sql_document.py +++ b/src/sqlbuild/rule_engine/classes/sql_document.py @@ -2,9 +2,8 @@ from typing import Any -from sqlbuild._native import RuleUsageError, SqlNode +from sqlbuild._native import AST_CTE_KIND, RuleUsageError, SqlNode from sqlbuild.compiler.sql_analysis.main.import_polyglot_sql import import_polyglot_sql -from sqlbuild.rule_engine.constants import AST_CTE_KIND class SqlDocument: diff --git a/src/sqlbuild/rule_engine/constants.py b/src/sqlbuild/rule_engine/constants.py index e95a2147b5..ee72f74f0d 100644 --- a/src/sqlbuild/rule_engine/constants.py +++ b/src/sqlbuild/rule_engine/constants.py @@ -1,9 +1,6 @@ """Rules engine constants.""" -CUSTOM_HOST_PROTOCOL_VERSION: int = 1 -CUSTOM_HOST_RUNTIME_VERSION: str = "sqlbuild-rules-custom-v2" CUSTOM_HOST_MAX_TRACKED_READS: int = 100_000 -CUSTOM_RULE_PROJECT_SUBJECT: str = "" CUSTOM_HOST_INPUT_TUPLE_SIZE: int = 2 SHARED_FACT_DATACLASS_TAG: str = "\0dataclass" CUSTOM_HOST_INPUT_POLL_SECONDS: float = 0.002 @@ -11,7 +8,6 @@ CUSTOM_HOST_MODULE: str = "sqlbuild.rule_engine._helpers.host.custom_host" CUSTOM_HOST_CANCELLED_EXIT_CODE: int = 75 CUSTOM_HOST_CANCELLED_MESSAGE: str = "custom host cancelled after another host failed" -CUSTOM_HOST_LAUNCH_MODULE: str = "sqlbuild.rule_engine._helpers.host.custom_host_launch" CUSTOM_HOST_HASH_SEED: str = "0" CUSTOM_HOST_EXEC_OS_NAME: str = "posix" CUSTOM_HOST_STARTUP_ENVIRONMENT: tuple[str, ...] = ( @@ -57,15 +53,9 @@ } ) -AST_CTE_KIND: str = "cte" -CONTRACT_ENFORCED: str = "enforced" - - RULE_DECORATOR_TOKEN: str = "@rule" -INIT_MODULE_NAME: str = "__init__" PARENT_DIRECTORY_TOKEN: str = ".." -MIN_CUSTOM_RULE_TEST_CASES: str = "min_custom_rule_test_cases" FACT_PROJECT_MODELS: str = "project.models" FACT_PROJECT_SOURCES: str = "project.sources" FACT_PROJECT_SEEDS: str = "project.seeds" diff --git a/src/sqlbuild/runtime/compute_logs/constants.py b/src/sqlbuild/runtime/compute_logs/constants.py index 4b6f540a00..b25b998008 100644 --- a/src/sqlbuild/runtime/compute_logs/constants.py +++ b/src/sqlbuild/runtime/compute_logs/constants.py @@ -1,11 +1,4 @@ """Compute log format and policy constants.""" -COMPUTE_LOG_FORMAT_VERSION: int = 1 -DEFAULT_RETENTION_COUNT: int = 20 -MAX_READ_BYTES: int = 1_048_576 -METADATA_FILE_NAME: str = "metadata.json" -COMPLETE_FILE_NAME: str = "complete" -PRUNE_LOCK_FILE_NAME: str = ".prune.lock" -COMPLETED_AT_FIELD: str = "completed_at" SQL_LOG_RECORD_FIELD: str = "sqlbuild_sql" REDACTED_SQL_LOG_MESSAGE: str = "SQL diagnostic omitted" diff --git a/src/sqlbuild/runtime/compute_logs/types.py b/src/sqlbuild/runtime/compute_logs/types.py index 3de00c2f04..f91a061af0 100644 --- a/src/sqlbuild/runtime/compute_logs/types.py +++ b/src/sqlbuild/runtime/compute_logs/types.py @@ -6,7 +6,7 @@ from types import TracebackType from typing import TYPE_CHECKING, Protocol, Self -from sqlbuild.runtime.compute_logs.constants import MAX_READ_BYTES +from sqlbuild._native import MAX_READ_BYTES if TYPE_CHECKING: from sqlbuild.runtime.compute_logs.models import ( diff --git a/src/sqlbuild/runtime/execution_history/constants.py b/src/sqlbuild/runtime/execution_history/constants.py deleted file mode 100644 index 06f7e8cad7..0000000000 --- a/src/sqlbuild/runtime/execution_history/constants.py +++ /dev/null @@ -1,9 +0,0 @@ -"""Execution history contract limits and schema defaults.""" - -DEFAULT_PAGE_LIMIT: int = 100 -MAX_PAGE_LIMIT: int = 1000 -CURRENT_EVENT_LOG_SCHEMA_VERSION: int = 1 -CURRENT_RUN_STORAGE_SCHEMA_VERSION: int = 1 -RUN_STARTED_EVENT_TYPE: str = "run_started" -RUN_COMPLETED_EVENT_TYPE: str = "run_completed" -RUN_FAILED_EVENT_TYPE: str = "run_failed" diff --git a/src/sqlbuild/runtime/execution_history/types.py b/src/sqlbuild/runtime/execution_history/types.py index b74bbd3984..5349b10614 100644 --- a/src/sqlbuild/runtime/execution_history/types.py +++ b/src/sqlbuild/runtime/execution_history/types.py @@ -7,7 +7,7 @@ from types import TracebackType from typing import TYPE_CHECKING, Protocol, Self -from sqlbuild.runtime.execution_history.constants import DEFAULT_PAGE_LIMIT +from sqlbuild._native import DEFAULT_PAGE_LIMIT if TYPE_CHECKING: from sqlbuild.runtime.execution_history.models import ( diff --git a/src/sqlbuild/runtime/output_capture/constants.py b/src/sqlbuild/runtime/output_capture/constants.py index 440d894096..9f12858e26 100644 --- a/src/sqlbuild/runtime/output_capture/constants.py +++ b/src/sqlbuild/runtime/output_capture/constants.py @@ -2,39 +2,7 @@ from importlib.metadata import version -DEFAULT_OUTPUT_BATCH_SIZE: int = 64 -DEFAULT_OUTPUT_MAX_RECORD_BYTES: int = 64 * 1024 -DEFAULT_OUTPUT_FLUSH_INTERVAL_SECONDS: float = 0.25 -DEFAULT_OUTPUT_QUEUE_CAPACITY: int = 1024 -DEFAULT_OUTPUT_SHUTDOWN_TIMEOUT_SECONDS: float = 2.0 -MIN_OUTPUT_RECORD_BYTES: int = 4 -UTF8_CONTINUATION_BYTE_MASK: int = 0xC0 -UTF8_CONTINUATION_BYTE_TAG: int = 0x80 -INVOCATION_CONTEXT_ENV: str = "SQLBUILD_INVOCATION_CONTEXT_JSON" -MAX_INVOCATION_CONTEXT_BYTES: int = 16 * 1024 -MAX_INVOCATION_CONTEXT_DEPTH: int = 8 -CURRENT_COMMAND_OUTPUT_SCHEMA_VERSION: int = 1 -COMMAND_OUTPUT_RECORD_TYPE: str = "command_output" -COMMAND_OUTPUT_LOSS_RECORD_TYPE: str = "command_output_loss" +from sqlbuild._native import INVOCATION_CONTEXT_ENV as INVOCATION_CONTEXT_ENV + COMMAND_OUTPUT_SINK_RECORD_PARAMETER_NAME: str = "record" -COMMAND_OUTPUT_ENVELOPE_FIELDS: frozenset[str] = frozenset( - { - "record_id", - "record_type", - "schema_version", - "producer", - "producer_version", - "occurred_at", - "invocation_id", - "run_id", - "sequence", - "stream", - "message", - "chunk_index", - "chunk_count", - "priority", - "dropped_records", - "external_context", - } -) SQLBUILD_COMMAND_OUTPUT_PRODUCER_VERSION: str = version("sqlbuild") diff --git a/src/sqlbuild/runtime/output_capture/models.py b/src/sqlbuild/runtime/output_capture/models.py index 7675bcda3b..a3e508bbc3 100644 --- a/src/sqlbuild/runtime/output_capture/models.py +++ b/src/sqlbuild/runtime/output_capture/models.py @@ -6,12 +6,12 @@ from dataclasses import dataclass, field from datetime import datetime -from sqlbuild._native import freeze_command_output_record -from sqlbuild.runtime.output_capture.constants import ( +from sqlbuild._native import ( COMMAND_OUTPUT_RECORD_TYPE, CURRENT_COMMAND_OUTPUT_SCHEMA_VERSION, - SQLBUILD_COMMAND_OUTPUT_PRODUCER_VERSION, + freeze_command_output_record, ) +from sqlbuild.runtime.output_capture.constants import SQLBUILD_COMMAND_OUTPUT_PRODUCER_VERSION from sqlbuild.runtime.output_capture.types import CommandOutputStream, OutputRecordPriority diff --git a/src/sqlbuild/spec/contracts/constants.py b/src/sqlbuild/spec/contracts/constants.py index 29539a90e6..6626f8339b 100644 --- a/src/sqlbuild/spec/contracts/constants.py +++ b/src/sqlbuild/spec/contracts/constants.py @@ -5,4 +5,3 @@ from sqlbuild._native import SeedCsvSettings DEFAULT_SEED_CSV_SETTINGS: SeedCsvSettings = SeedCsvSettings() -ZERO_DAY_CURSOR_DURATION: str = "0d" diff --git a/tests/conftest.py b/tests/conftest.py index d1b10120dc..7660cdbf82 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -6,9 +6,7 @@ import pytest -from sqlbuild.compiler.frontier.constants import ( - COMPILE_REUSE_DISABLE_ENV_VAR as REUSE_DISABLE_ENV_VAR, -) +from sqlbuild._native import COMPILE_REUSE_DISABLE_ENV_VAR as REUSE_DISABLE_ENV_VAR def pytest_configure(config: pytest.Config) -> None: diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/compile/fast_lineage/helpers.py b/tests/e2e/src/sqlbuild/cli/commands/main/compile/fast_lineage/helpers.py index 542142b45c..a3222d6af4 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/compile/fast_lineage/helpers.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/compile/fast_lineage/helpers.py @@ -7,9 +7,7 @@ from pathlib import Path from typing import NamedTuple, cast -from sqlbuild.compiler.frontier.constants import ( - COMPILE_REUSE_DISABLE_ENV_VAR as REUSE_DISABLE_ENV_VAR, -) +from sqlbuild._native import COMPILE_REUSE_DISABLE_ENV_VAR as REUSE_DISABLE_ENV_VAR from tests.e2e.src.sqlbuild.cli.commands.main.compile.helpers import run_installed_sqb _TIMING: re.Pattern[str] = re.compile(r" \(\d+\.\d+s\)") diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/compile/helpers.py b/tests/e2e/src/sqlbuild/cli/commands/main/compile/helpers.py index 86149d8555..fd248cbac8 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/compile/helpers.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/compile/helpers.py @@ -40,14 +40,12 @@ semantic_compile_fingerprint, ) from scripts.compiler_differential.constants import FAILURE_BASE_FILES +from sqlbuild._native import COMPILE_REUSE_DISABLE_ENV_VAR as REUSE_DISABLE_ENV_VAR from sqlbuild._native import ContractValidationResult, compiler_cache_directory from sqlbuild._native import run_cli as main from sqlbuild.adapter.contract.classes.duckdb_backed_adapter import DuckDbBackedAdapter from sqlbuild.adapter.contract.types import TypeDialect from sqlbuild.compiler.compile.types import CompiledProjectView -from sqlbuild.compiler.frontier.constants import ( - COMPILE_REUSE_DISABLE_ENV_VAR as REUSE_DISABLE_ENV_VAR, -) from sqlbuild.compiler.macro_bridge.constants import MACRO_CALL_STORE_FILE_NAME from sqlbuild.observability import EventDispatcher, LifecycleEvent from tests.e2e.src.sqlbuild.cli.commands.main.compile._test_types import ( diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/compile/semantic_checks/helpers.py b/tests/e2e/src/sqlbuild/cli/commands/main/compile/semantic_checks/helpers.py index 22e2a5d2bb..471586bd10 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/compile/semantic_checks/helpers.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/compile/semantic_checks/helpers.py @@ -6,9 +6,7 @@ from pathlib import Path from typing import Any, NamedTuple, cast -from sqlbuild.compiler.frontier.constants import ( - COMPILE_REUSE_DISABLE_ENV_VAR as REUSE_DISABLE_ENV_VAR, -) +from sqlbuild._native import COMPILE_REUSE_DISABLE_ENV_VAR as REUSE_DISABLE_ENV_VAR from tests.e2e.src.sqlbuild.cli.commands.main.compile.helpers import run_installed_sqb diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/scenario/helpers.py b/tests/e2e/src/sqlbuild/cli/commands/main/scenario/helpers.py index 7706e484ee..6440dbe38c 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/scenario/helpers.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/scenario/helpers.py @@ -14,10 +14,8 @@ from pathlib import Path from typing import IO, NamedTuple, cast +from sqlbuild._native import COMPILE_REUSE_DISABLE_ENV_VAR as REUSE_DISABLE_ENV_VAR from sqlbuild._native import ScenarioArtifactKind -from sqlbuild.compiler.frontier.constants import ( - COMPILE_REUSE_DISABLE_ENV_VAR as REUSE_DISABLE_ENV_VAR, -) from sqlbuild.executor.scenario._helpers.snapshots.core import ( build_scenario_snapshot_input_fingerprint, ) diff --git a/tests/integration/src/sqlbuild/adapters/bigquery/test_client.py b/tests/integration/src/sqlbuild/adapters/bigquery/test_client.py index 7a459fbc21..593f0693a7 100644 --- a/tests/integration/src/sqlbuild/adapters/bigquery/test_client.py +++ b/tests/integration/src/sqlbuild/adapters/bigquery/test_client.py @@ -8,6 +8,7 @@ import pytest from sqlbuild._native import ( + FINGERPRINT_TABLE_NAME, Fingerprint, FingerprintSet, SourceFreshnessValueKind, @@ -30,7 +31,6 @@ ) from sqlbuild.adapter.contract.types import FunctionNullabilityRule from sqlbuild.adapters.bigquery.classes.bigquery_adapter import BigQueryAdapter -from sqlbuild.compiler.fingerprints.constants import FINGERPRINT_TABLE_NAME from sqlbuild.compiler.lineage.types import InferredNullability from tests.integration.src.sqlbuild.adapters.bigquery._test_types import ( BigQueryBuildFlowTestCase, diff --git a/tests/integration/src/sqlbuild/adapters/databricks/test_client.py b/tests/integration/src/sqlbuild/adapters/databricks/test_client.py index 8ffa7946b9..415ed47ebe 100644 --- a/tests/integration/src/sqlbuild/adapters/databricks/test_client.py +++ b/tests/integration/src/sqlbuild/adapters/databricks/test_client.py @@ -7,6 +7,7 @@ import pytest from sqlbuild._native import ( + FINGERPRINT_TABLE_NAME, Fingerprint, FingerprintSet, SourceFreshnessValueKind, @@ -29,7 +30,6 @@ ) from sqlbuild.adapter.contract.types import FunctionNullabilityRule from sqlbuild.adapters.databricks.classes.databricks_adapter import DatabricksAdapter -from sqlbuild.compiler.fingerprints.constants import FINGERPRINT_TABLE_NAME from sqlbuild.compiler.lineage.types import InferredNullability from tests.integration.src.sqlbuild.adapters.databricks._test_types import ( DatabricksBuildFlowTestCase, diff --git a/tests/integration/src/sqlbuild/cli/commands/main/compile_outputs/helpers.py b/tests/integration/src/sqlbuild/cli/commands/main/compile_outputs/helpers.py index 6dab1de89a..9515c5c232 100644 --- a/tests/integration/src/sqlbuild/cli/commands/main/compile_outputs/helpers.py +++ b/tests/integration/src/sqlbuild/cli/commands/main/compile_outputs/helpers.py @@ -11,15 +11,16 @@ import pytest import sqlbuild._native as native_module -from sqlbuild._native import PreparedCompileArtifacts, compiler_cache_directory +from sqlbuild._native import COMPILE_REUSE_DISABLE_ENV_VAR as REUSE_DISABLE_ENV_VAR +from sqlbuild._native import ( + STAGE_CAPTURE_DIR_ENV_VAR, + PreparedCompileArtifacts, + compiler_cache_directory, +) from sqlbuild._native import run_cli as main from sqlbuild.cli.compile_reuse._helpers import provider_settings as provider_settings_module from sqlbuild.cli.compile_reuse._helpers import runtime_identity as runtime_identity_module from sqlbuild.cli.compile_reuse._helpers.runtime_identity import runtime_identity -from sqlbuild.compiler.frontier.constants import ( - COMPILE_REUSE_DISABLE_ENV_VAR as REUSE_DISABLE_ENV_VAR, -) -from sqlbuild.compiler.frontier.constants import STAGE_CAPTURE_DIR_ENV_VAR STAGING_PREFIX: str = ".sqlbuild-staging-" NATIVE_REUSE_DIRECTORY_NAME: str = "project-reuse-v2" diff --git a/tests/integration/src/sqlbuild/cli/commands/main/model_migrations/helpers.py b/tests/integration/src/sqlbuild/cli/commands/main/model_migrations/helpers.py index 3146c1624c..2090faf3ca 100644 --- a/tests/integration/src/sqlbuild/cli/commands/main/model_migrations/helpers.py +++ b/tests/integration/src/sqlbuild/cli/commands/main/model_migrations/helpers.py @@ -14,11 +14,11 @@ from _pytest.capture import CaptureResult import sqlbuild._native as fingerprint_cache_module +from sqlbuild._native import FINGERPRINT_TABLE_NAME from sqlbuild._native import run_cli as main from sqlbuild.adapter.contract.models import MigrationStagePlan from sqlbuild.adapter.contract.types import MigrationTransfer from sqlbuild.adapters.duckdb.classes.duckdb_adapter import DuckDbAdapter -from sqlbuild.compiler.fingerprints.constants import FINGERPRINT_TABLE_NAME from sqlbuild.executor.build._helpers import scheduler as scheduler_module DATABASE_FILE: str = "orders.duckdb" diff --git a/tests/integration/src/sqlbuild/cli/commands/main/test_discovery_work.py b/tests/integration/src/sqlbuild/cli/commands/main/test_discovery_work.py index 7ec8185905..4820e84708 100644 --- a/tests/integration/src/sqlbuild/cli/commands/main/test_discovery_work.py +++ b/tests/integration/src/sqlbuild/cli/commands/main/test_discovery_work.py @@ -6,8 +6,8 @@ import pytest +from sqlbuild._native import STAGE_CAPTURE_DIR_ENV_VAR from sqlbuild._native import run_cli as main -from sqlbuild.compiler.frontier.constants import STAGE_CAPTURE_DIR_ENV_VAR from tests.integration.src.sqlbuild.cli.commands.main._test_types import DiscoveryWorkTestCase from tests.integration.src.sqlbuild.cli.commands.main.helpers import ( eager_output_column_scans, diff --git a/tests/integration/src/sqlbuild/cli/commands/main/test_startup_import_footprint.py b/tests/integration/src/sqlbuild/cli/commands/main/test_startup_import_footprint.py index a748006d60..85d8a7db1d 100644 --- a/tests/integration/src/sqlbuild/cli/commands/main/test_startup_import_footprint.py +++ b/tests/integration/src/sqlbuild/cli/commands/main/test_startup_import_footprint.py @@ -11,9 +11,7 @@ import pytest -from sqlbuild.compiler.frontier.constants import ( - COMPILE_REUSE_DISABLE_ENV_VAR as REUSE_DISABLE_ENV_VAR, -) +from sqlbuild._native import COMPILE_REUSE_DISABLE_ENV_VAR as REUSE_DISABLE_ENV_VAR from tests.integration.src.sqlbuild.cli.commands.main._test_types import ( ReusedCompileImportFootprintTestCase, StartupImportFootprintTestCase, diff --git a/tests/integration/src/sqlbuild/cli/entry/helpers.py b/tests/integration/src/sqlbuild/cli/entry/helpers.py index 9e25be6485..f43dbaea72 100644 --- a/tests/integration/src/sqlbuild/cli/entry/helpers.py +++ b/tests/integration/src/sqlbuild/cli/entry/helpers.py @@ -7,9 +7,7 @@ from pathlib import Path import sqlbuild -from sqlbuild.compiler.frontier.constants import ( - COMPILE_REUSE_DISABLE_ENV_VAR as REUSE_DISABLE_ENV_VAR, -) +from sqlbuild._native import COMPILE_REUSE_DISABLE_ENV_VAR as REUSE_DISABLE_ENV_VAR type LauncherRun = tuple[int, bytes, bytes] diff --git a/tests/integration/src/sqlbuild/compiler/fingerprints/main/test_read_write.py b/tests/integration/src/sqlbuild/compiler/fingerprints/main/test_read_write.py index 60429d837e..effd6e49d8 100644 --- a/tests/integration/src/sqlbuild/compiler/fingerprints/main/test_read_write.py +++ b/tests/integration/src/sqlbuild/compiler/fingerprints/main/test_read_write.py @@ -8,6 +8,7 @@ import pytest from sqlbuild._native import ( + FINGERPRINT_TABLE_NAME, Fingerprint, FingerprintSet, read_latest_fingerprints, @@ -17,7 +18,6 @@ ) from sqlbuild.adapter.contract.types import FrameworkType from sqlbuild.adapters.duckdb.classes.duckdb_adapter import DuckDbAdapter -from sqlbuild.compiler.fingerprints.constants import FINGERPRINT_TABLE_NAME from tests.integration.src.sqlbuild.compiler.fingerprints.main._test_types import ( BatchWriteTestCase, InvalidDefinitionStorageTestCase, diff --git a/tests/integration/src/sqlbuild/compiler/source_freshness/main/test_read_write.py b/tests/integration/src/sqlbuild/compiler/source_freshness/main/test_read_write.py index 833bc8a83e..22fe766871 100644 --- a/tests/integration/src/sqlbuild/compiler/source_freshness/main/test_read_write.py +++ b/tests/integration/src/sqlbuild/compiler/source_freshness/main/test_read_write.py @@ -7,6 +7,7 @@ import pytest from sqlbuild._native import ( + SOURCE_FRESHNESS_TABLE_NAME, SourceFreshnessIdentity, SourceFreshnessRecord, SourceFreshnessRenderers, @@ -16,7 +17,6 @@ ) from sqlbuild.adapter.contract.types import FrameworkType from sqlbuild.adapters.duckdb.classes.duckdb_adapter import DuckDbAdapter -from sqlbuild.compiler.source_freshness.constants import SOURCE_FRESHNESS_TABLE_NAME from tests.integration.src.sqlbuild.compiler.source_freshness.main._test_types import ( SourceFreshnessLatestResolutionTestCase, SourceFreshnessPruneHistoryTestCase, diff --git a/tests/integration/src/sqlbuild/executor/run/incremental/test_incremental_execution.py b/tests/integration/src/sqlbuild/executor/run/incremental/test_incremental_execution.py index d97ae0f5d8..1ba5edc877 100644 --- a/tests/integration/src/sqlbuild/executor/run/incremental/test_incremental_execution.py +++ b/tests/integration/src/sqlbuild/executor/run/incremental/test_incremental_execution.py @@ -9,6 +9,8 @@ import pytest from sqlbuild._native import ( + MICROBATCH_END_SENTINEL, + MICROBATCH_START_SENTINEL, DiscoveredHookFunction, FutureCursorAction, FutureCursorsConfig, @@ -18,7 +20,6 @@ ) from sqlbuild.adapters.duckdb.classes.duckdb_adapter import DuckDbAdapter from sqlbuild.compiler.auditing.types import AuditRunScope -from sqlbuild.compiler.planner.constants import MICROBATCH_END_SENTINEL, MICROBATCH_START_SENTINEL from sqlbuild.executor.run.models import ModelExecutionResult from sqlbuild.executor.run.types import ExecutionPhase from tests.integration.src.sqlbuild.executor.run.incremental._test_types import ( diff --git a/tests/integration/src/sqlbuild/executor/run/microbatch/helpers.py b/tests/integration/src/sqlbuild/executor/run/microbatch/helpers.py index 20064a7020..7a19c2c803 100644 --- a/tests/integration/src/sqlbuild/executor/run/microbatch/helpers.py +++ b/tests/integration/src/sqlbuild/executor/run/microbatch/helpers.py @@ -8,6 +8,8 @@ from typing import Any, cast from sqlbuild._native import ( + MICROBATCH_END_SENTINEL, + MICROBATCH_START_SENTINEL, AuditPlanEntry, CompiledObjectKey, CompiledRelationLocation, @@ -27,10 +29,6 @@ AuditSeverity, ) from sqlbuild.compiler.compile.types import CompiledResourceType -from sqlbuild.compiler.planner.constants import ( - MICROBATCH_END_SENTINEL, - MICROBATCH_START_SENTINEL, -) from sqlbuild.executor.run._helpers.materializations.microbatch import execute_microbatch_entry from sqlbuild.executor.run._helpers.validation.cursor_bounds import resolve_runtime_cursor_bounds from sqlbuild.executor.run.models import ( diff --git a/tests/integration/src/sqlbuild/executor/run/microbatch/test_microbatch_execution.py b/tests/integration/src/sqlbuild/executor/run/microbatch/test_microbatch_execution.py index 2ff0714585..78215eeda5 100644 --- a/tests/integration/src/sqlbuild/executor/run/microbatch/test_microbatch_execution.py +++ b/tests/integration/src/sqlbuild/executor/run/microbatch/test_microbatch_execution.py @@ -11,6 +11,8 @@ import pytest from sqlbuild._native import ( + MICROBATCH_END_SENTINEL, + MICROBATCH_START_SENTINEL, DiscoveredHookFunction, FutureCursorAction, FutureCursorsConfig, @@ -21,10 +23,6 @@ ) from sqlbuild.adapters.duckdb.classes.duckdb_adapter import DuckDbAdapter from sqlbuild.compiler.auditing.types import AuditRunScope -from sqlbuild.compiler.planner.constants import ( - MICROBATCH_END_SENTINEL, - MICROBATCH_START_SENTINEL, -) from sqlbuild.executor.run._helpers.materializations import microbatch as microbatch_module from sqlbuild.executor.run.models import BatchWindow, ModelExecutionResult from sqlbuild.executor.run.types import ExecutionPhase diff --git a/tests/unit/src/sqlbuild/cli/commands/main/entry/test_observability_cleanup.py b/tests/unit/src/sqlbuild/cli/commands/main/entry/test_observability_cleanup.py index 62c9577aa9..ea4175c20a 100644 --- a/tests/unit/src/sqlbuild/cli/commands/main/entry/test_observability_cleanup.py +++ b/tests/unit/src/sqlbuild/cli/commands/main/entry/test_observability_cleanup.py @@ -10,12 +10,12 @@ from _pytest.capture import CaptureResult from sqlbuild._native import ( + INTEGRATION_RESULT_PATH_ENV, CliNamespace, NativeProgressProjector, cli_observability_scope, current_native_progress_projector, ) -from sqlbuild.cli.output.constants import INTEGRATION_RESULT_PATH_ENV from sqlbuild.observability import OperationLifecycle from tests.unit.src.sqlbuild.cli.commands.main.entry._test_types import ( MachineProgressRoutingCase, diff --git a/tests/unit/src/sqlbuild/cli/commands/main/entrypoint/test_dispatch_with_output_capture.py b/tests/unit/src/sqlbuild/cli/commands/main/entrypoint/test_dispatch_with_output_capture.py index 78ed6ae108..882ecb69f3 100644 --- a/tests/unit/src/sqlbuild/cli/commands/main/entrypoint/test_dispatch_with_output_capture.py +++ b/tests/unit/src/sqlbuild/cli/commands/main/entrypoint/test_dispatch_with_output_capture.py @@ -7,9 +7,8 @@ import pytest -from sqlbuild._native import EventExporterCommandScope +from sqlbuild._native import INVOCATION_CONTEXT_ENV, EventExporterCommandScope from sqlbuild.observability import ExecutionIdentity -from sqlbuild.runtime.output_capture.constants import INVOCATION_CONTEXT_ENV from sqlbuild.runtime.output_capture.main.configured_output_capture_scope import ( configured_output_capture_scope, ) diff --git a/tests/unit/src/sqlbuild/cli/compile_reuse/main/helpers.py b/tests/unit/src/sqlbuild/cli/compile_reuse/main/helpers.py index 356455b32f..b22f33f9d5 100644 --- a/tests/unit/src/sqlbuild/cli/compile_reuse/main/helpers.py +++ b/tests/unit/src/sqlbuild/cli/compile_reuse/main/helpers.py @@ -9,9 +9,7 @@ from pathlib import Path from typing import cast -from sqlbuild.compiler.frontier.constants import ( - COMPILE_REUSE_DISABLE_ENV_VAR as REUSE_DISABLE_ENV_VAR, -) +from sqlbuild._native import COMPILE_REUSE_DISABLE_ENV_VAR as REUSE_DISABLE_ENV_VAR def compile_in_fresh_process( diff --git a/tests/unit/src/sqlbuild/compiler/fingerprints/main/shared/_helpers/test_sql.py b/tests/unit/src/sqlbuild/compiler/fingerprints/main/shared/_helpers/test_sql.py index 7576a962a5..92cfc7dc16 100644 --- a/tests/unit/src/sqlbuild/compiler/fingerprints/main/shared/_helpers/test_sql.py +++ b/tests/unit/src/sqlbuild/compiler/fingerprints/main/shared/_helpers/test_sql.py @@ -6,6 +6,7 @@ import pytest from sqlbuild._native import ( + FINGERPRINT_TABLE_NAME, Fingerprint, build_create_table_sql, build_read_latest_sql, @@ -14,7 +15,6 @@ from sqlbuild.adapter.contract.types import FrameworkType from sqlbuild.adapters.bigquery.classes.bigquery_adapter import BigQueryAdapter from sqlbuild.adapters.duckdb.classes.duckdb_adapter import DuckDbAdapter -from sqlbuild.compiler.fingerprints.constants import FINGERPRINT_TABLE_NAME from tests.unit.src.sqlbuild.compiler.fingerprints.main.shared._helpers._test_types import ( BuildCreateTableSqlTestCase, BuildInsertSqlTestCase, diff --git a/tests/unit/src/sqlbuild/compiler/frontier/test_stage_capture.py b/tests/unit/src/sqlbuild/compiler/frontier/test_stage_capture.py index 6eab3842bf..86d7bd65ee 100644 --- a/tests/unit/src/sqlbuild/compiler/frontier/test_stage_capture.py +++ b/tests/unit/src/sqlbuild/compiler/frontier/test_stage_capture.py @@ -9,8 +9,12 @@ import pytest from scripts.compiler_differential.classes.capture_file import expand_capture_text -from sqlbuild._native import CompilerStage, compile_frontier, render_stage_capture -from sqlbuild.compiler.frontier.constants import STAGE_CAPTURE_DIR_ENV_VAR +from sqlbuild._native import ( + STAGE_CAPTURE_DIR_ENV_VAR, + CompilerStage, + compile_frontier, + render_stage_capture, +) from tests.unit.src.sqlbuild.compiler.frontier._test_types import ( FrontierCaptureTestCase, OmittedFieldTestCase, diff --git a/tests/unit/src/sqlbuild/compiler/planner/_helpers/resolve/test_cursor.py b/tests/unit/src/sqlbuild/compiler/planner/_helpers/resolve/test_cursor.py index cd72bfbf39..bac7d15b57 100644 --- a/tests/unit/src/sqlbuild/compiler/planner/_helpers/resolve/test_cursor.py +++ b/tests/unit/src/sqlbuild/compiler/planner/_helpers/resolve/test_cursor.py @@ -8,6 +8,8 @@ import pytest from sqlbuild._native import ( + MICROBATCH_END_SENTINEL, + MICROBATCH_START_SENTINEL, AlignedInterval, CursorBounds, CursorGrain, @@ -22,10 +24,6 @@ from sqlbuild._native import cursor_observed_partition as observed_partition from sqlbuild._native import cursor_parse as parse from sqlbuild._native import cursor_render as render -from sqlbuild.compiler.planner.constants import ( - MICROBATCH_END_SENTINEL, - MICROBATCH_START_SENTINEL, -) from sqlbuild.cursor_algebra.types import CursorScalar from tests.unit.src.sqlbuild.compiler.planner._helpers.resolve._test_types import ( CursorBoundsTestCase, diff --git a/tests/unit/src/sqlbuild/compiler/planner/_helpers/test_plan_entry_cursor_overrides.py b/tests/unit/src/sqlbuild/compiler/planner/_helpers/test_plan_entry_cursor_overrides.py index d8b006fa4b..878a03936b 100644 --- a/tests/unit/src/sqlbuild/compiler/planner/_helpers/test_plan_entry_cursor_overrides.py +++ b/tests/unit/src/sqlbuild/compiler/planner/_helpers/test_plan_entry_cursor_overrides.py @@ -8,6 +8,8 @@ import pytest from sqlbuild._native import ( + MICROBATCH_END_SENTINEL, + MICROBATCH_START_SENTINEL, BackfillAction, BackfillResult, CompiledModel, @@ -23,7 +25,6 @@ ) from sqlbuild._native import cursor_sentinel_to_token as sentinel_to_token from sqlbuild.compiler.compile.types import CompiledModelView, CompiledResourceType -from sqlbuild.compiler.planner.constants import MICROBATCH_END_SENTINEL, MICROBATCH_START_SENTINEL from tests.unit.src.sqlbuild.compiler.planner._helpers._test_types import ( AuthoritativeCursorOverrideTestCase, MicrobatchCursorEndPlanTestCase, diff --git a/tests/unit/src/sqlbuild/compiler/planner/_helpers/test_selection_staleness.py b/tests/unit/src/sqlbuild/compiler/planner/_helpers/test_selection_staleness.py index 5f982bfe37..b727f99bde 100644 --- a/tests/unit/src/sqlbuild/compiler/planner/_helpers/test_selection_staleness.py +++ b/tests/unit/src/sqlbuild/compiler/planner/_helpers/test_selection_staleness.py @@ -6,6 +6,7 @@ import pytest from sqlbuild._native import ( + NODE_TYPE_SEED, ChangeDetectionResult, ChangeKind, CompiledObjectKey, @@ -31,7 +32,6 @@ CompiledProjectView, CompiledResourceType, ) -from sqlbuild.compiler.fingerprints.constants import NODE_TYPE_SEED from tests.unit.src.sqlbuild.compiler.planner._helpers._test_types import ( CompleteModelChangesWorkTestCase, PlannerStaleWarningScopeTestCase, diff --git a/tests/unit/src/sqlbuild/compiler/source_freshness/main/helpers.py b/tests/unit/src/sqlbuild/compiler/source_freshness/main/helpers.py index b68477b7af..10ff692c35 100644 --- a/tests/unit/src/sqlbuild/compiler/source_freshness/main/helpers.py +++ b/tests/unit/src/sqlbuild/compiler/source_freshness/main/helpers.py @@ -6,6 +6,7 @@ from typing import Any from sqlbuild._native import ( + SOURCE_FRESHNESS_TABLE_NAME, CompiledObjectKey, SourceEntry, SourceFreshnessConfig, @@ -27,7 +28,6 @@ from sqlbuild.adapter.relations.main.relation_lookup import build_relation_lookup from sqlbuild.adapters.duckdb.classes.duckdb_adapter import DuckDbAdapter from sqlbuild.compiler.compile.types import CompiledResourceType -from sqlbuild.compiler.source_freshness.constants import SOURCE_FRESHNESS_TABLE_NAME def state_table_exists_map( diff --git a/tests/unit/src/sqlbuild/compiler/source_freshness/main/shared/_helpers/test_sql.py b/tests/unit/src/sqlbuild/compiler/source_freshness/main/shared/_helpers/test_sql.py index c49624b344..08f851d558 100644 --- a/tests/unit/src/sqlbuild/compiler/source_freshness/main/shared/_helpers/test_sql.py +++ b/tests/unit/src/sqlbuild/compiler/source_freshness/main/shared/_helpers/test_sql.py @@ -5,12 +5,12 @@ import pytest from sqlbuild._native import ( + SOURCE_FRESHNESS_TABLE_NAME, build_source_freshness_create_table_sql, build_source_freshness_read_latest_sql, ) from sqlbuild.adapter.contract.types import FrameworkType from sqlbuild.adapters.duckdb.classes.duckdb_adapter import DuckDbAdapter -from sqlbuild.compiler.source_freshness.constants import SOURCE_FRESHNESS_TABLE_NAME from tests.unit.src.sqlbuild.compiler.source_freshness.main.shared._helpers._test_types import ( BuildSourceFreshnessSqlTestCase, ) diff --git a/tests/unit/src/sqlbuild/executor/clone/_helpers/fingerprinting/helpers.py b/tests/unit/src/sqlbuild/executor/clone/_helpers/fingerprinting/helpers.py index 81e2534a69..18a2a6aa22 100644 --- a/tests/unit/src/sqlbuild/executor/clone/_helpers/fingerprinting/helpers.py +++ b/tests/unit/src/sqlbuild/executor/clone/_helpers/fingerprinting/helpers.py @@ -6,6 +6,7 @@ import pytest from sqlbuild._native import ( + FINGERPRINT_TABLE_NAME, CompiledObjectKey, CompiledRelationLocation, Fingerprint, @@ -19,7 +20,6 @@ ) from sqlbuild.adapter.contract.models import RelationInfo from sqlbuild.compiler.compile.types import CompiledResourceType -from sqlbuild.compiler.fingerprints.constants import FINGERPRINT_TABLE_NAME class CloneFingerprintAdapter: diff --git a/tests/unit/src/sqlbuild/executor/clone/_helpers/fingerprinting/test_fingerprinting.py b/tests/unit/src/sqlbuild/executor/clone/_helpers/fingerprinting/test_fingerprinting.py index 4e9a25ed87..52e78e64a3 100644 --- a/tests/unit/src/sqlbuild/executor/clone/_helpers/fingerprinting/test_fingerprinting.py +++ b/tests/unit/src/sqlbuild/executor/clone/_helpers/fingerprinting/test_fingerprinting.py @@ -6,6 +6,8 @@ import pytest from sqlbuild._native import ( + NODE_TYPE_MODEL, + NODE_TYPE_SEED, Fingerprint, FingerprintSet, MaterializationType, @@ -13,7 +15,6 @@ SeedPlanEntry, ) from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter -from sqlbuild.compiler.fingerprints.constants import NODE_TYPE_MODEL, NODE_TYPE_SEED from sqlbuild.executor.clone._helpers.fingerprinting import copy_clone_fingerprints from sqlbuild.executor.clone.models import CloneExecutionResult, CloneItemResult from sqlbuild.executor.clone.types import CloneAction, CloneStatus diff --git a/tests/unit/src/sqlbuild/executor/janitor/main/helpers.py b/tests/unit/src/sqlbuild/executor/janitor/main/helpers.py index 5e164641ad..ff95e704a1 100644 --- a/tests/unit/src/sqlbuild/executor/janitor/main/helpers.py +++ b/tests/unit/src/sqlbuild/executor/janitor/main/helpers.py @@ -10,6 +10,8 @@ from typing import Any, ClassVar, cast from sqlbuild._native import ( + FINGERPRINT_TABLE_NAME, + SOURCE_FRESHNESS_TABLE_NAME, CompiledModel, CompiledObjectKey, CompiledProject, @@ -37,8 +39,6 @@ CompiledResourceType, CompiledSourceView, ) -from sqlbuild.compiler.fingerprints.constants import FINGERPRINT_TABLE_NAME -from sqlbuild.compiler.source_freshness.constants import SOURCE_FRESHNESS_TABLE_NAME from sqlbuild.executor.janitor_events.constants import JANITOR_EVENTS_TABLE_NAME from sqlbuild.spec.contracts.constants import DEFAULT_SEED_CSV_SETTINGS diff --git a/tests/unit/src/sqlbuild/executor/janitor/main/test_plan.py b/tests/unit/src/sqlbuild/executor/janitor/main/test_plan.py index fbfd2166cc..4f85fd29c9 100644 --- a/tests/unit/src/sqlbuild/executor/janitor/main/test_plan.py +++ b/tests/unit/src/sqlbuild/executor/janitor/main/test_plan.py @@ -5,12 +5,12 @@ import pytest from sqlbuild._native import ( + FINGERPRINT_TABLE_NAME, + SOURCE_FRESHNESS_TABLE_NAME, build_janitor_plan, build_scenario_artifact_name, compute_scenario_hash_prefix, ) -from sqlbuild.compiler.fingerprints.constants import FINGERPRINT_TABLE_NAME -from sqlbuild.compiler.source_freshness.constants import SOURCE_FRESHNESS_TABLE_NAME from sqlbuild.executor.janitor.main.execute import execute_janitor_plan from sqlbuild.executor.janitor.models import ( JanitorExecutionResult, diff --git a/tests/unit/src/sqlbuild/executor/run/_helpers/test_fingerprint_metadata.py b/tests/unit/src/sqlbuild/executor/run/_helpers/test_fingerprint_metadata.py index df86220afe..74ce5ac758 100644 --- a/tests/unit/src/sqlbuild/executor/run/_helpers/test_fingerprint_metadata.py +++ b/tests/unit/src/sqlbuild/executor/run/_helpers/test_fingerprint_metadata.py @@ -9,9 +9,8 @@ import pytest -from sqlbuild._native import AuditPlanEntry, Fingerprint, ModelPlanEntry +from sqlbuild._native import AUDIT_GATE_METADATA_KEY, AuditPlanEntry, Fingerprint, ModelPlanEntry from sqlbuild.compiler.auditing.types import AuditOutcome, AuditSeverity -from sqlbuild.compiler.fingerprints.constants import AUDIT_GATE_METADATA_KEY from sqlbuild.executor.auditing.models import AuditExecutionResult from sqlbuild.executor.run._helpers.reuse import fingerprinting from sqlbuild.executor.run._helpers.reuse.fingerprint_metadata import ( diff --git a/tests/unit/src/sqlbuild/executor/run/_helpers/test_snapshot_lifecycle.py b/tests/unit/src/sqlbuild/executor/run/_helpers/test_snapshot_lifecycle.py index 41dc1b31d1..9217840fbd 100644 --- a/tests/unit/src/sqlbuild/executor/run/_helpers/test_snapshot_lifecycle.py +++ b/tests/unit/src/sqlbuild/executor/run/_helpers/test_snapshot_lifecycle.py @@ -8,6 +8,8 @@ import pytest from sqlbuild._native import ( + FINGERPRINT_TABLE_NAME, + NODE_TYPE_MODEL, DiscoveredHookFunction, ModelPlanEntry, PythonHookEntry, @@ -18,7 +20,6 @@ from sqlbuild.adapter.contract.models import ColumnInfo from sqlbuild.adapters.duckdb.classes.duckdb_adapter import DuckDbAdapter from sqlbuild.adapters.snowflake.classes.snowflake_adapter import SnowflakeAdapter -from sqlbuild.compiler.fingerprints.constants import FINGERPRINT_TABLE_NAME, NODE_TYPE_MODEL from sqlbuild.errors.contracts.exceptions import ExecutorInputError from sqlbuild.executor.run._helpers.materializations.snapshot import ( _apply_snapshot_schema_change, diff --git a/tests/unit/src/sqlbuild/executor/seed/main/test_execute.py b/tests/unit/src/sqlbuild/executor/seed/main/test_execute.py index b61f5a5ef2..88731ae8f5 100644 --- a/tests/unit/src/sqlbuild/executor/seed/main/test_execute.py +++ b/tests/unit/src/sqlbuild/executor/seed/main/test_execute.py @@ -7,9 +7,9 @@ import pytest +from sqlbuild._native import FINGERPRINT_TABLE_NAME from sqlbuild.adapter.contract.classes.statement_recorder import StatementRecorder from sqlbuild.adapters.duckdb.classes.duckdb_adapter import DuckDbAdapter -from sqlbuild.compiler.fingerprints.constants import FINGERPRINT_TABLE_NAME from sqlbuild.executor.build.models import SeedExecutionResult from sqlbuild.executor.scheduling.types import ExecutionStatus from sqlbuild.executor.seed.main._execute import execute_seed diff --git a/tests/unit/src/sqlbuild/rule_engine/_helpers/engine/helpers.py b/tests/unit/src/sqlbuild/rule_engine/_helpers/engine/helpers.py index d277fd2aba..0b72c88ed7 100644 --- a/tests/unit/src/sqlbuild/rule_engine/_helpers/engine/helpers.py +++ b/tests/unit/src/sqlbuild/rule_engine/_helpers/engine/helpers.py @@ -11,6 +11,7 @@ import sqlbuild._native as native from sqlbuild._native import ( + MIN_CUSTOM_RULE_TEST_CASES, CompiledDirectLogicSqlTestPayload, CompiledModelSqlTestPayload, CompiledObjectKey, @@ -37,7 +38,6 @@ from sqlbuild._native import evaluate_custom_rules as evaluate_custom_rules_cached from sqlbuild._native import evaluate_project_rules as evaluate from sqlbuild.compiler.compile.types import CompiledProjectView, CompiledResourceType, SqlTestMode -from sqlbuild.rule_engine.constants import MIN_CUSTOM_RULE_TEST_CASES from tests.unit.src.sqlbuild.rule_engine._helpers.engine._test_types import CustomRuleTestCase from tests.unit.src.sqlbuild.rule_engine.main.evaluate.helpers import build_project diff --git a/tests/unit/src/sqlbuild/rule_engine/main/evaluate_rule/test_evaluate_rule.py b/tests/unit/src/sqlbuild/rule_engine/main/evaluate_rule/test_evaluate_rule.py index 01e1b5d527..7b5140b457 100644 --- a/tests/unit/src/sqlbuild/rule_engine/main/evaluate_rule/test_evaluate_rule.py +++ b/tests/unit/src/sqlbuild/rule_engine/main/evaluate_rule/test_evaluate_rule.py @@ -6,10 +6,15 @@ import pytest -from sqlbuild._native import RulesCacheConfig, RulesConfig, RulesError, RulesResult +from sqlbuild._native import ( + MIN_CUSTOM_RULE_TEST_CASES, + RulesCacheConfig, + RulesConfig, + RulesError, + RulesResult, +) from sqlbuild._native import evaluate_project_rules as evaluate from sqlbuild.compiler.compile.types import CompiledProjectView -from sqlbuild.rule_engine.constants import MIN_CUSTOM_RULE_TEST_CASES from sqlbuild.rule_engine.types import RuleOptionValue from sqlbuild.rules.testing import RuleCase, RuleResult, evaluate_rule from tests.unit.src.sqlbuild.rule_engine.main.evaluate.helpers import build_project diff --git a/tests/unit/src/sqlbuild/runtime/event_exporting/classes/test_dispatcher.py b/tests/unit/src/sqlbuild/runtime/event_exporting/classes/test_dispatcher.py index 8a0b9e0176..0135132e6e 100644 --- a/tests/unit/src/sqlbuild/runtime/event_exporting/classes/test_dispatcher.py +++ b/tests/unit/src/sqlbuild/runtime/event_exporting/classes/test_dispatcher.py @@ -9,6 +9,7 @@ import pytest from sqlbuild._native import ( + COMMAND_OUTPUT_LOSS_RECORD_TYPE, BoundEventExporter, EventExporterCounts, EventExporterDispatcher, @@ -17,7 +18,6 @@ ) from sqlbuild.observability import LifecycleEvent from sqlbuild.runtime.event_exporting.exceptions import EventExporterInputError -from sqlbuild.runtime.output_capture.constants import COMMAND_OUTPUT_LOSS_RECORD_TYPE from sqlbuild.runtime.output_capture.models import BoundCommandOutputSink from sqlbuild.sinks import CommandOutputRecord, CommandOutputStream from tests.unit.src.sqlbuild.runtime.event_exporting.classes._test_types import ( diff --git a/tests/unit/src/sqlbuild/runtime/execution_history/conformance/conftest.py b/tests/unit/src/sqlbuild/runtime/execution_history/conformance/conftest.py index a17610d6bb..42a8d417e3 100644 --- a/tests/unit/src/sqlbuild/runtime/execution_history/conformance/conftest.py +++ b/tests/unit/src/sqlbuild/runtime/execution_history/conformance/conftest.py @@ -16,6 +16,11 @@ import pytest +from sqlbuild._native import ( + CURRENT_EVENT_LOG_SCHEMA_VERSION, + CURRENT_RUN_STORAGE_SCHEMA_VERSION, + DEFAULT_PAGE_LIMIT, +) from sqlbuild.execution_history import ( CanonicalLifecycleEvent, EventFilter, @@ -35,11 +40,6 @@ project_runs, validate_page_limit, ) -from sqlbuild.runtime.execution_history.constants import ( - CURRENT_EVENT_LOG_SCHEMA_VERSION, - CURRENT_RUN_STORAGE_SCHEMA_VERSION, - DEFAULT_PAGE_LIMIT, -) from sqlbuild.runtime.observability.models import LifecycleEvent, OpaqueLifecycleEvent from sqlbuild.runtime.observability.types import JSONValue from sqlbuild.sqlite_history import SQLiteExecutionHistory diff --git a/tests/unit/src/sqlbuild/runtime/output_capture/main/test_invocation_context.py b/tests/unit/src/sqlbuild/runtime/output_capture/main/test_invocation_context.py index 19a5031ccd..fd5de7c766 100644 --- a/tests/unit/src/sqlbuild/runtime/output_capture/main/test_invocation_context.py +++ b/tests/unit/src/sqlbuild/runtime/output_capture/main/test_invocation_context.py @@ -7,8 +7,7 @@ import pytest -from sqlbuild._native import invocation_context_from_environment -from sqlbuild.runtime.output_capture.constants import INVOCATION_CONTEXT_ENV +from sqlbuild._native import INVOCATION_CONTEXT_ENV, invocation_context_from_environment from sqlbuild.runtime.output_capture.exceptions import OutputCaptureInputError from tests.unit.src.sqlbuild.runtime.output_capture.main._test_types import ( InvocationContextTestCase, From 0470462d88aef5a50a294545ec4183c7f99b2f16 Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Thu, 15 Oct 2026 22:57:15 +0100 Subject: [PATCH 25/48] chore: move cost package fensu exception to its models module --- fensu.toml | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/fensu.toml b/fensu.toml index db5fceb35c..81e90ca63c 100644 --- a/fensu.toml +++ b/fensu.toml @@ -23,6 +23,11 @@ exclude = ["tests/e2e/fixtures/**"] [targets.python.thresholds] max_positional_args = 1 +[[targets.python.rule_exceptions]] +rule = "FFR309" +path = "src/sqlbuild/cost/models.py" +reason = "Native bindings own run-cost allocation, Snowflake cost collection, the statement ledger and the cost context; this package only declares the cost models and types they use." + [[targets.python.rule_exceptions]] rule = "FFR309" path = "src/sqlbuild/lint/__init__.py" @@ -68,11 +73,6 @@ rule = "FFR309" path = "src/sqlbuild/compiler/source_freshness/__init__.py" reason = "Native bindings own source freshness observation, planning, propagation and state reads and writes; this package only declares the models, types and constants they build." -[[targets.python.rule_exceptions]] -rule = "FFR309" -path = "src/sqlbuild/cost/constants.py" -reason = "Native bindings own run-cost allocation, Snowflake cost collection, the statement ledger and the cost context; this package only declares the cost models, types and constants they use." - [[targets.python.rule_exceptions]] rule = "FFR309" path = "src/sqlbuild/compiler/pipeline/__init__.py" From 6da776545315a7f2c075f5f97d282a3481351e29 Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Thu, 15 Oct 2026 18:08:19 +0100 Subject: [PATCH 26/48] refactor: render append-only state table DDL and source freshness inserts natively --- .../_helpers/contracts/state_literals.rs | 2 +- .../bindings/_helpers/freshness/state_io.rs | 30 ++-- .../bindings/_helpers/freshness/support.rs | 2 - .../bindings/_helpers/state_rows/constants.rs | 2 - .../_helpers/state_rows/migration_rows.rs | 27 ++-- .../src/bindings/_helpers/state_rows/mod.rs | 1 + .../_helpers/state_rows/registration.rs | 3 +- .../_helpers/state_rows/state_tables.rs | 150 ++++++++++++++++++ crates/sqlbuild-state/src/lib.rs | 1 + .../src/state_tables/main/create_table_sql.rs | 18 +++ .../src/state_tables/main/mod.rs | 2 + .../state_tables/main/values_insert_sql.rs | 14 ++ crates/sqlbuild-state/src/state_tables/mod.rs | 5 + .../sqlbuild-state/src/state_tables/models.rs | 9 ++ .../sqlbuild-state/src/state_tables/tests.rs | 6 + .../src/state_tables/tests/helpers.rs | 10 ++ .../src/state_tables/tests/test_sql.rs | 50 ++++++ .../src/state_tables/tests/test_types.rs | 13 ++ python-boundary.toml | 6 +- src/sqlbuild/_native.pyi | 15 ++ .../adapter/contract/classes/base_adapter.py | 4 +- .../contract/classes/duckdb_backed_adapter.py | 11 +- src/sqlbuild/adapter/state_sql/__init__.py | 1 - .../adapter/state_sql/_helpers/__init__.py | 1 - .../state_sql/_helpers/source_freshness.py | 81 ---------- .../adapter/state_sql/_helpers/table_ddl.py | 30 ---- .../adapter/state_sql/main/__init__.py | 1 - ...der_insert_source_freshness_records_sql.py | 25 --- .../main/render_state_table_create_sql.py | 26 --- .../bigquery/classes/bigquery_adapter.py | 4 +- .../databricks/classes/databricks_adapter.py | 11 +- .../postgres/classes/postgres_adapter.py | 11 +- .../snowflake/classes/snowflake_adapter.py | 4 +- .../sqlserver/classes/sqlserver_adapter.py | 4 +- .../executor/audit_results/_helpers/sql.py | 5 +- .../executor/janitor_events/_helpers/sql.py | 5 +- .../executor/node_results/_helpers/sql.py | 5 +- .../sqlbuild/adapter/state_sql/_test_types.py | 7 - .../state_sql/test_source_freshness.py | 60 ------- .../src/sqlbuild/sql_values/_test_types.py | 15 +- .../sql_values/test_state_table_sql.py | 120 ++++++++++++++ 41 files changed, 485 insertions(+), 312 deletions(-) create mode 100644 crates/sqlbuild-python/src/bindings/_helpers/state_rows/state_tables.rs create mode 100644 crates/sqlbuild-state/src/state_tables/main/create_table_sql.rs create mode 100644 crates/sqlbuild-state/src/state_tables/main/mod.rs create mode 100644 crates/sqlbuild-state/src/state_tables/main/values_insert_sql.rs create mode 100644 crates/sqlbuild-state/src/state_tables/mod.rs create mode 100644 crates/sqlbuild-state/src/state_tables/models.rs create mode 100644 crates/sqlbuild-state/src/state_tables/tests.rs create mode 100644 crates/sqlbuild-state/src/state_tables/tests/helpers.rs create mode 100644 crates/sqlbuild-state/src/state_tables/tests/test_sql.rs create mode 100644 crates/sqlbuild-state/src/state_tables/tests/test_types.rs delete mode 100644 src/sqlbuild/adapter/state_sql/__init__.py delete mode 100644 src/sqlbuild/adapter/state_sql/_helpers/__init__.py delete mode 100644 src/sqlbuild/adapter/state_sql/_helpers/source_freshness.py delete mode 100644 src/sqlbuild/adapter/state_sql/_helpers/table_ddl.py delete mode 100644 src/sqlbuild/adapter/state_sql/main/__init__.py delete mode 100644 src/sqlbuild/adapter/state_sql/main/render_insert_source_freshness_records_sql.py delete mode 100644 src/sqlbuild/adapter/state_sql/main/render_state_table_create_sql.py delete mode 100644 tests/unit/src/sqlbuild/adapter/state_sql/_test_types.py delete mode 100644 tests/unit/src/sqlbuild/adapter/state_sql/test_source_freshness.py create mode 100644 tests/unit/src/sqlbuild/sql_values/test_state_table_sql.py diff --git a/crates/sqlbuild-python/src/bindings/_helpers/contracts/state_literals.rs b/crates/sqlbuild-python/src/bindings/_helpers/contracts/state_literals.rs index 58635c12f9..3c65cc66aa 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/contracts/state_literals.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/contracts/state_literals.rs @@ -21,7 +21,7 @@ const JSON_TYPE: &str = "json"; /// `render_state_sql_literal(*, value, declared_type)`: `value` as its state column's literal. #[pyfunction] #[pyo3(signature = (*, value, declared_type))] -fn render_state_sql_literal( +pub(crate) fn render_state_sql_literal( py: Python<'_>, value: &Bound<'_, PyAny>, declared_type: &str, diff --git a/crates/sqlbuild-python/src/bindings/_helpers/freshness/state_io.rs b/crates/sqlbuild-python/src/bindings/_helpers/freshness/state_io.rs index e52cb787e4..3e2bc9a4ff 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/freshness/state_io.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/freshness/state_io.rs @@ -13,9 +13,10 @@ use crate::bindings::_helpers::compile_driver::python_calls::{ }; use crate::bindings::_helpers::cursor::values::DATETIME_MODULE; use crate::bindings::_helpers::freshness::support::{ - FRESHNESS_CONSTANTS, FRESHNESS_MODELS, INPUT_ERROR, STATE_TABLE_DDL_MODULE, exception_text, - freshness_error, optional_str, presumed_utc_datetime, text, + FRESHNESS_CONSTANTS, FRESHNESS_MODELS, INPUT_ERROR, exception_text, freshness_error, + optional_str, presumed_utc_datetime, text, }; +use crate::bindings::_helpers::state_rows::state_tables::state_table_create_sql; use crate::bindings::_helpers::state_rows::timestamps::date_time_typed; const NATIVE_RECORDS: &str = "sqlbuild._native"; @@ -83,22 +84,15 @@ fn create_table_sql<'py>( let py: Python<'py> = database.py(); let qualified_name: String = qualified_table_name(render_qualified_name, (database, schema))?; let constants: Bound<'py, PyModule> = py.import(FRESHNESS_CONSTANTS)?; - call_with( - &imported(py, STATE_TABLE_DDL_MODULE, "render_state_table_create_sql")?, - &[ - ("qualified_name", text(py, &qualified_name)), - ("columns", constants.getattr("SOURCE_FRESHNESS_COLUMNS")?), - ( - "column_types", - constants.getattr("SOURCE_FRESHNESS_COLUMN_TYPES")?, - ), - ( - "required_columns", - PyFrozenSet::new(py, REQUIRED_STATE_COLUMNS)?.into_any(), - ), - ("render_framework_type", render_framework_type.clone()), - ], - ) + let columns: Bound<'py, PyAny> = constants.getattr("SOURCE_FRESHNESS_COLUMNS")?; + let column_types: Bound<'py, PyAny> = constants.getattr("SOURCE_FRESHNESS_COLUMN_TYPES")?; + let required: Bound<'py, PyAny> = PyFrozenSet::new(py, REQUIRED_STATE_COLUMNS)?.into_any(); + let sql: String = state_table_create_sql( + &qualified_name, + (&columns, &column_types, &required), + render_framework_type, + )?; + Ok(text(py, &sql)) } /// The callbacks and location one latest-state read uses. diff --git a/crates/sqlbuild-python/src/bindings/_helpers/freshness/support.rs b/crates/sqlbuild-python/src/bindings/_helpers/freshness/support.rs index 2a52df1756..575818f625 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/freshness/support.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/freshness/support.rs @@ -19,8 +19,6 @@ pub(crate) const ADAPTER_EXCEPTIONS: &str = "sqlbuild.adapter.contract.exception pub(crate) const LIFECYCLE_MODULE: &str = "sqlbuild.runtime.observability.classes.operation_lifecycle"; pub(crate) const OBSERVABILITY_MODELS: &str = "sqlbuild.runtime.observability.models"; -pub(crate) const STATE_TABLE_DDL_MODULE: &str = - "sqlbuild.adapter.state_sql.main.render_state_table_create_sql"; pub(crate) const OBSERVATION_ERROR: &str = "SourceFreshnessObservationError"; pub(crate) const INPUT_ERROR: &str = "SourceFreshnessInputError"; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/state_rows/constants.rs b/crates/sqlbuild-python/src/bindings/_helpers/state_rows/constants.rs index 9535e6cc60..2d5507f85e 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/state_rows/constants.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/state_rows/constants.rs @@ -3,8 +3,6 @@ pub(crate) const MIGRATION_MODELS: &str = "sqlbuild._native"; pub(crate) const MIGRATION_TYPES: &str = "sqlbuild._native"; pub(crate) const MIGRATION_EXCEPTIONS: &str = "sqlbuild._native"; -pub(crate) const STATE_TABLE_DDL: &str = - "sqlbuild.adapter.state_sql.main.render_state_table_create_sql"; pub(crate) const SQL_VALUE_TYPES: &str = "sqlbuild._native"; pub(crate) const OLD_NAME_VIEW_CONFIG_KEY: &str = "old_name_view"; pub(crate) const MODEL_KIND: &str = "model"; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/state_rows/migration_rows.rs b/crates/sqlbuild-python/src/bindings/_helpers/state_rows/migration_rows.rs index 9cbe53d4e2..737dc4d356 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/state_rows/migration_rows.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/state_rows/migration_rows.rs @@ -20,13 +20,14 @@ use crate::bindings::_helpers::boundary::keyword_request::Request; use crate::bindings::_helpers::compile_driver::python_calls::{call_with, imported}; use crate::bindings::_helpers::cursor::values::DATETIME_MODULE; use crate::bindings::_helpers::state_rows::constants::{ - MIGRATION_MODELS, MIGRATION_TYPES, SQL_VALUE_TYPES, STATE_TABLE_DDL, + MIGRATION_MODELS, MIGRATION_TYPES, SQL_VALUE_TYPES, }; use crate::bindings::_helpers::state_rows::events::typed_insert_sql; use crate::bindings::_helpers::state_rows::migration_objects::{ layout, qualified_table, state_error, }; use crate::bindings::_helpers::state_rows::models::{EventTable, EventTableLayout}; +use crate::bindings::_helpers::state_rows::state_tables::state_table_create_sql; use crate::bindings::_helpers::state_rows::timestamps::date_time_typed; /// DDL that creates one migration state table when it is missing. @@ -51,19 +52,17 @@ fn create_sql<'py>(request: Option<&Bound<'py, PyDict>>) -> PyResult = PyTuple::new(py, layout.columns)?.into_any(); + let sql: String = state_table_create_sql( + &table, + ( + &columns, + column_types.as_any(), + PyFrozenSet::empty(py)?.as_any(), + ), + &request.get("render_framework_type")?, + )?; + Ok(PyString::new(py, &sql).into_any()) } /// The SQL reading every stored event of one schema, and the columns it selects. diff --git a/crates/sqlbuild-python/src/bindings/_helpers/state_rows/mod.rs b/crates/sqlbuild-python/src/bindings/_helpers/state_rows/mod.rs index a130979144..59acf36d54 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/state_rows/mod.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/state_rows/mod.rs @@ -7,4 +7,5 @@ pub(crate) mod migration_projections; pub(crate) mod migration_rows; pub(crate) mod models; pub(crate) mod registration; +pub(crate) mod state_tables; pub(crate) mod timestamps; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/state_rows/registration.rs b/crates/sqlbuild-python/src/bindings/_helpers/state_rows/registration.rs index df9f69c0f7..cb78b58903 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/state_rows/registration.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/state_rows/registration.rs @@ -3,7 +3,7 @@ use pyo3::prelude::{Bound, PyModule, PyResult}; use crate::bindings::_helpers::state_rows::{ - events, freshness, migration_io, migration_projections, migration_rows, + events, freshness, migration_io, migration_projections, migration_rows, state_tables, }; pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { @@ -12,5 +12,6 @@ pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { migration_rows::register(module)?; migration_io::register(module)?; migration_projections::register(module)?; + state_tables::register(module)?; Ok(()) } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/state_rows/state_tables.rs b/crates/sqlbuild-python/src/bindings/_helpers/state_rows/state_tables.rs new file mode 100644 index 0000000000..6499dad472 --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/state_rows/state_tables.rs @@ -0,0 +1,150 @@ +//! Append-only state table DDL and the source freshness insert, rendered through adapter callbacks. + +use pyo3::exceptions::PyKeyError; +use pyo3::prelude::{ + Bound, PyAny, PyAnyMethods, PyErr, PyModule, PyModuleMethods, PyResult, Python, +}; +use pyo3::types::{PyDict, PyString, PyStringMethods}; +use pyo3::{pyfunction, wrap_pyfunction}; +use sqlbuild_state::freshness::constants::SOURCE_FRESHNESS_TABLE_NAME; +use sqlbuild_state::state_tables::main::create_table_sql::create_table_sql; +use sqlbuild_state::state_tables::main::values_insert_sql::values_insert_sql; +use sqlbuild_state::state_tables::models::TableColumn; + +use crate::bindings::_helpers::boundary::keyword_request::Request; +use crate::bindings::_helpers::compile_driver::python_calls::{call_with, imported}; +use crate::bindings::_helpers::contracts::state_literals::render_state_sql_literal; +use crate::bindings::_helpers::freshness::support::{ADAPTER_EXCEPTIONS, FRESHNESS_CONSTANTS}; + +const ADAPTER_TYPES: &str = "sqlbuild.adapter.contract.types"; +const MISSING_SCHEMA_MESSAGE: &str = "source freshness table requires a target schema"; +/// Each state value type a state table may declare and the framework type that renders it. +const DECLARED_TYPES: [(&str, &str); 3] = [ + ("string", "STRING"), + ("integer", "INTEGER"), + ("text_timestamp", "TIMESTAMP"), +]; + +/// Create-if-missing DDL whose column types the adapter renders. +pub(crate) fn state_table_create_sql<'py>( + qualified_name: &str, + (columns, column_types, required_columns): ( + &Bound<'py, PyAny>, + &Bound<'py, PyAny>, + &Bound<'py, PyAny>, + ), + render_framework_type: &Bound<'py, PyAny>, +) -> PyResult { + let py: Python<'py> = render_framework_type.py(); + let framework_types: Bound<'py, PyAny> = imported(py, ADAPTER_TYPES, "FrameworkType")?; + let mut rendered: Vec<(&str, String)> = Vec::with_capacity(DECLARED_TYPES.len()); + for (declared, framework) in DECLARED_TYPES { + let text: String = render_framework_type + .call1((framework_types.getattr(framework)?,))? + .extract()?; + rendered.push((declared, text)); + } + let mut definitions: Vec = Vec::new(); + for column in columns.try_iter()? { + let column: Bound<'py, PyAny> = column?; + let declared: Bound<'py, PyAny> = column_types.get_item(&column)?; + let declared_text: String = declared.str()?.to_str()?.to_owned(); + let Some((_, rendered_type)) = rendered.iter().find(|(name, _)| *name == declared_text) + else { + return Err(PyKeyError::new_err(declared.unbind())); + }; + definitions.push(TableColumn { + name: column.extract()?, + rendered_type: rendered_type.clone(), + required: required_columns.contains(&column)?, + }); + } + Ok(create_table_sql(qualified_name, &definitions)) +} + +/// Render create-if-missing DDL for string, integer, and text-timestamp state columns. +#[pyfunction(name = "render_state_table_create_sql")] +#[pyo3(signature = (**request))] +fn render_create_sql(request: Option<&Bound<'_, PyDict>>) -> PyResult { + let request = Request::new("render_state_table_create_sql", request)?; + state_table_create_sql( + &request.get("qualified_name")?.extract::()?, + ( + &request.get("columns")?, + &request.get("column_types")?, + &request.get("required_columns")?, + ), + &request.get("render_framework_type")?, + ) +} + +fn adapter_user_error(py: Python<'_>, message: &str) -> PyErr { + match imported(py, ADAPTER_EXCEPTIONS, "AdapterUserError").and_then(|class| { + call_with( + &class, + &[("message", PyString::new(py, message).into_any())], + ) + }) { + Ok(error) => PyErr::from_value(error), + Err(error) => error, + } +} + +/// Render DML that appends source freshness records. +#[pyfunction] +#[pyo3(signature = (*, database, schema, records, render_qualified_name))] +fn render_insert_source_freshness_records_sql<'py>( + database: &Bound<'py, PyAny>, + schema: &Bound<'py, PyAny>, + records: &Bound<'py, PyAny>, + render_qualified_name: &Bound<'py, PyAny>, +) -> PyResult { + let py: Python<'py> = database.py(); + let table: Bound<'py, PyAny> = call_with( + render_qualified_name, + &[ + ("database", database.clone()), + ("schema", schema.clone()), + ( + "name", + PyString::new(py, SOURCE_FRESHNESS_TABLE_NAME).into_any(), + ), + ], + )?; + if table.is_none() { + return Err(adapter_user_error(py, MISSING_SCHEMA_MESSAGE)); + } + let constants: Bound<'py, PyModule> = py.import(FRESHNESS_CONSTANTS)?; + let column_types: Bound<'py, PyAny> = constants.getattr("SOURCE_FRESHNESS_COLUMN_TYPES")?; + let columns: Vec = constants.getattr("SOURCE_FRESHNESS_COLUMNS")?.extract()?; + let declared: Vec = columns + .iter() + .map(|column| column_types.get_item(column)?.str()?.extract()) + .collect::>()?; + let mut rows: Vec> = Vec::new(); + for record in records.try_iter()? { + let record: Bound<'py, PyAny> = record?; + let row: Vec = columns + .iter() + .zip(&declared) + .map(|(column, declared)| { + render_state_sql_literal(py, &record.getattr(column.as_str())?, declared) + }) + .collect::>()?; + rows.push(row); + } + let names: Vec<&str> = columns.iter().map(String::as_str).collect(); + Ok(values_insert_sql( + &table.extract::()?, + &names, + &rows, + )) +} + +pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { + module.add_function(wrap_pyfunction!(render_create_sql, module)?)?; + module.add_function(wrap_pyfunction!( + render_insert_source_freshness_records_sql, + module + )?) +} diff --git a/crates/sqlbuild-state/src/lib.rs b/crates/sqlbuild-state/src/lib.rs index 3e6c8463ae..22bec10215 100644 --- a/crates/sqlbuild-state/src/lib.rs +++ b/crates/sqlbuild-state/src/lib.rs @@ -5,3 +5,4 @@ pub mod events; pub mod execution_history; pub mod freshness; +pub mod state_tables; diff --git a/crates/sqlbuild-state/src/state_tables/main/create_table_sql.rs b/crates/sqlbuild-state/src/state_tables/main/create_table_sql.rs new file mode 100644 index 0000000000..e164cf9037 --- /dev/null +++ b/crates/sqlbuild-state/src/state_tables/main/create_table_sql.rs @@ -0,0 +1,18 @@ +//! `create_table_sql`: create-if-missing DDL for one append-only state table. + +use crate::state_tables::models::TableColumn; + +/// `CREATE TABLE IF NOT EXISTS ( [ NOT NULL], ...)`. +pub fn create_table_sql(qualified_name: &str, columns: &[TableColumn]) -> String { + let definitions: Vec = columns + .iter() + .map(|column| { + let required: &str = if column.required { " NOT NULL" } else { "" }; + format!("{} {}{required}", column.name, column.rendered_type) + }) + .collect(); + format!( + "CREATE TABLE IF NOT EXISTS {qualified_name} ({})", + definitions.join(", ") + ) +} diff --git a/crates/sqlbuild-state/src/state_tables/main/mod.rs b/crates/sqlbuild-state/src/state_tables/main/mod.rs new file mode 100644 index 0000000000..712abe33fc --- /dev/null +++ b/crates/sqlbuild-state/src/state_tables/main/mod.rs @@ -0,0 +1,2 @@ +pub mod create_table_sql; +pub mod values_insert_sql; diff --git a/crates/sqlbuild-state/src/state_tables/main/values_insert_sql.rs b/crates/sqlbuild-state/src/state_tables/main/values_insert_sql.rs new file mode 100644 index 0000000000..ce9212da9f --- /dev/null +++ b/crates/sqlbuild-state/src/state_tables/main/values_insert_sql.rs @@ -0,0 +1,14 @@ +//! `values_insert_sql`: rows of rendered literals appended with one `INSERT ... VALUES`. + +/// `INSERT INTO
() VALUES (), ...`. +pub fn values_insert_sql(qualified_name: &str, columns: &[&str], rows: &[Vec]) -> String { + let values: Vec = rows + .iter() + .map(|row| format!("({})", row.join(", "))) + .collect(); + format!( + "INSERT INTO {qualified_name} ({}) VALUES {}", + columns.join(", "), + values.join(", ") + ) +} diff --git a/crates/sqlbuild-state/src/state_tables/mod.rs b/crates/sqlbuild-state/src/state_tables/mod.rs new file mode 100644 index 0000000000..1a70a48981 --- /dev/null +++ b/crates/sqlbuild-state/src/state_tables/mod.rs @@ -0,0 +1,5 @@ +pub mod main; +pub mod models; + +#[cfg(test)] +mod tests; diff --git a/crates/sqlbuild-state/src/state_tables/models.rs b/crates/sqlbuild-state/src/state_tables/models.rs new file mode 100644 index 0000000000..7a343e134d --- /dev/null +++ b/crates/sqlbuild-state/src/state_tables/models.rs @@ -0,0 +1,9 @@ +//! Columns of append-only state tables as their DDL declares them. + +/// One declared column: its name, the adapter's rendered type and whether it is `NOT NULL`. +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct TableColumn { + pub name: String, + pub rendered_type: String, + pub required: bool, +} diff --git a/crates/sqlbuild-state/src/state_tables/tests.rs b/crates/sqlbuild-state/src/state_tables/tests.rs new file mode 100644 index 0000000000..4af576b46f --- /dev/null +++ b/crates/sqlbuild-state/src/state_tables/tests.rs @@ -0,0 +1,6 @@ +#[path = "tests/helpers.rs"] +mod helpers; +#[path = "tests/test_sql.rs"] +mod sql; +#[path = "tests/test_types.rs"] +mod test_types; diff --git a/crates/sqlbuild-state/src/state_tables/tests/helpers.rs b/crates/sqlbuild-state/src/state_tables/tests/helpers.rs new file mode 100644 index 0000000000..e69081ecf1 --- /dev/null +++ b/crates/sqlbuild-state/src/state_tables/tests/helpers.rs @@ -0,0 +1,10 @@ +use crate::state_tables::models::TableColumn; + +/// One declared column of a test table. +pub(crate) fn column(name: &str, rendered_type: &str, required: bool) -> TableColumn { + TableColumn { + name: name.to_owned(), + rendered_type: rendered_type.to_owned(), + required, + } +} diff --git a/crates/sqlbuild-state/src/state_tables/tests/test_sql.rs b/crates/sqlbuild-state/src/state_tables/tests/test_sql.rs new file mode 100644 index 0000000000..5c9ef41669 --- /dev/null +++ b/crates/sqlbuild-state/src/state_tables/tests/test_sql.rs @@ -0,0 +1,50 @@ +use crate::state_tables::main::create_table_sql::create_table_sql; +use crate::state_tables::main::values_insert_sql::values_insert_sql; +use crate::state_tables::tests::helpers::column; +use crate::state_tables::tests::test_types::{CreateTableTestCase, ValuesInsertTestCase}; + +#[test] +fn given_columns_when_rendering_ddl_then_required_columns_are_not_null() { + let test_cases = [ + CreateTableTestCase { + description: "mixed required columns", + columns: vec![column("a", "VARCHAR", true), column("b", "BIGINT", false)], + expected_sql: "CREATE TABLE IF NOT EXISTS s.t (a VARCHAR NOT NULL, b BIGINT)", + }, + CreateTableTestCase { + description: "no columns", + columns: Vec::new(), + expected_sql: "CREATE TABLE IF NOT EXISTS s.t ()", + }, + ]; + + for test_case in test_cases { + assert_eq!( + create_table_sql("s.t", &test_case.columns), + test_case.expected_sql, + "{}", + test_case.description + ); + } +} + +#[test] +fn given_rows_when_rendering_insert_then_each_row_is_one_values_tuple() { + let test_cases = [ValuesInsertTestCase { + description: "two rows", + rows: vec![ + vec!["'a'".to_owned(), "NULL".to_owned()], + vec!["'b'".to_owned(), "'c'".to_owned()], + ], + expected_sql: "INSERT INTO s.t (x, y) VALUES ('a', NULL), ('b', 'c')", + }]; + + for test_case in test_cases { + assert_eq!( + values_insert_sql("s.t", &["x", "y"], &test_case.rows), + test_case.expected_sql, + "{}", + test_case.description + ); + } +} diff --git a/crates/sqlbuild-state/src/state_tables/tests/test_types.rs b/crates/sqlbuild-state/src/state_tables/tests/test_types.rs new file mode 100644 index 0000000000..836ccf548d --- /dev/null +++ b/crates/sqlbuild-state/src/state_tables/tests/test_types.rs @@ -0,0 +1,13 @@ +use crate::state_tables::models::TableColumn; + +pub(crate) struct CreateTableTestCase { + pub(crate) description: &'static str, + pub(crate) columns: Vec, + pub(crate) expected_sql: &'static str, +} + +pub(crate) struct ValuesInsertTestCase { + pub(crate) description: &'static str, + pub(crate) rows: Vec>, + pub(crate) expected_sql: &'static str, +} diff --git a/python-boundary.toml b/python-boundary.toml index e066008ffd..e6a9229456 100644 --- a/python-boundary.toml +++ b/python-boundary.toml @@ -29,11 +29,6 @@ path = "adapter/relations" reason = "adapter" note = "Relation inspection wrappers adapters call, with per-connection caches; executed-statement classification and invalidation keys are native." -[[stays]] -path = "adapter/state_sql" -reason = "adapter" -note = "State-table SQL rendered through adapter callbacks." - [[stays]] path = "adapter/type_system" reason = "adapter" @@ -678,6 +673,7 @@ note = "Shape rule: data records only (CompileReuseRequest, SettingsEnvironmentI path = "compiler/source_freshness/constants.py" reason = "adapter" note = "Shape rule: source freshness column names and adapter column types, no logic; SOURCE_FRESHNESS_TABLE_NAME is re-exported from sqlbuild._native." +note = "Shape rule: data constants only (SOURCE_FRESHNESS_TABLE_NAME, COLUMN_SOURCE_NAME, COLUMN_TARGET_DATABASE, COLUMN_TARGET_SCHEMA, ...), no logic; consumed by adapter/contract/classes/duckdb_backed_adapter.py, adapters/bigquery/classes/bigquery_adapter.py, adapters/databricks/classes/databricks_adapter.py and 3 more, and read natively by the source freshness state bindings." [[stays]] path = "compiler/auditing/types.py" diff --git a/src/sqlbuild/_native.pyi b/src/sqlbuild/_native.pyi index 6f1ad030d1..db071fdcfc 100644 --- a/src/sqlbuild/_native.pyi +++ b/src/sqlbuild/_native.pyi @@ -5022,6 +5022,21 @@ def normalize_sql_value( limits: SqlValueLimits | None = None, ) -> SqlValue: ... def render_state_sql_literal(*, value: object | None, declared_type: StateSqlValueType) -> str: ... +def render_state_table_create_sql( + *, + qualified_name: str, + columns: tuple[str, ...], + column_types: Mapping[str, StateSqlValueType], + required_columns: frozenset[str], + render_framework_type: Callable[[FrameworkType], str], +) -> str: ... +def render_insert_source_freshness_records_sql( + *, + database: str | None, + schema: str, + records: tuple[SourceFreshnessRecord, ...], + render_qualified_name: Callable[..., str | None], +) -> str: ... def model_header_declarations( header: tuple[object | None, object | None], model_name: str, relative_path: Path, / ) -> tuple[tuple[object, ...], tuple[object, ...]]: ... diff --git a/src/sqlbuild/adapter/contract/classes/base_adapter.py b/src/sqlbuild/adapter/contract/classes/base_adapter.py index ad513a0bc5..cc5a230ce8 100644 --- a/src/sqlbuild/adapter/contract/classes/base_adapter.py +++ b/src/sqlbuild/adapter/contract/classes/base_adapter.py @@ -15,6 +15,7 @@ SqlLexicalSyntax, SqlValue, SqlValueKind, + render_insert_source_freshness_records_sql, ) from sqlbuild.adapter.contract.classes.historical_check_snapshot_sql import ( HistoricalCheckSnapshotSql, @@ -80,9 +81,6 @@ from sqlbuild.adapter.relations.main.get_columns_for_relations import ( get_columns_for_relations_bulk, ) -from sqlbuild.adapter.state_sql.main.render_insert_source_freshness_records_sql import ( - render_insert_source_freshness_records_sql, -) from sqlbuild.compiler.compile.types import FunctionLanguage from sqlbuild.spec.contracts.constants import DEFAULT_SEED_CSV_SETTINGS from sqlbuild.sql_values.exceptions import SqlValueRenderingError diff --git a/src/sqlbuild/adapter/contract/classes/duckdb_backed_adapter.py b/src/sqlbuild/adapter/contract/classes/duckdb_backed_adapter.py index 934de58d39..7d3f607161 100644 --- a/src/sqlbuild/adapter/contract/classes/duckdb_backed_adapter.py +++ b/src/sqlbuild/adapter/contract/classes/duckdb_backed_adapter.py @@ -12,7 +12,13 @@ from types import ModuleType from typing import Any, ClassVar -from sqlbuild._native import SeedCsvSettings, SourceFreshnessRecord, SqlLexicalSyntax, SqlValue +from sqlbuild._native import ( + SeedCsvSettings, + SourceFreshnessRecord, + SqlLexicalSyntax, + SqlValue, + render_insert_source_freshness_records_sql, +) from sqlbuild.adapter.contract._helpers.seed_csv import get_seed_csv_null_values from sqlbuild.adapter.contract.classes.base_adapter import ( BaseAdapter, @@ -79,9 +85,6 @@ from sqlbuild.adapter.relations.main.get_columns_for_relations import ( get_columns_for_relations_bulk, ) -from sqlbuild.adapter.state_sql.main.render_insert_source_freshness_records_sql import ( - render_insert_source_freshness_records_sql, -) from sqlbuild.adapter.type_system.main.first_arg_nullability import first_arg_nullability from sqlbuild.adapter.type_system.main.normalize_numeric_family import normalize_numeric_family from sqlbuild.adapter.type_system.main.types_equal import types_equal diff --git a/src/sqlbuild/adapter/state_sql/__init__.py b/src/sqlbuild/adapter/state_sql/__init__.py deleted file mode 100644 index 0163779a0a..0000000000 --- a/src/sqlbuild/adapter/state_sql/__init__.py +++ /dev/null @@ -1 +0,0 @@ -"""Adapter-owned state SQL rendering.""" diff --git a/src/sqlbuild/adapter/state_sql/_helpers/__init__.py b/src/sqlbuild/adapter/state_sql/_helpers/__init__.py deleted file mode 100644 index ce3f537a3d..0000000000 --- a/src/sqlbuild/adapter/state_sql/_helpers/__init__.py +++ /dev/null @@ -1 +0,0 @@ -"""Adapter state SQL implementations.""" diff --git a/src/sqlbuild/adapter/state_sql/_helpers/source_freshness.py b/src/sqlbuild/adapter/state_sql/_helpers/source_freshness.py deleted file mode 100644 index 904604cbe9..0000000000 --- a/src/sqlbuild/adapter/state_sql/_helpers/source_freshness.py +++ /dev/null @@ -1,81 +0,0 @@ -"""Adapter-owned renderers for source freshness state.""" - -from __future__ import annotations - -from collections.abc import Callable - -from sqlbuild._native import SourceFreshnessRecord, render_state_sql_literal -from sqlbuild.adapter.contract.exceptions import AdapterUserError -from sqlbuild.compiler.source_freshness.constants import ( - COLUMN_DATA_VERSION, - COLUMN_DATA_VERSION_HASH, - COLUMN_OBSERVED_AT, - COLUMN_RUN_ID, - COLUMN_SOURCE_NAME, - COLUMN_STRATEGY, - COLUMN_TARGET_DATABASE, - COLUMN_TARGET_NAME, - COLUMN_TARGET_SCHEMA, - COLUMN_VALUE_KIND, - SOURCE_FRESHNESS_COLUMN_TYPES, - SOURCE_FRESHNESS_COLUMNS, - SOURCE_FRESHNESS_TABLE_NAME, -) - - -def render_insert_source_freshness_records_sql( - *, - database: str | None, - schema: str, - records: tuple[SourceFreshnessRecord, ...], - render_qualified_name: Callable[..., str | None], -) -> str: - """Render DML that appends source freshness records.""" - - table_name: str | None = render_qualified_name( - database=database, - schema=schema, - name=SOURCE_FRESHNESS_TABLE_NAME, - ) - if table_name is None: - raise AdapterUserError(message="source freshness table requires a target schema") - values_sql: str = ", ".join(_record_values_sql(record) for record in records) - return ( - f"INSERT INTO {table_name} (" - f"{COLUMN_SOURCE_NAME}, " - f"{COLUMN_TARGET_DATABASE}, " - f"{COLUMN_TARGET_SCHEMA}, " - f"{COLUMN_TARGET_NAME}, " - f"{COLUMN_RUN_ID}, " - f"{COLUMN_STRATEGY}, " - f"{COLUMN_VALUE_KIND}, " - f"{COLUMN_DATA_VERSION}, " - f"{COLUMN_DATA_VERSION_HASH}, " - f"{COLUMN_OBSERVED_AT}" - f") VALUES {values_sql}" - ) - - -def _record_values_sql(record: SourceFreshnessRecord) -> str: - values: tuple[object | None, ...] = ( - record.source_name, - record.target_database, - record.target_schema, - record.target_name, - record.run_id, - record.strategy, - record.value_kind, - record.data_version, - record.data_version_hash, - record.observed_at, - ) - return ( - "(" - + ", ".join( - render_state_sql_literal( - value=value, declared_type=SOURCE_FRESHNESS_COLUMN_TYPES[column] - ) - for column, value in zip(SOURCE_FRESHNESS_COLUMNS, values, strict=True) - ) - + ")" - ) diff --git a/src/sqlbuild/adapter/state_sql/_helpers/table_ddl.py b/src/sqlbuild/adapter/state_sql/_helpers/table_ddl.py deleted file mode 100644 index 86fb8e7095..0000000000 --- a/src/sqlbuild/adapter/state_sql/_helpers/table_ddl.py +++ /dev/null @@ -1,30 +0,0 @@ -"""Portable append-only state table DDL implementations.""" - -from __future__ import annotations - -from collections.abc import Callable, Mapping - -from sqlbuild._native import StateSqlValueType -from sqlbuild.adapter.contract.types import FrameworkType - - -def render_state_table_create_sql_impl( - *, - qualified_name: str, - columns: tuple[str, ...], - column_types: Mapping[str, StateSqlValueType], - required_columns: frozenset[str], - render_framework_type: Callable[[FrameworkType], str], -) -> str: - """Render create-if-missing DDL for string, integer, and text-timestamp state columns.""" - - rendered_types: dict[StateSqlValueType, str] = { - StateSqlValueType.STRING: render_framework_type(FrameworkType.STRING), - StateSqlValueType.INTEGER: render_framework_type(FrameworkType.INTEGER), - StateSqlValueType.TEXT_TIMESTAMP: render_framework_type(FrameworkType.TIMESTAMP), - } - definitions: list[str] = [] - for column in columns: - required: str = " NOT NULL" if column in required_columns else "" - definitions.append(f"{column} {rendered_types[column_types[column]]}{required}") - return f"CREATE TABLE IF NOT EXISTS {qualified_name} ({', '.join(definitions)})" diff --git a/src/sqlbuild/adapter/state_sql/main/__init__.py b/src/sqlbuild/adapter/state_sql/main/__init__.py deleted file mode 100644 index 810274fc07..0000000000 --- a/src/sqlbuild/adapter/state_sql/main/__init__.py +++ /dev/null @@ -1 +0,0 @@ -"""Public adapter state SQL entries.""" diff --git a/src/sqlbuild/adapter/state_sql/main/render_insert_source_freshness_records_sql.py b/src/sqlbuild/adapter/state_sql/main/render_insert_source_freshness_records_sql.py deleted file mode 100644 index 33a2db2e93..0000000000 --- a/src/sqlbuild/adapter/state_sql/main/render_insert_source_freshness_records_sql.py +++ /dev/null @@ -1,25 +0,0 @@ -"""Public source freshness insert rendering operation.""" - -from collections.abc import Callable - -from sqlbuild._native import SourceFreshnessRecord -from sqlbuild.adapter.state_sql._helpers.source_freshness import ( - render_insert_source_freshness_records_sql as _render_sql, -) - - -def render_insert_source_freshness_records_sql( - *, - database: str | None, - schema: str, - records: tuple[SourceFreshnessRecord, ...], - render_qualified_name: Callable[..., str | None], -) -> str: - """Render DML that appends source freshness records.""" - - return _render_sql( - database=database, - schema=schema, - records=records, - render_qualified_name=render_qualified_name, - ) diff --git a/src/sqlbuild/adapter/state_sql/main/render_state_table_create_sql.py b/src/sqlbuild/adapter/state_sql/main/render_state_table_create_sql.py deleted file mode 100644 index 4b9c1fe1ff..0000000000 --- a/src/sqlbuild/adapter/state_sql/main/render_state_table_create_sql.py +++ /dev/null @@ -1,26 +0,0 @@ -"""Public append-only state table DDL rendering operation.""" - -from collections.abc import Callable, Mapping - -from sqlbuild._native import StateSqlValueType -from sqlbuild.adapter.contract.types import FrameworkType -from sqlbuild.adapter.state_sql._helpers.table_ddl import render_state_table_create_sql_impl - - -def render_state_table_create_sql( - *, - qualified_name: str, - columns: tuple[str, ...], - column_types: Mapping[str, StateSqlValueType], - required_columns: frozenset[str], - render_framework_type: Callable[[FrameworkType], str], -) -> str: - """Render create-if-missing DDL for one append-only state table.""" - - return render_state_table_create_sql_impl( - qualified_name=qualified_name, - columns=columns, - column_types=column_types, - required_columns=required_columns, - render_framework_type=render_framework_type, - ) diff --git a/src/sqlbuild/adapters/bigquery/classes/bigquery_adapter.py b/src/sqlbuild/adapters/bigquery/classes/bigquery_adapter.py index 27d1c538aa..e0356b7fd3 100644 --- a/src/sqlbuild/adapters/bigquery/classes/bigquery_adapter.py +++ b/src/sqlbuild/adapters/bigquery/classes/bigquery_adapter.py @@ -16,6 +16,7 @@ SqlLexicalSyntax, SqlValue, SqlValueKind, + render_insert_source_freshness_records_sql, ) from sqlbuild.adapter.contract.classes.base_adapter import ( _COLLECTION_SQL_VALUE_KINDS, @@ -105,9 +106,6 @@ ) from sqlbuild.adapter.relations.main.invalidate_cached_relations import invalidate_cached_relations from sqlbuild.adapter.relations.main.relation_age_timestamp import relation_age_timestamp_utc -from sqlbuild.adapter.state_sql.main.render_insert_source_freshness_records_sql import ( - render_insert_source_freshness_records_sql, -) from sqlbuild.adapter.type_system.main.conditional_result_nullability import ( conditional_result_nullability, ) diff --git a/src/sqlbuild/adapters/databricks/classes/databricks_adapter.py b/src/sqlbuild/adapters/databricks/classes/databricks_adapter.py index 5b18d976db..00caaef22e 100644 --- a/src/sqlbuild/adapters/databricks/classes/databricks_adapter.py +++ b/src/sqlbuild/adapters/databricks/classes/databricks_adapter.py @@ -13,7 +13,13 @@ from pathlib import Path from typing import Any, ClassVar -from sqlbuild._native import SeedCsvSettings, SourceFreshnessRecord, SqlLexicalSyntax, SqlValue +from sqlbuild._native import ( + SeedCsvSettings, + SourceFreshnessRecord, + SqlLexicalSyntax, + SqlValue, + render_insert_source_freshness_records_sql, +) from sqlbuild.adapter.contract.classes.base_adapter import ( BaseAdapter, _encode_typed_json, @@ -97,9 +103,6 @@ get_columns_for_relations_bulk, ) from sqlbuild.adapter.relations.main.relation_age_timestamp import relation_age_timestamp_utc -from sqlbuild.adapter.state_sql.main.render_insert_source_freshness_records_sql import ( - render_insert_source_freshness_records_sql, -) from sqlbuild.adapter.type_system.main.conditional_result_nullability import ( conditional_result_nullability, ) diff --git a/src/sqlbuild/adapters/postgres/classes/postgres_adapter.py b/src/sqlbuild/adapters/postgres/classes/postgres_adapter.py index 06f5ca83fb..4514800e9f 100644 --- a/src/sqlbuild/adapters/postgres/classes/postgres_adapter.py +++ b/src/sqlbuild/adapters/postgres/classes/postgres_adapter.py @@ -10,7 +10,13 @@ from pathlib import Path from typing import Any, ClassVar -from sqlbuild._native import SeedCsvSettings, SourceFreshnessRecord, SqlLexicalSyntax, SqlValue +from sqlbuild._native import ( + SeedCsvSettings, + SourceFreshnessRecord, + SqlLexicalSyntax, + SqlValue, + render_insert_source_freshness_records_sql, +) from sqlbuild.adapter.contract.classes.base_adapter import ( BaseAdapter, _build_names_filter, @@ -88,9 +94,6 @@ from sqlbuild.adapter.relations.main.get_columns_for_relations import ( get_columns_for_relations_bulk, ) -from sqlbuild.adapter.state_sql.main.render_insert_source_freshness_records_sql import ( - render_insert_source_freshness_records_sql, -) from sqlbuild.adapter.type_system.main.normalize_numeric_family import normalize_numeric_family from sqlbuild.adapter.type_system.main.types_equal import types_equal from sqlbuild.adapters.postgres._helpers.grants import ( diff --git a/src/sqlbuild/adapters/snowflake/classes/snowflake_adapter.py b/src/sqlbuild/adapters/snowflake/classes/snowflake_adapter.py index 229155db69..c6affbc356 100644 --- a/src/sqlbuild/adapters/snowflake/classes/snowflake_adapter.py +++ b/src/sqlbuild/adapters/snowflake/classes/snowflake_adapter.py @@ -19,6 +19,7 @@ SqlValue, TableType, collect_snowflake_cost, + render_insert_source_freshness_records_sql, ) from sqlbuild.adapter.contract.classes.base_adapter import ( BaseAdapter, @@ -110,9 +111,6 @@ SchemaColumnListing, SchemaRelationListing, ) -from sqlbuild.adapter.state_sql.main.render_insert_source_freshness_records_sql import ( - render_insert_source_freshness_records_sql, -) from sqlbuild.adapter.type_system.main.conditional_result_nullability import ( conditional_result_nullability, ) diff --git a/src/sqlbuild/adapters/sqlserver/classes/sqlserver_adapter.py b/src/sqlbuild/adapters/sqlserver/classes/sqlserver_adapter.py index 4527779538..c6768c3aa6 100644 --- a/src/sqlbuild/adapters/sqlserver/classes/sqlserver_adapter.py +++ b/src/sqlbuild/adapters/sqlserver/classes/sqlserver_adapter.py @@ -16,6 +16,7 @@ SqlLexicalSyntax, SqlValue, SqlValueKind, + render_insert_source_freshness_records_sql, ) from sqlbuild.adapter.contract.classes.base_adapter import ( BaseAdapter, @@ -96,9 +97,6 @@ from sqlbuild.adapter.relations.main.get_columns_for_relations import ( get_columns_for_relations_bulk, ) -from sqlbuild.adapter.state_sql.main.render_insert_source_freshness_records_sql import ( - render_insert_source_freshness_records_sql, -) from sqlbuild.adapters.sqlserver._helpers.grants import ( render_sqlserver_view_grants, render_sqlserver_view_revokes, diff --git a/src/sqlbuild/executor/audit_results/_helpers/sql.py b/src/sqlbuild/executor/audit_results/_helpers/sql.py index 986206bad2..cd79175624 100644 --- a/src/sqlbuild/executor/audit_results/_helpers/sql.py +++ b/src/sqlbuild/executor/audit_results/_helpers/sql.py @@ -4,11 +4,8 @@ from collections.abc import Callable, Sequence -from sqlbuild._native import render_state_sql_literal +from sqlbuild._native import render_state_sql_literal, render_state_table_create_sql from sqlbuild.adapter.contract.types import FrameworkType -from sqlbuild.adapter.state_sql.main.render_state_table_create_sql import ( - render_state_table_create_sql, -) from sqlbuild.errors.contracts.exceptions import ExecutorInputError from sqlbuild.executor.audit_results.constants import ( AUDIT_RESULT_COLUMN_TYPES, diff --git a/src/sqlbuild/executor/janitor_events/_helpers/sql.py b/src/sqlbuild/executor/janitor_events/_helpers/sql.py index 5d85cae77f..1de4e10a49 100644 --- a/src/sqlbuild/executor/janitor_events/_helpers/sql.py +++ b/src/sqlbuild/executor/janitor_events/_helpers/sql.py @@ -4,11 +4,8 @@ from collections.abc import Callable -from sqlbuild._native import render_state_sql_literal +from sqlbuild._native import render_state_sql_literal, render_state_table_create_sql from sqlbuild.adapter.contract.types import FrameworkType -from sqlbuild.adapter.state_sql.main.render_state_table_create_sql import ( - render_state_table_create_sql, -) from sqlbuild.errors.contracts.exceptions import ExecutorInputError from sqlbuild.executor.janitor_events.constants import ( COLUMN_ARCHIVE_NAME, diff --git a/src/sqlbuild/executor/node_results/_helpers/sql.py b/src/sqlbuild/executor/node_results/_helpers/sql.py index d8669e4ee9..003996c925 100644 --- a/src/sqlbuild/executor/node_results/_helpers/sql.py +++ b/src/sqlbuild/executor/node_results/_helpers/sql.py @@ -4,11 +4,8 @@ from collections.abc import Callable -from sqlbuild._native import render_state_sql_literal +from sqlbuild._native import render_state_sql_literal, render_state_table_create_sql from sqlbuild.adapter.contract.types import FrameworkType -from sqlbuild.adapter.state_sql.main.render_state_table_create_sql import ( - render_state_table_create_sql, -) from sqlbuild.errors.contracts.exceptions import ExecutorInputError from sqlbuild.executor.node_results.constants import ( COLUMN_ERROR_MESSAGE, diff --git a/tests/unit/src/sqlbuild/adapter/state_sql/_test_types.py b/tests/unit/src/sqlbuild/adapter/state_sql/_test_types.py deleted file mode 100644 index 7bdcd5cf85..0000000000 --- a/tests/unit/src/sqlbuild/adapter/state_sql/_test_types.py +++ /dev/null @@ -1,7 +0,0 @@ -from dataclasses import dataclass - - -@dataclass(frozen=True) -class StateSqlGoldenTestCase: - description: str - expected_sql: str diff --git a/tests/unit/src/sqlbuild/adapter/state_sql/test_source_freshness.py b/tests/unit/src/sqlbuild/adapter/state_sql/test_source_freshness.py deleted file mode 100644 index a525402edd..0000000000 --- a/tests/unit/src/sqlbuild/adapter/state_sql/test_source_freshness.py +++ /dev/null @@ -1,60 +0,0 @@ -"""Golden SQL tests for adapter-owned source freshness state.""" - -from __future__ import annotations - -from datetime import datetime, timedelta, timezone - -import pytest - -from sqlbuild._native import SourceFreshnessRecord -from sqlbuild.adapter.state_sql._helpers.source_freshness import ( - render_insert_source_freshness_records_sql, -) -from sqlbuild.adapters.duckdb.classes.duckdb_adapter import DuckDbAdapter -from tests.unit.src.sqlbuild.adapter.state_sql._test_types import StateSqlGoldenTestCase - - -@pytest.mark.parametrize( - "test_case", - [ - StateSqlGoldenTestCase( - description="representative source freshness insert", - expected_sql=( - "INSERT INTO analytics._sqlbuild_source_freshness (source_name, " - "target_database, target_schema, target_name, run_id, strategy, value_kind, " - "data_version, data_version_hash, observed_at) VALUES ('raw.o''rders', NULL, " - "'raw', 'orders', 'run-1', 'adapter', 'timestamp', " - "'2026-01-01T01:00:00+01:00', 'hash', '2026-01-01T00:00:00+00:00')" - ), - ) - ], - ids=lambda case: case.description, -) -def test_given_representative_freshness_record_when_rendering_then_sql_matches_golden( - test_case: StateSqlGoldenTestCase, -) -> None: - sql: str = render_insert_source_freshness_records_sql( - database=None, - schema="analytics", - records=( - SourceFreshnessRecord( - source_name="raw.o'rders", - target_database=None, - target_schema="raw", - target_name="orders", - run_id="run-1", - strategy="adapter", - value_kind="timestamp", - data_version="2026-01-01T01:00:00+01:00", - data_version_hash="hash", - observed_at=datetime(2026, 1, 1, 1, tzinfo=timezone(timedelta(hours=1))), - ), - ), - render_qualified_name=DuckDbAdapter().render_qualified_name, - ) - - assert sql == test_case.expected_sql - - -if __name__ == "__main__": - pytest.main([__file__, "-vv"]) diff --git a/tests/unit/src/sqlbuild/sql_values/_test_types.py b/tests/unit/src/sqlbuild/sql_values/_test_types.py index b79e8082f9..5ca8456e01 100644 --- a/tests/unit/src/sqlbuild/sql_values/_test_types.py +++ b/tests/unit/src/sqlbuild/sql_values/_test_types.py @@ -1,6 +1,6 @@ from dataclasses import dataclass -from sqlbuild._native import SqlValueLimits +from sqlbuild._native import SqlValueLimits, StateSqlValueType @dataclass(frozen=True) @@ -38,3 +38,16 @@ class SqlValueBehaviorTestCase: class StateLiteralGoldenTestCase: description: str expected_literals: dict[str, str] + + +@dataclass(frozen=True) +class StateSqlGoldenTestCase: + description: str + expected_sql: str + + +@dataclass(frozen=True) +class StateSqlTypeErrorTestCase: + description: str + column_type: StateSqlValueType + expected_error: type[Exception] diff --git a/tests/unit/src/sqlbuild/sql_values/test_state_table_sql.py b/tests/unit/src/sqlbuild/sql_values/test_state_table_sql.py new file mode 100644 index 0000000000..74e61e3e14 --- /dev/null +++ b/tests/unit/src/sqlbuild/sql_values/test_state_table_sql.py @@ -0,0 +1,120 @@ +"""Golden SQL tests for native append-only state table rendering.""" + +from __future__ import annotations + +from datetime import datetime, timedelta, timezone + +import pytest + +from sqlbuild._native import ( + SourceFreshnessRecord, + StateSqlValueType, + render_insert_source_freshness_records_sql, + render_state_table_create_sql, +) +from sqlbuild.adapters.duckdb.classes.duckdb_adapter import DuckDbAdapter +from tests.unit.src.sqlbuild.sql_values._test_types import ( + StateSqlGoldenTestCase, + StateSqlTypeErrorTestCase, +) + + +@pytest.mark.parametrize( + "test_case", + [ + StateSqlGoldenTestCase( + description="representative source freshness insert", + expected_sql=( + "INSERT INTO analytics._sqlbuild_source_freshness (source_name, " + "target_database, target_schema, target_name, run_id, strategy, value_kind, " + "data_version, data_version_hash, observed_at) VALUES ('raw.o''rders', NULL, " + "'raw', 'orders', 'run-1', 'adapter', 'timestamp', " + "'2026-01-01T01:00:00+01:00', 'hash', '2026-01-01T00:00:00+00:00')" + ), + ) + ], + ids=lambda case: case.description, +) +def test_given_representative_freshness_record_when_rendering_then_sql_matches_golden( + test_case: StateSqlGoldenTestCase, +) -> None: + sql: str = render_insert_source_freshness_records_sql( + database=None, + schema="analytics", + records=( + SourceFreshnessRecord( + source_name="raw.o'rders", + target_database=None, + target_schema="raw", + target_name="orders", + run_id="run-1", + strategy="adapter", + value_kind="timestamp", + data_version="2026-01-01T01:00:00+01:00", + data_version_hash="hash", + observed_at=datetime(2026, 1, 1, 1, tzinfo=timezone(timedelta(hours=1))), + ), + ), + render_qualified_name=DuckDbAdapter().render_qualified_name, + ) + + assert sql == test_case.expected_sql + + +@pytest.mark.parametrize( + "test_case", + [ + StateSqlGoldenTestCase( + description="string integer and text timestamp columns with required columns", + expected_sql=( + "CREATE TABLE IF NOT EXISTS main.state " + "(name VARCHAR NOT NULL, row_count BIGINT, observed_at TIMESTAMP NOT NULL)" + ), + ) + ], + ids=lambda case: case.description, +) +def test_given_state_columns_when_rendering_table_ddl_then_sql_matches_golden( + test_case: StateSqlGoldenTestCase, +) -> None: + sql: str = render_state_table_create_sql( + qualified_name="main.state", + columns=("name", "row_count", "observed_at"), + column_types={ + "name": StateSqlValueType.STRING, + "row_count": StateSqlValueType.INTEGER, + "observed_at": StateSqlValueType.TEXT_TIMESTAMP, + }, + required_columns=frozenset({"name", "observed_at"}), + render_framework_type=DuckDbAdapter().render_framework_type, + ) + + assert sql == test_case.expected_sql + + +@pytest.mark.parametrize( + "test_case", + [ + StateSqlTypeErrorTestCase( + description="boolean columns are not state table columns", + column_type=StateSqlValueType.BOOLEAN, + expected_error=KeyError, + ) + ], + ids=lambda case: case.description, +) +def test_given_unsupported_column_type_when_rendering_table_ddl_then_raises_error( + test_case: StateSqlTypeErrorTestCase, +) -> None: + with pytest.raises(test_case.expected_error): + render_state_table_create_sql( + qualified_name="main.state", + columns=("flag",), + column_types={"flag": test_case.column_type}, + required_columns=frozenset(), + render_framework_type=DuckDbAdapter().render_framework_type, + ) + + +if __name__ == "__main__": + pytest.main([__file__, "-vv"]) From af40f391feabefe242ca397ed748bd1d530adca4 Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Thu, 15 Oct 2026 18:28:03 +0100 Subject: [PATCH 27/48] refactor: format diagnostics console lines natively behind the logging formatter --- .../src/diagnostic_text/_helpers/mod.rs | 1 + .../diagnostic_text/_helpers/record_parts.rs | 58 +++++++++++ .../src/diagnostic_text/constants.rs | 8 ++ .../main/format_diagnostic_record.rs | 24 +++++ .../src/diagnostic_text/main/mod.rs | 1 + .../sqlbuild-core/src/diagnostic_text/mod.rs | 7 ++ .../src/diagnostic_text/models.rs | 19 ++++ .../src/diagnostic_text/tests.rs | 4 + .../src/diagnostic_text/tests/test_format.rs | 74 ++++++++++++++ .../src/diagnostic_text/tests/test_types.rs | 8 ++ crates/sqlbuild-core/src/lib.rs | 1 + .../_helpers/diagnostics/console_records.rs | 71 +++++++++++++ .../src/bindings/_helpers/diagnostics/mod.rs | 1 + .../src/bindings/main/register.rs | 1 + python-boundary.toml | 7 +- src/sqlbuild/_native.pyi | 2 + .../_helpers/logging_formatters.py | 99 ------------------- .../classes/diagnostics_console_formatter.py | 26 +---- 18 files changed, 286 insertions(+), 126 deletions(-) create mode 100644 crates/sqlbuild-core/src/diagnostic_text/_helpers/mod.rs create mode 100644 crates/sqlbuild-core/src/diagnostic_text/_helpers/record_parts.rs create mode 100644 crates/sqlbuild-core/src/diagnostic_text/constants.rs create mode 100644 crates/sqlbuild-core/src/diagnostic_text/main/format_diagnostic_record.rs create mode 100644 crates/sqlbuild-core/src/diagnostic_text/main/mod.rs create mode 100644 crates/sqlbuild-core/src/diagnostic_text/mod.rs create mode 100644 crates/sqlbuild-core/src/diagnostic_text/models.rs create mode 100644 crates/sqlbuild-core/src/diagnostic_text/tests.rs create mode 100644 crates/sqlbuild-core/src/diagnostic_text/tests/test_format.rs create mode 100644 crates/sqlbuild-core/src/diagnostic_text/tests/test_types.rs create mode 100644 crates/sqlbuild-python/src/bindings/_helpers/diagnostics/console_records.rs delete mode 100644 src/sqlbuild/diagnostics/_helpers/logging_formatters.py diff --git a/crates/sqlbuild-core/src/diagnostic_text/_helpers/mod.rs b/crates/sqlbuild-core/src/diagnostic_text/_helpers/mod.rs new file mode 100644 index 0000000000..b16f5f3402 --- /dev/null +++ b/crates/sqlbuild-core/src/diagnostic_text/_helpers/mod.rs @@ -0,0 +1 @@ +pub(crate) mod record_parts; diff --git a/crates/sqlbuild-core/src/diagnostic_text/_helpers/record_parts.rs b/crates/sqlbuild-core/src/diagnostic_text/_helpers/record_parts.rs new file mode 100644 index 0000000000..fee887e958 --- /dev/null +++ b/crates/sqlbuild-core/src/diagnostic_text/_helpers/record_parts.rs @@ -0,0 +1,58 @@ +//! The header pieces of a `--debug` console line. + +use crate::cli_text::models::CliStyle; +use crate::diagnostic_text::constants::DIAGNOSTIC_LOGGER_PREFIX; +use crate::diagnostic_text::models::DiagnosticRecordText; + +/// Whether Python's `str.isspace()` holds for `character`. +pub(crate) fn python_space(character: char) -> bool { + character.is_whitespace() || ('\u{1c}'..='\u{1f}').contains(&character) +} + +/// The message with ` SQL` lowered to ` sql`. +fn normalized_message(message: &str) -> String { + message.replace(" SQL", " sql") +} + +/// The subject, name, event, context tokens and message, when the record has a subject. +fn context_message(record: &DiagnosticRecordText, style: CliStyle) -> Option { + let subject: &String = record.subject.as_ref()?; + let mut parts: Vec = vec![subject.clone()]; + if let Some(name) = &record.name { + parts.push(style.object_name(name)); + } + if let Some(event) = &record.event { + parts.push(event.clone()); + } + for (label, value) in &record.tokens { + parts.push(format!("{label}={value}")); + } + if let Some(duration_ms) = record.duration_ms { + parts.push(format!("duration={:.2}s", duration_ms as f64 / 1000.0)); + } + let message: String = normalized_message(&record.message); + if !message.is_empty() { + parts.push(message); + } + Some(parts.join(" ")) +} + +/// The level tag and context or logger scope that open a console line. +pub(crate) fn header(record: &DiagnosticRecordText, style: CliStyle) -> String { + let level_tag: String = format!("[{}]", record.level_name.to_lowercase()); + let context: String = context_message(record, style).unwrap_or_else(|| { + let scope: &str = record + .logger_name + .strip_prefix(DIAGNOSTIC_LOGGER_PREFIX) + .unwrap_or(&record.logger_name); + format!( + "{} {}", + style.object_name(scope), + normalized_message(&record.message) + ) + }); + if !style.use_color { + return format!("{level_tag} {context}"); + } + format!("{} {context}", style.log_label(&level_tag)) +} diff --git a/crates/sqlbuild-core/src/diagnostic_text/constants.rs b/crates/sqlbuild-core/src/diagnostic_text/constants.rs new file mode 100644 index 0000000000..59af01ed09 --- /dev/null +++ b/crates/sqlbuild-core/src/diagnostic_text/constants.rs @@ -0,0 +1,8 @@ +//! Prefixes, statements and widths of `--debug` console lines. + +/// The logger prefix diagnostics console headers drop from SQLBuild logger names. +pub(crate) const DIAGNOSTIC_LOGGER_PREFIX: &str = "sqlbuild."; +/// Statements a diagnostics console line shows inline instead of in a block. +pub(crate) const INLINE_SQL_STATEMENTS: [&str; 3] = ["BEGIN", "COMMIT", "ROLLBACK"]; +/// The rule above and below a block of logged SQL. +pub(crate) const SQL_SEPARATOR_WIDTH: usize = 80; diff --git a/crates/sqlbuild-core/src/diagnostic_text/main/format_diagnostic_record.rs b/crates/sqlbuild-core/src/diagnostic_text/main/format_diagnostic_record.rs new file mode 100644 index 0000000000..2b6e9c4571 --- /dev/null +++ b/crates/sqlbuild-core/src/diagnostic_text/main/format_diagnostic_record.rs @@ -0,0 +1,24 @@ +//! `format_diagnostic_record`: one `--debug` stderr line for a Python log record. + +use crate::cli_text::models::CliStyle; +use crate::diagnostic_text::_helpers::record_parts::{header, python_space}; +use crate::diagnostic_text::constants::{INLINE_SQL_STATEMENTS, SQL_SEPARATOR_WIDTH}; +use crate::diagnostic_text::models::DiagnosticRecordText; + +/// The header, then any logged SQL inline or between separator rules; no exception text. +#[must_use] +pub fn format_diagnostic_record(record: &DiagnosticRecordText, style: CliStyle) -> String { + let header: String = header(record, style); + let Some(sql) = &record.sql else { + return header; + }; + let stripped: &str = sql.trim_matches(python_space); + if INLINE_SQL_STATEMENTS.contains(&stripped.to_uppercase().as_str()) { + return format!("{header} {stripped}"); + } + let separator: String = style.muted(&"-".repeat(SQL_SEPARATOR_WIDTH)); + format!( + "{header}\n{separator}\n{}\n{separator}", + sql.trim_end_matches(python_space) + ) +} diff --git a/crates/sqlbuild-core/src/diagnostic_text/main/mod.rs b/crates/sqlbuild-core/src/diagnostic_text/main/mod.rs new file mode 100644 index 0000000000..e001a0c182 --- /dev/null +++ b/crates/sqlbuild-core/src/diagnostic_text/main/mod.rs @@ -0,0 +1 @@ +pub mod format_diagnostic_record; diff --git a/crates/sqlbuild-core/src/diagnostic_text/mod.rs b/crates/sqlbuild-core/src/diagnostic_text/mod.rs new file mode 100644 index 0000000000..1e64f3d116 --- /dev/null +++ b/crates/sqlbuild-core/src/diagnostic_text/mod.rs @@ -0,0 +1,7 @@ +pub(crate) mod _helpers; +pub(crate) mod constants; +pub mod main; +pub mod models; + +#[cfg(test)] +mod tests; diff --git a/crates/sqlbuild-core/src/diagnostic_text/models.rs b/crates/sqlbuild-core/src/diagnostic_text/models.rs new file mode 100644 index 0000000000..415827f428 --- /dev/null +++ b/crates/sqlbuild-core/src/diagnostic_text/models.rs @@ -0,0 +1,19 @@ +//! The log record fields a `--debug` console line shows. + +/// The fields of one Python log record a diagnostics console line shows. +#[derive(Clone, Debug, Default, PartialEq)] +pub struct DiagnosticRecordText { + pub level_name: String, + pub logger_name: String, + pub message: String, + /// `sqlbuild_subject`, `sqlbuild_name` and `sqlbuild_event` when they are strings. + pub subject: Option, + pub name: Option, + pub event: Option, + /// Each `label=value` context token whose record field is a string, in display order. + pub tokens: Vec<(&'static str, String)>, + /// `sqlbuild_duration_ms` when it is an integer. + pub duration_ms: Option, + /// `sqlbuild_sql` when it is a string. + pub sql: Option, +} diff --git a/crates/sqlbuild-core/src/diagnostic_text/tests.rs b/crates/sqlbuild-core/src/diagnostic_text/tests.rs new file mode 100644 index 0000000000..9069258726 --- /dev/null +++ b/crates/sqlbuild-core/src/diagnostic_text/tests.rs @@ -0,0 +1,4 @@ +#[path = "tests/test_format.rs"] +mod format; +#[path = "tests/test_types.rs"] +mod test_types; diff --git a/crates/sqlbuild-core/src/diagnostic_text/tests/test_format.rs b/crates/sqlbuild-core/src/diagnostic_text/tests/test_format.rs new file mode 100644 index 0000000000..e947c420f1 --- /dev/null +++ b/crates/sqlbuild-core/src/diagnostic_text/tests/test_format.rs @@ -0,0 +1,74 @@ +use crate::cli_text::models::CliStyle; +use crate::diagnostic_text::main::format_diagnostic_record::format_diagnostic_record; +use crate::diagnostic_text::models::DiagnosticRecordText; +use crate::diagnostic_text::tests::test_types::DiagnosticRecordTestCase; + +#[test] +fn given_log_record_fields_when_formatting_then_matches_console_line() { + let test_cases = [ + DiagnosticRecordTestCase { + description: "plain record drops the root logger prefix", + record: DiagnosticRecordText { + level_name: "DEBUG".to_owned(), + logger_name: "sqlbuild.compiler".to_owned(), + message: "Running SQL".to_owned(), + ..DiagnosticRecordText::default() + }, + use_color: false, + expected_text: "[debug] compiler Running sql", + }, + DiagnosticRecordTestCase { + description: "context tokens and rounded duration", + record: DiagnosticRecordText { + level_name: "INFO".to_owned(), + logger_name: "sqlbuild.exec".to_owned(), + message: "done".to_owned(), + subject: Some("model".to_owned()), + name: Some("orders".to_owned()), + tokens: vec![("phase", "run".to_owned())], + duration_ms: Some(1005), + ..DiagnosticRecordText::default() + }, + use_color: true, + expected_text: "\x1b[2m[info]\x1b[0m model orders phase=run duration=1.00s done", + }, + DiagnosticRecordTestCase { + description: "transaction statements stay inline", + record: DiagnosticRecordText { + level_name: "DEBUG".to_owned(), + logger_name: "sqlbuild.sql".to_owned(), + message: "Executing".to_owned(), + sql: Some(" commit\n".to_owned()), + ..DiagnosticRecordText::default() + }, + use_color: false, + expected_text: "[debug] sql Executing commit", + }, + DiagnosticRecordTestCase { + description: "other statements sit between separators", + record: DiagnosticRecordText { + level_name: "DEBUG".to_owned(), + logger_name: "other".to_owned(), + message: String::new(), + sql: Some("SELECT 1 \n".to_owned()), + ..DiagnosticRecordText::default() + }, + use_color: false, + expected_text: "[debug] other \n--------------------------------------------------------------------------------\nSELECT 1\n--------------------------------------------------------------------------------", + }, + ]; + + for test_case in test_cases { + assert_eq!( + format_diagnostic_record( + &test_case.record, + CliStyle { + use_color: test_case.use_color + } + ), + test_case.expected_text, + "{}", + test_case.description + ); + } +} diff --git a/crates/sqlbuild-core/src/diagnostic_text/tests/test_types.rs b/crates/sqlbuild-core/src/diagnostic_text/tests/test_types.rs new file mode 100644 index 0000000000..46d67d836e --- /dev/null +++ b/crates/sqlbuild-core/src/diagnostic_text/tests/test_types.rs @@ -0,0 +1,8 @@ +use crate::diagnostic_text::models::DiagnosticRecordText; + +pub(crate) struct DiagnosticRecordTestCase { + pub(crate) description: &'static str, + pub(crate) record: DiagnosticRecordText, + pub(crate) use_color: bool, + pub(crate) expected_text: &'static str, +} diff --git a/crates/sqlbuild-core/src/lib.rs b/crates/sqlbuild-core/src/lib.rs index f099845997..7f9bf5d3a5 100644 --- a/crates/sqlbuild-core/src/lib.rs +++ b/crates/sqlbuild-core/src/lib.rs @@ -24,6 +24,7 @@ pub mod dbt_graph; pub mod dbt_plan; pub mod dbt_process; pub mod debug_report; +pub mod diagnostic_text; pub mod event_export; pub mod format_command; pub mod freshness_text; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/diagnostics/console_records.rs b/crates/sqlbuild-python/src/bindings/_helpers/diagnostics/console_records.rs new file mode 100644 index 0000000000..76a093f321 --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/diagnostics/console_records.rs @@ -0,0 +1,71 @@ +//! The `--debug` stderr line of one Python log record. + +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyModule, PyModuleMethods, PyResult}; +use pyo3::types::{PyInt, PyString}; +use pyo3::{pyfunction, wrap_pyfunction}; +use sqlbuild_core::cli_text::models::CliStyle; +use sqlbuild_core::diagnostic_text::main::format_diagnostic_record::format_diagnostic_record; +use sqlbuild_core::diagnostic_text::models::DiagnosticRecordText; + +/// Each context field shown as a `label=value` token, in display order. +const CONTEXT_TOKENS: [(&str, &str); 7] = [ + ("sqlbuild_phase", "phase"), + ("sqlbuild_kind", "kind"), + ("sqlbuild_action_name", "action"), + ("sqlbuild_audit_name", "audit"), + ("sqlbuild_column_name", "column"), + ("sqlbuild_window", "window"), + ("sqlbuild_status", "status"), +]; + +/// `getattr(record, field, None)` when it is a `str`. +fn text_field(record: &Bound<'_, PyAny>, field: &str) -> PyResult> { + if !record.hasattr(field)? { + return Ok(None); + } + let value: Bound<'_, PyAny> = record.getattr(field)?; + if value.is_instance_of::() { + return Ok(Some(value.extract()?)); + } + Ok(None) +} + +/// The console line of `record` without its exception text. +#[pyfunction] +#[pyo3(signature = (record, *, use_color))] +fn format_diagnostics_console_record( + record: &Bound<'_, PyAny>, + use_color: bool, +) -> PyResult { + let mut tokens: Vec<(&'static str, String)> = Vec::new(); + for (field, label) in CONTEXT_TOKENS { + if let Some(value) = text_field(record, field)? { + tokens.push((label, value)); + } + } + let duration_ms: Option = if record.hasattr("sqlbuild_duration_ms")? { + let value: Bound<'_, PyAny> = record.getattr("sqlbuild_duration_ms")?; + value + .is_instance_of::() + .then(|| value.extract()) + .transpose()? + } else { + None + }; + let text = DiagnosticRecordText { + level_name: record.getattr("levelname")?.extract()?, + logger_name: record.getattr("name")?.extract()?, + message: record.call_method0("getMessage")?.extract()?, + subject: text_field(record, "sqlbuild_subject")?, + name: text_field(record, "sqlbuild_name")?, + event: text_field(record, "sqlbuild_event")?, + tokens, + duration_ms, + sql: text_field(record, "sqlbuild_sql")?, + }; + Ok(format_diagnostic_record(&text, CliStyle { use_color })) +} + +pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { + module.add_function(wrap_pyfunction!(format_diagnostics_console_record, module)?) +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/diagnostics/mod.rs b/crates/sqlbuild-python/src/bindings/_helpers/diagnostics/mod.rs index 14fc125d3d..2dd94ad479 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/diagnostics/mod.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/diagnostics/mod.rs @@ -1,2 +1,3 @@ +pub(crate) mod console_records; pub(crate) mod redaction; pub(crate) mod trackers; diff --git a/crates/sqlbuild-python/src/bindings/main/register.rs b/crates/sqlbuild-python/src/bindings/main/register.rs index aa81acaf02..8fce78a9ae 100644 --- a/crates/sqlbuild-python/src/bindings/main/register.rs +++ b/crates/sqlbuild-python/src/bindings/main/register.rs @@ -54,6 +54,7 @@ fn register_project_stages(module: &Bound<'_, PyModule>) -> PyResult<()> { cursor::registration::register(module)?; exported_constants::registration::register(module)?; state_rows::registration::register(module)?; + diagnostics::console_records::register(module)?; diagnostics::redaction::register(module)?; diagnostics::trackers::register(module)?; analysis::sql_test_comparisons::register(module)?; diff --git a/python-boundary.toml b/python-boundary.toml index e6a9229456..b01dd38623 100644 --- a/python-boundary.toml +++ b/python-boundary.toml @@ -59,11 +59,6 @@ path = "diagnostics/_helpers/logging.py" reason = "public-api" note = "Installs SQLBuild's handlers on Python's logging, which users and integrations configure." -[[stays]] -path = "diagnostics/_helpers/logging_formatters.py" -reason = "public-api" -note = "Python logging formatters for console and file diagnostics." - [[stays]] path = "diagnostics/_helpers/redaction.py" reason = "public-api" @@ -77,7 +72,7 @@ note = "Redaction entry points that Python logging handlers call per record." [[stays]] path = "diagnostics/classes/diagnostics_console_formatter.py" reason = "public-api" -note = "A logging.Formatter subclass for the console stream." +note = "Rule 4 glue: the logging.Formatter subclass for the console stream; the line text is native (format_diagnostics_console_record) and only stdlib exception formatting stays here." [[stays]] path = "diagnostics/classes/dynamic_stderr_handler.py" diff --git a/src/sqlbuild/_native.pyi b/src/sqlbuild/_native.pyi index db071fdcfc..4c5cdb35d9 100644 --- a/src/sqlbuild/_native.pyi +++ b/src/sqlbuild/_native.pyi @@ -1,5 +1,6 @@ """Private SQLBuild native engine bindings.""" +import logging import threading from collections import Counter from collections.abc import Callable, Iterable, Iterator, Mapping, Sequence, Set @@ -5022,6 +5023,7 @@ def normalize_sql_value( limits: SqlValueLimits | None = None, ) -> SqlValue: ... def render_state_sql_literal(*, value: object | None, declared_type: StateSqlValueType) -> str: ... +def format_diagnostics_console_record(record: logging.LogRecord, *, use_color: bool) -> str: ... def render_state_table_create_sql( *, qualified_name: str, diff --git a/src/sqlbuild/diagnostics/_helpers/logging_formatters.py b/src/sqlbuild/diagnostics/_helpers/logging_formatters.py deleted file mode 100644 index 5705251f17..0000000000 --- a/src/sqlbuild/diagnostics/_helpers/logging_formatters.py +++ /dev/null @@ -1,99 +0,0 @@ -"""Diagnostics logging formatters.""" - -from __future__ import annotations - -import logging - -from sqlbuild._native import CliStyle - -_LOGGER_ROOT_NAME: str = "sqlbuild" -_INLINE_SQL_TRANSACTION_STATEMENTS: tuple[str, ...] = ("BEGIN", "COMMIT", "ROLLBACK") -_SQL_EVENT_FIELD: str = "sqlbuild_sql" - - -def _short_logger_name(name: str) -> str: - prefix: str = f"{_LOGGER_ROOT_NAME}." - if name.startswith(prefix): - return name.removeprefix(prefix) - return name - - -def _format_console_header(*, record: logging.LogRecord, use_color: bool) -> str: - style: CliStyle = CliStyle(use_color=use_color) - level_tag: str = f"[{record.levelname.lower()}]" - context_message: str | None = _format_context_message(record=record, use_color=use_color) - if context_message is None: - scope: str = _short_logger_name(record.name) - colored_scope: str = style.object_name(scope) - context_message = f"{colored_scope} {_normalize_console_message(record.getMessage())}" - if not use_color: - return f"{level_tag} {context_message}" - return f"{style.log_label(level_tag)} {context_message}" - - -def _normalize_console_message(message: str) -> str: - return message.replace(" SQL", " sql") - - -def _render_sql_inline(sql: str) -> bool: - normalized: str = sql.strip().upper() - return normalized in _INLINE_SQL_TRANSACTION_STATEMENTS - - -def _format_context_message(*, record: logging.LogRecord, use_color: bool) -> str | None: - subject: object = getattr(record, "sqlbuild_subject", None) - if not isinstance(subject, str): - return None - parts: list[str] = [subject] - name: object = getattr(record, "sqlbuild_name", None) - if isinstance(name, str): - parts.append(_colorable_name(name=name, use_color=use_color)) - event: object = getattr(record, "sqlbuild_event", None) - if isinstance(event, str): - parts.append(event) - parts.extend(_format_context_tokens(record)) - message: str = _normalize_console_message(record.getMessage()) - if message: - parts.append(message) - return " ".join(parts) - - -def _format_context_tokens(record: logging.LogRecord) -> list[str]: - token_specs: tuple[tuple[str, str], ...] = ( - ("sqlbuild_phase", "phase"), - ("sqlbuild_kind", "kind"), - ("sqlbuild_action_name", "action"), - ("sqlbuild_audit_name", "audit"), - ("sqlbuild_column_name", "column"), - ("sqlbuild_window", "window"), - ("sqlbuild_status", "status"), - ) - rendered: list[str] = [] - field_name: str - label: str - for field_name, label in token_specs: - value: object = getattr(record, field_name, None) - if isinstance(value, str): - rendered.append(f"{label}={value}") - duration_ms: object = getattr(record, "sqlbuild_duration_ms", None) - if isinstance(duration_ms, int): - rendered.append(f"duration={duration_ms / 1000:.2f}s") - return rendered - - -def _colorable_name(*, name: str, use_color: bool) -> str: - return CliStyle(use_color=use_color).object_name(name) - - -def _get_record_sql(record: logging.LogRecord) -> str | None: - value: object = getattr(record, _SQL_EVENT_FIELD, None) - if isinstance(value, str): - return value - return None - - -def _append_exception(*, record: logging.LogRecord, rendered: str) -> str: - if record.exc_info is None: - return rendered - exception_text: str = logging.Formatter().formatException(record.exc_info) - return f"{rendered}\n{exception_text}" diff --git a/src/sqlbuild/diagnostics/classes/diagnostics_console_formatter.py b/src/sqlbuild/diagnostics/classes/diagnostics_console_formatter.py index 2c1a556162..741ebd95a3 100644 --- a/src/sqlbuild/diagnostics/classes/diagnostics_console_formatter.py +++ b/src/sqlbuild/diagnostics/classes/diagnostics_console_formatter.py @@ -4,15 +4,7 @@ import logging -from sqlbuild._native import CliStyle -from sqlbuild.diagnostics._helpers.logging_formatters import ( - _append_exception, - _format_console_header, - _get_record_sql, - _render_sql_inline, -) - -_SQL_SEPARATOR: str = "-" * 80 +from sqlbuild._native import format_diagnostics_console_record class DiagnosticsConsoleFormatter(logging.Formatter): @@ -23,15 +15,7 @@ def __init__(self, *, use_color: bool = False) -> None: self._use_color: bool = use_color def format(self, record: logging.LogRecord) -> str: - header: str = _format_console_header(record=record, use_color=self._use_color) - sql: str | None = _get_record_sql(record) - if sql is None: - return _append_exception(record=record, rendered=header) - if _render_sql_inline(sql): - return _append_exception(record=record, rendered=f"{header} {sql.strip()}") - style: CliStyle = CliStyle(use_color=self._use_color) - separator: str = style.muted(_SQL_SEPARATOR) - return _append_exception( - record=record, - rendered=f"{header}\n{separator}\n{sql.rstrip()}\n{separator}", - ) + rendered: str = format_diagnostics_console_record(record, use_color=self._use_color) + if record.exc_info is None: + return rendered + return f"{rendered}\n{logging.Formatter().formatException(record.exc_info)}" From d9eeab6a9a6ec216cc59509257dd6fe814d0dc2b Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Thu, 15 Oct 2026 18:44:56 +0100 Subject: [PATCH 28/48] refactor: stamp compile reuse and macro call store interpreter identity natively --- crates/sqlbuild-cli/src/replay/main/mod.rs | 1 + .../src/replay/main/output_file_identities.rs | 9 ++ .../bindings/_helpers/compile_command/mod.rs | 1 + .../_helpers/compile_command/registration.rs | 3 +- .../_helpers/compile_command/reuse_attempt.rs | 21 +--- .../compile_command/runtime_identity.rs | 113 ++++++++++++++++++ python-boundary.toml | 15 +-- src/sqlbuild/_native.pyi | 1 + .../_helpers/runtime_identity.py | 67 ----------- src/sqlbuild/cli/compile_reuse/constants.py | 6 - .../_helpers/store_environment.py | 5 +- .../commands/main/compile_outputs/helpers.py | 8 +- 12 files changed, 137 insertions(+), 113 deletions(-) create mode 100644 crates/sqlbuild-cli/src/replay/main/output_file_identities.rs create mode 100644 crates/sqlbuild-python/src/bindings/_helpers/compile_command/runtime_identity.rs delete mode 100644 src/sqlbuild/cli/compile_reuse/_helpers/runtime_identity.py delete mode 100644 src/sqlbuild/cli/compile_reuse/constants.py diff --git a/crates/sqlbuild-cli/src/replay/main/mod.rs b/crates/sqlbuild-cli/src/replay/main/mod.rs index f377c949dc..dc3b58e79e 100644 --- a/crates/sqlbuild-cli/src/replay/main/mod.rs +++ b/crates/sqlbuild-cli/src/replay/main/mod.rs @@ -1,2 +1,3 @@ +pub mod output_file_identities; pub mod serve_reused_compile; pub mod write_launch_record; diff --git a/crates/sqlbuild-cli/src/replay/main/output_file_identities.rs b/crates/sqlbuild-cli/src/replay/main/output_file_identities.rs new file mode 100644 index 0000000000..a607253b98 --- /dev/null +++ b/crates/sqlbuild-cli/src/replay/main/output_file_identities.rs @@ -0,0 +1,9 @@ +//! Identify the regular files standard output and error write to. + +use crate::replay::_helpers::console::redirected_output_files; + +/// Sorted, distinct device and inode pairs of stdout and stderr when they are regular files. +#[must_use] +pub fn output_file_identities() -> Vec<(u64, u64)> { + redirected_output_files() +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compile_command/mod.rs b/crates/sqlbuild-python/src/bindings/_helpers/compile_command/mod.rs index 73544bca28..c6f2c8f654 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compile_command/mod.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compile_command/mod.rs @@ -6,6 +6,7 @@ pub(crate) mod registration; pub(crate) mod report; pub(crate) mod reuse_attempt; pub(crate) mod reuse_recording; +pub(crate) mod runtime_identity; pub(crate) mod surrogates; pub(crate) mod target_writer; pub(crate) mod test_artifact_cache; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compile_command/registration.rs b/crates/sqlbuild-python/src/bindings/_helpers/compile_command/registration.rs index 8e6f6d42e9..add7a0460c 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compile_command/registration.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compile_command/registration.rs @@ -5,7 +5,7 @@ use pyo3::prelude::{Bound, PyModule, PyModuleMethods, PyResult}; use crate::bindings::_helpers::compile_command::models::CompileReuseAttempt; use crate::bindings::_helpers::compile_command::{ - prepared_artifacts, report, reuse_attempt, reuse_recording, target_writer, + prepared_artifacts, report, reuse_attempt, reuse_recording, runtime_identity, target_writer, }; pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { @@ -13,6 +13,7 @@ pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { report::register(module)?; reuse_attempt::register(module)?; reuse_recording::register(module)?; + runtime_identity::register(module)?; target_writer::register(module)?; prepared_artifacts::register(module) } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compile_command/reuse_attempt.rs b/crates/sqlbuild-python/src/bindings/_helpers/compile_command/reuse_attempt.rs index 028e51ef7b..dfaef4e9b9 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compile_command/reuse_attempt.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compile_command/reuse_attempt.rs @@ -27,6 +27,9 @@ use crate::bindings::_helpers::compile_command::constants::{ TOTAL_TIMING, TRACKED_ENVIRONMENT_PREFIXES, UNTRACKED_ENVIRONMENT_NAMES, }; use crate::bindings::_helpers::compile_command::models::CompileReuseAttempt; +use crate::bindings::_helpers::compile_command::runtime_identity::{ + invocation_identity, redirected_output_files, runtime_identity, +}; use crate::bindings::_helpers::compile_driver::phases::{elapsed_ms, monotonic}; use crate::bindings::_helpers::compile_driver::python_calls::{call_with, imported, keywords}; use crate::bindings::_helpers::compile_outputs::json_report::plain_json; @@ -35,7 +38,6 @@ use crate::bindings::_helpers::presentation_text::terminal::stdout_supports_colo const NATIVE_MODULE: &str = "sqlbuild._native"; const REUSE_REQUEST: (&str, &str) = ("sqlbuild.cli.compile_reuse.models", "CompileReuseRequest"); -const RUNTIME_IDENTITY_MODULE: &str = "sqlbuild.cli.compile_reuse._helpers.runtime_identity"; const STARTUP_INPUTS_MODULE: &str = "sqlbuild.cli.compile_reuse._helpers.startup_inputs"; const COMPUTE_LOG_TEE_MODULE: &str = "sqlbuild.runtime.compute_logs.classes.text_tee"; const HIT: &str = "hit"; @@ -68,8 +70,7 @@ pub(crate) fn reuse_rules(py: Python<'_>, with_output_files: bool) -> PyResult = - imported(py, RUNTIME_IDENTITY_MODULE, "redirected_output_files")?.call0()?; + let output_files: Bound<'_, PyList> = redirected_output_files(py)?; if with_output_files { rules.set_item("output_files", output_files)?; } @@ -270,19 +271,9 @@ fn reuse_check<'py>( check.set_item("selected_target", request.getattr("selected_target")?)?; check.set_item( "invocation", - call_with( - &imported(py, RUNTIME_IDENTITY_MODULE, "invocation_identity")?, - &[ - ("request", request.clone()), - ("project_dir", project_dir.clone()), - ("use_color", use_color.into_bound_py_any(py)?), - ], - )?, - )?; - check.set_item( - "runtime", - imported(py, RUNTIME_IDENTITY_MODULE, "runtime_identity")?.call0()?, + invocation_identity(request, project_dir, use_color)?, )?; + check.set_item("runtime", runtime_identity(py)?)?; check.set_item("search_path", listed(py.import("sys")?.getattr("path")?)?)?; check.set_item( "bypass_requested", diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compile_command/runtime_identity.rs b/crates/sqlbuild-python/src/bindings/_helpers/compile_command/runtime_identity.rs new file mode 100644 index 0000000000..45de138c84 --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/compile_command/runtime_identity.rs @@ -0,0 +1,113 @@ +//! Interpreter, native build and invocation facts that decide whether a stored compile is reusable. + +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyModule, PyModuleMethods, PyResult, Python}; +use pyo3::types::{PyDict, PyDictMethods, PyList, PyTuple}; +use pyo3::{IntoPyObjectExt, pyfunction, wrap_pyfunction}; +use sqlbuild_cli::replay::main::output_file_identities::output_file_identities; + +use crate::bindings::_helpers::compile_driver::python_calls::keywords; + +const NATIVE_MODULE: &str = "sqlbuild._native"; +/// The version of the runtime identity a stored compile records. +const REUSE_FORMAT_VERSION: &str = "1"; +/// Each `CompileReuseRequest` field the invocation identity records unchanged. +const REQUEST_CHOICES: [&str; 9] = [ + "no_sql_validation", + "defer_to", + "selected_target", + "json_output", + "manifest", + "dag_path", + "no_color", + "lineage_mode", + "cli_vars", +]; + +/// `sys.version`, `sys.executable`, `sys.platform` and `str(sys.implementation.cache_tag)`. +pub(crate) fn interpreter_values(py: Python<'_>) -> PyResult<[String; 4]> { + let sys: Bound<'_, PyModule> = py.import("sys")?; + Ok([ + sys.getattr("version")?.extract()?, + sys.getattr("executable")?.extract()?, + sys.getattr("platform")?.extract()?, + sys.getattr("implementation")? + .getattr("cache_tag")? + .str()? + .extract()?, + ]) +} + +/// The interpreter facts every interpreter-sensitive cache key stamps, in one order. +#[pyfunction] +fn interpreter_identity(py: Python<'_>) -> PyResult> { + PyTuple::new(py, interpreter_values(py)?) +} + +/// The interpreter and native build identity that produced a compile. +pub(crate) fn runtime_identity(py: Python<'_>) -> PyResult> { + let [version, executable, platform, cache_tag] = interpreter_values(py)?; + let identity: Bound<'_, PyDict> = PyDict::new(py); + identity.set_item("format", REUSE_FORMAT_VERSION)?; + identity.set_item("python", version)?; + identity.set_item("executable", executable)?; + identity.set_item("platform", platform)?; + identity.set_item("cache_tag", cache_tag)?; + identity.set_item( + "native_build", + py.import(NATIVE_MODULE)?.getattr("BUILD_IDENTITY")?.str()?, + )?; + Ok(identity) +} + +fn listed<'py>(value: Bound<'py, PyAny>) -> PyResult> { + Ok(PyList::new(value.py(), value.try_iter()?.collect::>>()?)?.into_any()) +} + +/// Every command-line choice and location that can change compile output, as JSON bytes. +pub(crate) fn invocation_identity<'py>( + request: &Bound<'py, PyAny>, + project_dir: &Bound<'py, PyAny>, + use_color: bool, +) -> PyResult> { + let py: Python<'py> = request.py(); + let payload: Bound<'py, PyDict> = PyDict::new(py); + payload.set_item("project_dir", project_dir)?; + let requested: Bound<'py, PyAny> = request.getattr("project_dir")?; + let requested: Bound<'py, PyAny> = if requested.is_none() { + requested + } else { + requested.str()?.into_any() + }; + payload.set_item("requested_project_dir", requested)?; + for field in REQUEST_CHOICES { + payload.set_item(field, request.getattr(field)?)?; + } + payload.set_item("cwd", py.import("os")?.getattr("getcwd")?.call0()?)?; + payload.set_item("use_color", use_color)?; + payload.set_item("select", listed(request.getattr("select")?)?)?; + payload.set_item("exclude", listed(request.getattr("exclude")?)?)?; + payload.set_item("sys_path", listed(py.import("sys")?.getattr("path")?)?)?; + let options: Bound<'py, PyDict> = keywords( + py, + &[ + ("sort_keys", true.into_bound_py_any(py)?), + ("separators", (",", ":").into_bound_py_any(py)?), + ("default", py.import("builtins")?.getattr("repr")?), + ], + )?; + let text: Bound<'py, PyAny> = py + .import("json")? + .getattr("dumps")? + .call((payload,), Some(&options))?; + text.call_method1("encode", ("utf-8", "surrogateescape")) +} + +/// Device and inode pairs of stdout and stderr files, which hold output rather than inputs. +pub(crate) fn redirected_output_files(py: Python<'_>) -> PyResult> { + PyList::new(py, output_file_identities()) +} + +pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { + module.add_function(wrap_pyfunction!(interpreter_identity, module)?) +} + diff --git a/python-boundary.toml b/python-boundary.toml index b01dd38623..67a120c464 100644 --- a/python-boundary.toml +++ b/python-boundary.toml @@ -419,11 +419,6 @@ path = "cli/compile_reuse/_helpers/provider_settings.py" reason = "user-code" note = "Inspects user provider settings classes for the environment and files they read." -[[stays]] -path = "cli/compile_reuse/_helpers/runtime_identity.py" -reason = "user-code" -note = "Interpreter and invocation facts of the Python host that user code runs in." - [[stays]] path = "cli/compile_reuse/_helpers/startup_inputs.py" reason = "user-code" @@ -452,7 +447,7 @@ note = "Identity hashing over live user functions (source and globals)." [[stays]] path = "compiler/macro_bridge/_helpers/store_environment.py" reason = "user-code" -note = "Fingerprints the Python interpreter and installed packages that project macros import from; the macro call store cache key." +note = "Fingerprints the installed packages, import path and live module files that project macros import from; the macro call store cache key. The interpreter stamp it shares with compile reuse is native (interpreter_identity)." [[stays]] path = "rule_engine/classes/runtime_guard.py" @@ -662,13 +657,12 @@ note = "Shape rule: data records only (StatementExecutionTelemetry, CostResource [[stays]] path = "cli/compile_reuse/models.py" reason = "user-code" -note = "Shape rule: data records only (CompileReuseRequest, SettingsEnvironmentInputs, SettingsInputsResult), no logic; consumed by cli/compile_reuse/_helpers/provider_settings.py, cli/compile_reuse/_helpers/runtime_identity.py." +note = "Shape rule: data records only (CompileReuseRequest, SettingsEnvironmentInputs, SettingsInputsResult), no logic; consumed by cli/compile_reuse/_helpers/provider_settings.py, and read natively by the compile reuse check." [[stays]] path = "compiler/source_freshness/constants.py" reason = "adapter" note = "Shape rule: source freshness column names and adapter column types, no logic; SOURCE_FRESHNESS_TABLE_NAME is re-exported from sqlbuild._native." -note = "Shape rule: data constants only (SOURCE_FRESHNESS_TABLE_NAME, COLUMN_SOURCE_NAME, COLUMN_TARGET_DATABASE, COLUMN_TARGET_SCHEMA, ...), no logic; consumed by adapter/contract/classes/duckdb_backed_adapter.py, adapters/bigquery/classes/bigquery_adapter.py, adapters/databricks/classes/databricks_adapter.py and 3 more, and read natively by the source freshness state bindings." [[stays]] path = "compiler/auditing/types.py" @@ -765,11 +759,6 @@ path = "cli/entry/constants.py" reason = "public-api" note = "Shape rule: data constants only (NATIVE_ALLOCATOR_PURGE_DELAY_VARIABLE, NATIVE_ALLOCATOR_PURGE_DELAY_MILLISECONDS, OUTPUT_STREAM_ENCODING, OUTPUT_STREAM_ERRORS, ...), no logic; consumed by cli/entry/_helpers/streams.py, cli/entry/main/entry.py, cli/entry/main/launched.py." -[[stays]] -path = "cli/compile_reuse/constants.py" -reason = "user-code" -note = "Shape rule: data constants only (REUSE_FORMAT_VERSION, OUTPUT_FILE_DESCRIPTORS), no logic; consumed by cli/compile_reuse/_helpers/runtime_identity.py." - [[stays]] path = "sql_values/exceptions.py" reason = "adapter" diff --git a/src/sqlbuild/_native.pyi b/src/sqlbuild/_native.pyi index 4c5cdb35d9..777f965a47 100644 --- a/src/sqlbuild/_native.pyi +++ b/src/sqlbuild/_native.pyi @@ -5024,6 +5024,7 @@ def normalize_sql_value( ) -> SqlValue: ... def render_state_sql_literal(*, value: object | None, declared_type: StateSqlValueType) -> str: ... def format_diagnostics_console_record(record: logging.LogRecord, *, use_color: bool) -> str: ... +def interpreter_identity() -> tuple[str, str, str, str]: ... def render_state_table_create_sql( *, qualified_name: str, diff --git a/src/sqlbuild/cli/compile_reuse/_helpers/runtime_identity.py b/src/sqlbuild/cli/compile_reuse/_helpers/runtime_identity.py deleted file mode 100644 index a75a54a0d4..0000000000 --- a/src/sqlbuild/cli/compile_reuse/_helpers/runtime_identity.py +++ /dev/null @@ -1,67 +0,0 @@ -"""Interpreter and invocation facts the Python host supplies to native compile reuse.""" - -from __future__ import annotations - -import json -import os -import stat -import sys - -import sqlbuild._native as native -from sqlbuild.cli.compile_reuse.constants import OUTPUT_FILE_DESCRIPTORS, REUSE_FORMAT_VERSION -from sqlbuild.cli.compile_reuse.models import CompileReuseRequest - - -def runtime_identity() -> dict[str, str]: - """Return the interpreter and native build identity that produced a compile.""" - - return { - "format": str(REUSE_FORMAT_VERSION), - "python": sys.version, - "executable": sys.executable, - "platform": sys.platform, - "cache_tag": str(sys.implementation.cache_tag), - "native_build": str(native.BUILD_IDENTITY), - } - - -def invocation_identity( - *, request: CompileReuseRequest, project_dir: str, use_color: bool -) -> bytes: - """Encode every command-line choice and location that can change compile output.""" - - payload: dict[str, object] = { - "project_dir": project_dir, - "requested_project_dir": None if request.project_dir is None else str(request.project_dir), - "cwd": os.getcwd(), - "no_sql_validation": request.no_sql_validation, - "defer_to": request.defer_to, - "selected_target": request.selected_target, - "json_output": request.json_output, - "manifest": request.manifest, - "dag_path": request.dag_path, - "no_color": request.no_color, - "use_color": use_color, - "lineage_mode": request.lineage_mode, - "select": list(request.select), - "exclude": list(request.exclude), - "cli_vars": request.cli_vars, - "sys_path": list(sys.path), - } - return json.dumps(payload, sort_keys=True, separators=(",", ":"), default=repr).encode( - "utf-8", "surrogateescape" - ) - - -def redirected_output_files() -> list[tuple[int, int]]: - """Identify stdout and stderr files, which hold this command's output, not its inputs.""" - - identities: set[tuple[int, int]] = set() - for descriptor in OUTPUT_FILE_DESCRIPTORS: - try: - status: os.stat_result = os.fstat(descriptor) - except OSError: - continue - if stat.S_ISREG(status.st_mode): - identities.add((status.st_dev, status.st_ino)) - return sorted(identities) diff --git a/src/sqlbuild/cli/compile_reuse/constants.py b/src/sqlbuild/cli/compile_reuse/constants.py deleted file mode 100644 index c115ceb35f..0000000000 --- a/src/sqlbuild/cli/compile_reuse/constants.py +++ /dev/null @@ -1,6 +0,0 @@ -"""Compile reuse runtime identity constants.""" - -from __future__ import annotations - -REUSE_FORMAT_VERSION: int = 1 -OUTPUT_FILE_DESCRIPTORS: tuple[int, ...] = (1, 2) diff --git a/src/sqlbuild/compiler/macro_bridge/_helpers/store_environment.py b/src/sqlbuild/compiler/macro_bridge/_helpers/store_environment.py index 4b53025cbe..6d6ad99c4e 100644 --- a/src/sqlbuild/compiler/macro_bridge/_helpers/store_environment.py +++ b/src/sqlbuild/compiler/macro_bridge/_helpers/store_environment.py @@ -40,10 +40,7 @@ def store_environment(*, project_dir: Path, fingerprint: str) -> str: [ MACRO_CALL_STORE_ENVIRONMENT_VERSION, _native.BUILD_IDENTITY, - sys.version, - sys.executable, - sys.platform, - str(sys.implementation.cache_tag), + *_native.interpreter_identity(), json.dumps(_search_path_stamps(project_dir=project_dir)), json.dumps(_tracked_environment()), json.dumps(_installed_distribution_records()), diff --git a/tests/integration/src/sqlbuild/cli/commands/main/compile_outputs/helpers.py b/tests/integration/src/sqlbuild/cli/commands/main/compile_outputs/helpers.py index 9515c5c232..f8e6902028 100644 --- a/tests/integration/src/sqlbuild/cli/commands/main/compile_outputs/helpers.py +++ b/tests/integration/src/sqlbuild/cli/commands/main/compile_outputs/helpers.py @@ -19,8 +19,6 @@ ) from sqlbuild._native import run_cli as main from sqlbuild.cli.compile_reuse._helpers import provider_settings as provider_settings_module -from sqlbuild.cli.compile_reuse._helpers import runtime_identity as runtime_identity_module -from sqlbuild.cli.compile_reuse._helpers.runtime_identity import runtime_identity STAGING_PREFIX: str = ".sqlbuild-staging-" NATIVE_REUSE_DIRECTORY_NAME: str = "project-reuse-v2" @@ -206,11 +204,7 @@ def change_runtime(project_dir: Path, monkeypatch: pytest.MonkeyPatch) -> None: """Report a different interpreter from now on.""" del project_dir - - def changed() -> dict[str, str]: - return {**runtime_identity(), "python": "3.0.0 (simulated)"} - - monkeypatch.setattr(runtime_identity_module, "runtime_identity", changed) + monkeypatch.setattr(sys, "version", "3.0.0 (main, Jan 1 2026, 00:00:00) [simulated]") def change_native_build(project_dir: Path, monkeypatch: pytest.MonkeyPatch) -> None: From c5e46dd9451fdda8de52ed25acddf915e9c22cac Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Thu, 15 Oct 2026 20:14:13 +0100 Subject: [PATCH 29/48] refactor: project microbatch coverage and render microbatch state SQL natively --- .../compile_command/runtime_identity.rs | 1 - .../_helpers/microbatches/conversion.rs | 90 +++++ .../src/bindings/_helpers/microbatches/mod.rs | 4 + .../_helpers/microbatches/projection.rs | 249 ++++++++++++ .../_helpers/microbatches/registration.rs | 10 + .../_helpers/microbatches/statements.rs | 295 +++++++++++++++ .../src/bindings/_helpers/mod.rs | 1 + .../src/bindings/main/register.rs | 3 +- .../src/events/main/microbatch_event_id.rs | 25 ++ crates/sqlbuild-state/src/events/main/mod.rs | 1 + crates/sqlbuild-state/src/lib.rs | 1 + .../src/microbatches/_helpers/coverage.rs | 146 +++++++ .../src/microbatches/_helpers/mod.rs | 2 + .../src/microbatches/_helpers/ordering.rs | 47 +++ .../microbatches/main/create_index_sqls.rs | 15 + .../main/existing_event_ids_sql.rs | 10 + .../src/microbatches/main/insert_many_sql.rs | 24 ++ .../microbatches/main/latest_requirement.rs | 29 ++ .../src/microbatches/main/mod.rs | 7 + .../src/microbatches/main/project_coverage.rs | 73 ++++ .../src/microbatches/main/project_replay.rs | 71 ++++ .../src/microbatches/main/read_scope_sql.rs | 19 + crates/sqlbuild-state/src/microbatches/mod.rs | 6 + .../sqlbuild-state/src/microbatches/models.rs | 121 ++++++ .../sqlbuild-state/src/microbatches/tests.rs | 8 + .../src/microbatches/tests/helpers.rs | 69 ++++ .../src/microbatches/tests/test_projection.rs | 133 +++++++ .../src/microbatches/tests/test_statements.rs | 47 +++ .../src/microbatches/tests/test_types.rs | 33 ++ fensu.toml | 7 +- python-boundary.toml | 2 +- src/sqlbuild/_native.pyi | 57 ++- .../adapter/contract/classes/base_adapter.py | 4 +- .../adapter/contract/classes/microbatch.py | 4 +- .../adapters/duckdb/classes/duckdb_adapter.py | 4 +- .../postgres/classes/postgres_adapter.py | 4 +- .../sqlserver/classes/sqlserver_adapter.py | 4 +- .../_helpers/materializations/microbatch.py | 12 +- .../microbatches/_helpers/projection.py | 358 ------------------ src/sqlbuild/microbatches/_helpers/sql.py | 170 --------- .../microbatches/classes/direct_store.py | 19 +- src/sqlbuild/microbatches/main/__init__.py | 0 .../microbatches/main/create_index_sqls.py | 33 -- .../microbatches/main/create_table_sql.py | 25 -- .../main/deterministic_event_id.py | 36 -- .../main/latest_replay_requirement.py | 16 - .../microbatches/main/project_coverage.py | 25 -- .../microbatches/main/project_replay.py | 30 -- .../sqlbuild/microbatches/classes/helpers.py | 4 +- .../classes/test_retired_columns.py | 8 +- .../sqlbuild/microbatches/test_event_codec.py | 2 +- .../sqlbuild/microbatches/test_projection.py | 3 +- .../src/sqlbuild/microbatches/test_sql.py | 82 ++-- 53 files changed, 1681 insertions(+), 768 deletions(-) create mode 100644 crates/sqlbuild-python/src/bindings/_helpers/microbatches/conversion.rs create mode 100644 crates/sqlbuild-python/src/bindings/_helpers/microbatches/mod.rs create mode 100644 crates/sqlbuild-python/src/bindings/_helpers/microbatches/projection.rs create mode 100644 crates/sqlbuild-python/src/bindings/_helpers/microbatches/registration.rs create mode 100644 crates/sqlbuild-python/src/bindings/_helpers/microbatches/statements.rs create mode 100644 crates/sqlbuild-state/src/events/main/microbatch_event_id.rs create mode 100644 crates/sqlbuild-state/src/microbatches/_helpers/coverage.rs create mode 100644 crates/sqlbuild-state/src/microbatches/_helpers/mod.rs create mode 100644 crates/sqlbuild-state/src/microbatches/_helpers/ordering.rs create mode 100644 crates/sqlbuild-state/src/microbatches/main/create_index_sqls.rs create mode 100644 crates/sqlbuild-state/src/microbatches/main/existing_event_ids_sql.rs create mode 100644 crates/sqlbuild-state/src/microbatches/main/insert_many_sql.rs create mode 100644 crates/sqlbuild-state/src/microbatches/main/latest_requirement.rs create mode 100644 crates/sqlbuild-state/src/microbatches/main/mod.rs create mode 100644 crates/sqlbuild-state/src/microbatches/main/project_coverage.rs create mode 100644 crates/sqlbuild-state/src/microbatches/main/project_replay.rs create mode 100644 crates/sqlbuild-state/src/microbatches/main/read_scope_sql.rs create mode 100644 crates/sqlbuild-state/src/microbatches/mod.rs create mode 100644 crates/sqlbuild-state/src/microbatches/models.rs create mode 100644 crates/sqlbuild-state/src/microbatches/tests.rs create mode 100644 crates/sqlbuild-state/src/microbatches/tests/helpers.rs create mode 100644 crates/sqlbuild-state/src/microbatches/tests/test_projection.rs create mode 100644 crates/sqlbuild-state/src/microbatches/tests/test_statements.rs create mode 100644 crates/sqlbuild-state/src/microbatches/tests/test_types.rs delete mode 100644 src/sqlbuild/microbatches/_helpers/projection.py delete mode 100644 src/sqlbuild/microbatches/_helpers/sql.py delete mode 100644 src/sqlbuild/microbatches/main/__init__.py delete mode 100644 src/sqlbuild/microbatches/main/create_index_sqls.py delete mode 100644 src/sqlbuild/microbatches/main/create_table_sql.py delete mode 100644 src/sqlbuild/microbatches/main/deterministic_event_id.py delete mode 100644 src/sqlbuild/microbatches/main/latest_replay_requirement.py delete mode 100644 src/sqlbuild/microbatches/main/project_coverage.py delete mode 100644 src/sqlbuild/microbatches/main/project_replay.py diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compile_command/runtime_identity.rs b/crates/sqlbuild-python/src/bindings/_helpers/compile_command/runtime_identity.rs index 45de138c84..9e4f5df9cf 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compile_command/runtime_identity.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compile_command/runtime_identity.rs @@ -110,4 +110,3 @@ pub(crate) fn redirected_output_files(py: Python<'_>) -> PyResult) -> PyResult<()> { module.add_function(wrap_pyfunction!(interpreter_identity, module)?) } - diff --git a/crates/sqlbuild-python/src/bindings/_helpers/microbatches/conversion.rs b/crates/sqlbuild-python/src/bindings/_helpers/microbatches/conversion.rs new file mode 100644 index 0000000000..dca974875b --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/microbatches/conversion.rs @@ -0,0 +1,90 @@ +//! Microbatch events, scopes and intervals read from their Python shapes. + +use std::collections::HashMap; + +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyErr, PyResult, Python}; +use pyo3::types::PyString; +use sqlbuild_cursor::algebra::main::cursor_sort_key::cursor_sort_key; +use sqlbuild_cursor::algebra::models::{Scalar, SortKey}; +use sqlbuild_state::microbatches::models::{BoundInterval, CursorBound}; + +use crate::bindings::_helpers::compile_driver::python_calls::imported; +use crate::bindings::_helpers::cursor::conversion::CursorSession; +use crate::bindings::_helpers::cursor::scalars::as_scalar; + +pub(crate) const MODELS_MODULE: &str = "sqlbuild.microbatches.models"; +pub(crate) const TYPES_MODULE: &str = "sqlbuild.microbatches.types"; +pub(crate) const CONSTANTS_MODULE: &str = "sqlbuild.microbatches.constants"; +const EXCEPTIONS_MODULE: &str = "sqlbuild.microbatches.exceptions"; + +/// `MicrobatchStateError(message)`. +pub(crate) fn state_error(py: Python<'_>, message: &str) -> PyErr { + match imported(py, EXCEPTIONS_MODULE, "MicrobatchStateError") + .and_then(|class| class.call1((message,))) + { + Ok(error) => PyErr::from_value(error), + Err(error) => error, + } +} + +/// `str(value)` of a string enum member or string. +pub(crate) fn enum_text(value: &Bound<'_, PyAny>) -> PyResult { + value.str()?.extract() +} + +/// `None`, or the string an optional field holds. +pub(crate) fn optional_text(value: &Bound<'_, PyAny>) -> PyResult> { + if value.is_none() { + return Ok(None); + } + Ok(Some(value.extract()?)) +} + +/// `value.timestamp()` of a `datetime`. +pub(crate) fn timestamp(value: &Bound<'_, PyAny>) -> PyResult { + value.call_method0("timestamp")?.extract() +} + +/// Raw cursor bounds of one cursor type, parsed once per distinct text. +pub(crate) struct BoundParser<'py> { + session: CursorSession<'py>, + cursor_type: Bound<'py, PyAny>, + keys: HashMap, +} + +impl<'py> BoundParser<'py> { + pub(crate) fn new(cursor_type: &Bound<'py, PyAny>) -> PyResult { + Ok(Self { + session: CursorSession::new(cursor_type.py())?, + cursor_type: cursor_type.clone(), + keys: HashMap::new(), + }) + } + + /// One raw bound and its sort key. + pub(crate) fn bound(&mut self, raw: &Bound<'py, PyAny>) -> PyResult { + let text: String = raw.extract()?; + if let Some(key) = self.keys.get(&text) { + return Ok(CursorBound { text, key: *key }); + } + let py: Python<'py> = raw.py(); + let scalar: Scalar = as_scalar( + &self.session, + &PyString::new(py, &text).into_any(), + &self.cursor_type, + )?; + let key: SortKey = self + .session + .checked(cursor_sort_key(&scalar, &self.session))?; + self.keys.insert(text.clone(), key); + Ok(CursorBound { text, key }) + } + + /// The bounds of an object with `start` and `end` attributes. + pub(crate) fn interval(&mut self, interval: &Bound<'py, PyAny>) -> PyResult { + Ok(BoundInterval { + start: self.bound(&interval.getattr("start")?)?, + end: self.bound(&interval.getattr("end")?)?, + }) + } +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/microbatches/mod.rs b/crates/sqlbuild-python/src/bindings/_helpers/microbatches/mod.rs new file mode 100644 index 0000000000..e1b436ef3d --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/microbatches/mod.rs @@ -0,0 +1,4 @@ +pub(crate) mod conversion; +pub(crate) mod projection; +pub(crate) mod registration; +pub(crate) mod statements; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/microbatches/projection.rs b/crates/sqlbuild-python/src/bindings/_helpers/microbatches/projection.rs new file mode 100644 index 0000000000..10c5c32f7e --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/microbatches/projection.rs @@ -0,0 +1,249 @@ +//! Coverage, replay-requirement and latest-requirement projections over append-only events. + +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyModule, PyModuleMethods, PyResult, Python}; +use pyo3::types::{PyDict, PyString, PyTuple}; +use pyo3::{IntoPyObjectExt, pyfunction, wrap_pyfunction}; +use sqlbuild_state::microbatches::main::latest_requirement::latest_requirement; +use sqlbuild_state::microbatches::main::project_coverage::project_coverage; +use sqlbuild_state::microbatches::main::project_replay::project_replay; +use sqlbuild_state::microbatches::models::{ + BoundInterval, CompletionFact, CoveragePiece, CoverageProjection, ReplayProjection, + ReplayState, ReplayVersions, RequirementFact, TextInterval, +}; + +use crate::bindings::_helpers::boundary::keyword_request::Request; +use crate::bindings::_helpers::compile_driver::python_calls::{call_with, imported}; +use crate::bindings::_helpers::microbatches::conversion::{ + BoundParser, MODELS_MODULE, TYPES_MODULE, enum_text, optional_text, timestamp, +}; + +const COMPLETION_RECORD_TYPES: [&str; 2] = ["partition_completion", "synthetic_completion"]; +const ORDINARY_COMPLETION: &str = "partition_completion"; +const REPLAY_REQUIREMENT: &str = "replay_requirement"; +const UNKNOWN_FINGERPRINT: &str = "unknown"; + +fn python_intervals<'py>( + py: Python<'py>, + intervals: &[TextInterval], +) -> PyResult> { + let class: Bound<'py, PyAny> = imported(py, MODELS_MODULE, "MicrobatchInterval")?; + let built: Vec> = intervals + .iter() + .map(|interval| { + call_with( + &class, + &[ + ("start", PyString::new(py, &interval.start).into_any()), + ("end", PyString::new(py, &interval.end).into_any()), + ], + ) + }) + .collect::>()?; + PyTuple::new(py, built) +} + +fn python_coverage<'py>( + py: Python<'py>, + projection: &CoverageProjection, + events: &[Bound<'py, PyAny>], +) -> PyResult> { + let projected_class: Bound<'py, PyAny> = + imported(py, MODELS_MODULE, "ProjectedMicrobatchInterval")?; + let mut projected: Vec> = Vec::with_capacity(projection.segments.len()); + for segment in &projection.segments { + let event: &Bound<'py, PyAny> = &events[segment.completion]; + projected.push(call_with( + &projected_class, + &[ + ( + "start", + PyString::new(py, &segment.interval.start).into_any(), + ), + ("end", PyString::new(py, &segment.interval.end).into_any()), + ("fingerprint_status", event.getattr("fingerprint_status")?), + ("model_version_hash", event.getattr("model_version_hash")?), + ("record_type", event.getattr("record_type")?), + ("completion_type", event.getattr("completion_type")?), + ("event_id", event.getattr("event_id")?), + ], + )?); + } + call_with( + &imported(py, MODELS_MODULE, "MicrobatchCoverageProjection")?, + &[ + ("intervals", PyTuple::new(py, projected)?.into_any()), + ( + "contiguous_frontier", + projection + .contiguous_frontier + .as_deref() + .into_bound_py_any(py)?, + ), + ( + "known_missing", + python_intervals(py, &projection.known_missing)?.into_any(), + ), + ( + "unaccounted", + python_intervals(py, &projection.unaccounted)?.into_any(), + ), + ( + "unknown_fingerprints", + python_intervals(py, &projection.unknown_fingerprints)?.into_any(), + ), + ], + ) +} + +/// Fold completion facts and classify uncovered expected intervals. +#[pyfunction(name = "project_microbatch_coverage")] +#[pyo3(signature = (*, events, expected_intervals, cursor_type))] +fn coverage<'py>( + events: &Bound<'py, PyAny>, + expected_intervals: &Bound<'py, PyAny>, + cursor_type: &Bound<'py, PyAny>, +) -> PyResult> { + let py: Python<'py> = events.py(); + let mut parser: BoundParser<'py> = BoundParser::new(cursor_type)?; + let mut completion_events: Vec> = Vec::new(); + let mut completions: Vec = Vec::new(); + for event in events.try_iter()? { + let event: Bound<'py, PyAny> = event?; + let record_type: String = enum_text(&event.getattr("record_type")?)?; + let (start, end) = ( + event.getattr("partition_start")?, + event.getattr("partition_end")?, + ); + if !COMPLETION_RECORD_TYPES.contains(&record_type.as_str()) + || start.is_none() + || end.is_none() + { + continue; + } + completions.push(CompletionFact { + interval: BoundInterval { + start: parser.bound(&start)?, + end: parser.bound(&end)?, + }, + ordinary: record_type == ORDINARY_COMPLETION, + unknown_fingerprint: enum_text(&event.getattr("fingerprint_status")?)? + == UNKNOWN_FINGERPRINT, + created: timestamp(&event.getattr("created_at")?)?, + event_id: event.getattr("event_id")?.extract()?, + }); + completion_events.push(event); + } + let expected: Vec = expected_intervals + .try_iter()? + .map(|interval| parser.interval(&interval?)) + .collect::>()?; + python_coverage( + py, + &project_coverage(&completions, &expected), + &completion_events, + ) +} + +fn replay_state_member(state: ReplayState) -> &'static str { + match state { + ReplayState::Incomplete => "INCOMPLETE", + ReplayState::VerifiedComplete => "VERIFIED_COMPLETE", + ReplayState::CompleteWithUnknownFingerprints => "COMPLETE_WITH_UNKNOWN_FINGERPRINTS", + ReplayState::Superseded => "SUPERSEDED", + } +} + +/// Derive replay completion without writing mutable requirement status. +#[pyfunction(name = "project_replay_requirement")] +#[pyo3(signature = (**request))] +fn replay<'py>(request: Option<&Bound<'py, PyDict>>) -> PyResult> { + let request = Request::new("project_replay_requirement", request)?; + let py: Python<'py> = request.py(); + let requirement: Bound<'py, PyAny> = request.get("requirement")?; + let current: String = request.get("current_model_version_hash")?.extract()?; + let required: Option = + optional_text(&requirement.getattr("required_model_version_hash")?)?; + let versions = ReplayVersions { + required: required.as_deref(), + current: ¤t, + }; + let expected_objects: Vec> = request + .get("expected_intervals")? + .try_iter()? + .collect::>()?; + let mut expected: Vec = Vec::new(); + let mut pieces: Vec = Vec::new(); + if required.as_deref() == Some(current.as_str()) { + let mut parser: BoundParser<'py> = BoundParser::new(&request.get("cursor_type")?)?; + for interval in &expected_objects { + expected.push(parser.interval(interval)?); + } + for piece in request.get("coverage")?.getattr("intervals")?.try_iter()? { + let piece: Bound<'py, PyAny> = piece?; + pieces.push(CoveragePiece { + interval: parser.interval(&piece)?, + unknown_fingerprint: enum_text(&piece.getattr("fingerprint_status")?)? + == UNKNOWN_FINGERPRINT, + model_version_hash: optional_text(&piece.getattr("model_version_hash")?)?, + }); + } + } + let projection: ReplayProjection = project_replay(versions, &expected, &pieces); + let selected = |indexes: &[usize]| { + PyTuple::new( + py, + indexes.iter().map(|index| expected_objects[*index].clone()), + ) + }; + call_with( + &imported(py, MODELS_MODULE, "ReplayRequirementProjection")?, + &[ + ("requirement", requirement), + ( + "state", + imported(py, TYPES_MODULE, "ReplayRequirementState")? + .getattr(replay_state_member(projection.state))?, + ), + ("missing", selected(&projection.missing)?.into_any()), + ( + "unknown_fingerprints", + selected(&projection.unknown_fingerprints)?.into_any(), + ), + ], + ) +} + +/// Return only the latest requirement created for the currently compiled version. +#[pyfunction(name = "latest_active_replay_requirement")] +#[pyo3(signature = (*, events, current_model_version_hash))] +fn latest<'py>( + events: &Bound<'py, PyAny>, + current_model_version_hash: &str, +) -> PyResult>> { + let objects: Vec> = events.try_iter()?.collect::>()?; + let mut facts: Vec = Vec::with_capacity(objects.len()); + for event in &objects { + let is_requirement: bool = enum_text(&event.getattr("record_type")?)? == REPLAY_REQUIREMENT; + let required: Option = + optional_text(&event.getattr("required_model_version_hash")?)?; + let matching: bool = + is_requirement && required.as_deref() == Some(current_model_version_hash); + facts.push(RequirementFact { + is_requirement, + required_model_version_hash: required, + created: if matching { + timestamp(&event.getattr("created_at")?)? + } else { + 0.0 + }, + event_id: event.getattr("event_id")?.extract()?, + }); + } + Ok(latest_requirement(&facts, current_model_version_hash).map(|index| objects[index].clone())) +} + +pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { + module.add_function(wrap_pyfunction!(coverage, module)?)?; + module.add_function(wrap_pyfunction!(replay, module)?)?; + module.add_function(wrap_pyfunction!(latest, module)?) +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/microbatches/registration.rs b/crates/sqlbuild-python/src/bindings/_helpers/microbatches/registration.rs new file mode 100644 index 0000000000..b114a939df --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/microbatches/registration.rs @@ -0,0 +1,10 @@ +//! Register the native microbatch history projections and statements. + +use pyo3::prelude::{Bound, PyModule, PyResult}; + +use crate::bindings::_helpers::microbatches::{projection, statements}; + +pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { + projection::register(module)?; + statements::register(module) +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/microbatches/statements.rs b/crates/sqlbuild-python/src/bindings/_helpers/microbatches/statements.rs new file mode 100644 index 0000000000..abec6c6f0c --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/microbatches/statements.rs @@ -0,0 +1,295 @@ +//! Direct-mode microbatch state DDL, idempotent inserts, reads and event IDs. + +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyModule, PyModuleMethods, PyResult, Python}; +use pyo3::types::{PyDict, PyTuple}; +use pyo3::{pyfunction, wrap_pyfunction}; +use sqlbuild_state::events::main::microbatch_event_id::microbatch_event_id; +use sqlbuild_state::microbatches::main::create_index_sqls::create_index_sqls; +use sqlbuild_state::microbatches::main::existing_event_ids_sql::existing_event_ids_sql; +use sqlbuild_state::microbatches::main::insert_many_sql::insert_many_sql; +use sqlbuild_state::microbatches::main::read_scope_sql::read_scope_sql; +use sqlbuild_state::microbatches::models::{MicrobatchEventIdentity, ScopeRead}; +use sqlbuild_state::state_tables::main::create_table_sql::create_table_sql; +use sqlbuild_state::state_tables::models::TableColumn; + +use crate::bindings::_helpers::boundary::keyword_request::Request; +use crate::bindings::_helpers::compile_driver::python_calls::{call_with, imported}; +use crate::bindings::_helpers::contracts::state_literals::render_state_sql_literal; +use crate::bindings::_helpers::frontier::encoder::emit_error; +use crate::bindings::_helpers::microbatches::conversion::{ + CONSTANTS_MODULE, optional_text, state_error, +}; + +const CODEC_MODULE: &str = "sqlbuild.microbatches.classes.event_codec"; +const ADAPTER_TYPES: &str = "sqlbuild.adapter.contract.types"; +const MISSING_SCHEMA: &str = "microbatch history requires a target schema"; +const EMPTY_INSERT: &str = "bulk microbatch event insert requires at least one event"; +const EMPTY_LOOKUP: &str = "microbatch event lookup requires at least one event"; +const INTEGER_COLUMN_TYPE: &str = "BIGINT"; +const EVENT_ID_COLUMN: &str = "event_id"; + +/// The state table columns and each column's declared state value type. +struct Layout { + columns: Vec, + declared: Vec, +} + +impl Layout { + fn new(py: Python<'_>) -> PyResult { + let constants: Bound<'_, PyModule> = py.import(CONSTANTS_MODULE)?; + let columns: Vec = constants.getattr("MICROBATCH_COLUMNS")?.extract()?; + let types: Bound<'_, PyAny> = constants.getattr("MICROBATCH_COLUMN_TYPES")?; + let declared: Vec = columns + .iter() + .map(|column| types.get_item(column)?.str()?.extract()) + .collect::>()?; + Ok(Self { columns, declared }) + } + + fn declared(&self, column: &str) -> &str { + self.columns + .iter() + .position(|known| known == column) + .map_or("", |index| self.declared[index].as_str()) + } + + /// One event's column literals in column order. + fn literals(&self, event: &Bound<'_, PyAny>) -> PyResult> { + let py: Python<'_> = event.py(); + let values: Bound<'_, PyAny> = + imported(py, CODEC_MODULE, "MicrobatchEventCodec")?.call_method1("values", (event,))?; + values + .try_iter()? + .zip(&self.declared) + .map(|(value, declared)| render_state_sql_literal(py, &value?, declared)) + .collect() + } + + fn literal(&self, column: &str, value: &Bound<'_, PyAny>) -> PyResult { + render_state_sql_literal(value.py(), value, self.declared(column)) + } +} + +/// The adapter's qualified name of the state table, failing without a schema. +fn qualified_table<'py>( + render_qualified_name: &Bound<'py, PyAny>, + (database, schema): (&Bound<'py, PyAny>, &Bound<'py, PyAny>), +) -> PyResult> { + let py: Python<'py> = render_qualified_name.py(); + let table_name: Bound<'py, PyAny> = imported(py, CONSTANTS_MODULE, "MICROBATCH_TABLE_NAME")?; + optional_text(&call_with( + render_qualified_name, + &[ + ("database", database.clone()), + ("schema", schema.clone()), + ("name", table_name), + ], + )?) +} + +/// The state table of one scope: its target database and required schema. +fn scope_table( + render_qualified_name: &Bound<'_, PyAny>, + scope: &Bound<'_, PyAny>, +) -> PyResult { + let py: Python<'_> = scope.py(); + let schema: Bound<'_, PyAny> = scope.getattr("target_schema")?; + if schema.is_none() { + return Err(state_error(py, MISSING_SCHEMA)); + } + qualified_table( + render_qualified_name, + (&scope.getattr("target_database")?, &schema), + )? + .ok_or_else(|| state_error(py, MISSING_SCHEMA)) +} + +/// The events as a list, and the table of the first event's scope; empty fails with `empty`. +fn events_table<'py>( + events: &Bound<'py, PyAny>, + render_qualified_name: &Bound<'py, PyAny>, + empty: &str, +) -> PyResult<(Vec>, String)> { + let listed: Vec> = events.try_iter()?.collect::>()?; + let Some(first) = listed.first() else { + return Err(state_error(events.py(), empty)); + }; + let table: String = scope_table(render_qualified_name, &first.getattr("scope")?)?; + Ok((listed, table)) +} + +/// Build one guarded set-based insert for events in the same state table. +#[pyfunction(name = "build_microbatch_insert_many_sql")] +#[pyo3(signature = (*, events, render_qualified_name))] +fn insert_many( + events: &Bound<'_, PyAny>, + render_qualified_name: &Bound<'_, PyAny>, +) -> PyResult { + let (listed, table) = events_table(events, render_qualified_name, EMPTY_INSERT)?; + let layout: Layout = Layout::new(events.py())?; + let rows: Vec> = listed + .iter() + .map(|event| layout.literals(event)) + .collect::>()?; + Ok(insert_many_sql(&table, &layout.columns, &rows)) +} + +/// Build one lookup for event IDs already present before a bulk insert. +#[pyfunction(name = "build_microbatch_existing_event_ids_sql")] +#[pyo3(signature = (*, events, render_qualified_name))] +fn existing_ids( + events: &Bound<'_, PyAny>, + render_qualified_name: &Bound<'_, PyAny>, +) -> PyResult { + let (listed, table) = events_table(events, render_qualified_name, EMPTY_LOOKUP)?; + let layout: Layout = Layout::new(events.py())?; + let literals: Vec = listed + .iter() + .map(|event| layout.literal(EVENT_ID_COLUMN, &event.getattr(EVENT_ID_COLUMN)?)) + .collect::>()?; + Ok(existing_event_ids_sql(&table, &literals)) +} + +/// Build the ordered read of one scope's events, every generation for the wildcard. +#[pyfunction(name = "build_microbatch_read_scope_sql")] +#[pyo3(signature = (*, scope, render_qualified_name))] +fn read_scope( + scope: &Bound<'_, PyAny>, + render_qualified_name: &Bound<'_, PyAny>, +) -> PyResult { + let py: Python<'_> = scope.py(); + let table: String = scope_table(render_qualified_name, scope)?; + let layout: Layout = Layout::new(py)?; + let wildcard: String = + imported(py, CONSTANTS_MODULE, "MICROBATCH_GENERATION_WILDCARD")?.extract()?; + let generation: Bound<'_, PyAny> = scope.getattr("physical_generation_id")?; + let generation: Option = if generation.extract::()? == wildcard { + None + } else { + Some(layout.literal("physical_generation_id", &generation)?) + }; + let scope_kind: String = layout.literal("scope_kind", &scope.getattr("scope_kind")?)?; + let scope_key: String = layout.literal("scope_key", &scope.getattr("scope_key")?)?; + Ok(read_scope_sql( + &table, + &ScopeRead { + columns: &layout.columns, + scope_kind: &scope_kind, + scope_key: &scope_key, + generation: generation.as_deref(), + }, + )) +} + +/// Build direct microbatch-state DDL with the adapter's text and timestamp types. +#[pyfunction(name = "build_microbatch_create_table_sql")] +#[pyo3(signature = (**request))] +fn create_table(request: Option<&Bound<'_, PyDict>>) -> PyResult { + let request = Request::new("build_microbatch_create_table_sql", request)?; + let py: Python<'_> = request.py(); + let table: String = qualified_table( + &request.get("render_qualified_name")?, + (&request.get("database")?, &request.get("schema")?), + )? + .ok_or_else(|| state_error(py, MISSING_SCHEMA))?; + let render_framework_type: Bound<'_, PyAny> = request.get("render_framework_type")?; + let framework: Bound<'_, PyAny> = imported(py, ADAPTER_TYPES, "FrameworkType")?; + let text_type: String = render_framework_type + .call1((framework.getattr("STRING")?,))? + .extract()?; + let timestamp_type: String = render_framework_type + .call1((framework.getattr("TIMESTAMP")?,))? + .extract()?; + let layout: Layout = Layout::new(py)?; + let columns: Vec = layout + .columns + .iter() + .zip(&layout.declared) + .map(|(name, declared)| TableColumn { + name: name.clone(), + rendered_type: match declared.as_str() { + "timestamp" => timestamp_type.clone(), + "integer" => INTEGER_COLUMN_TYPE.to_owned(), + _ => text_type.clone(), + }, + required: false, + }) + .collect(); + Ok(create_table_sql(&table, &columns)) +} + +/// Build non-unique scope and provenance indexes for indexed warehouses. +#[pyfunction(name = "build_microbatch_create_index_sqls")] +#[pyo3(signature = (**request))] +fn create_indexes<'py>(request: Option<&Bound<'py, PyDict>>) -> PyResult> { + let request = Request::new("build_microbatch_create_index_sqls", request)?; + let py: Python<'_> = request.py(); + let schema: Bound<'_, PyAny> = request.get("schema")?; + let rendered: Bound<'_, PyAny> = call_with( + &request.get("render_qualified_name")?, + &[ + ("database", request.get("database")?), + ("schema", schema.clone()), + ( + "name", + imported(py, CONSTANTS_MODULE, "MICROBATCH_TABLE_NAME")?, + ), + ], + )?; + let table: String = if rendered.is_truthy()? { + rendered.extract()? + } else { + let name: String = imported(py, CONSTANTS_MODULE, "MICROBATCH_TABLE_NAME")?.extract()?; + format!("{}.{name}", schema.str()?) + }; + let render_identifier: Bound<'_, PyAny> = request.get("render_identifier")?; + let identifier = |name: &Bound<'_, PyAny>| -> PyResult { + render_identifier.call1((name,))?.extract() + }; + let mut indexes: Vec<(String, Vec)> = Vec::new(); + let definitions: Bound<'_, PyAny> = + imported(py, CONSTANTS_MODULE, "MICROBATCH_DIRECT_INDEXES")?; + for item in definitions.call_method0("items")?.try_iter()? { + let (name, columns): (Bound<'_, PyAny>, Bound<'_, PyAny>) = item?.extract()?; + let columns: Vec = columns + .try_iter()? + .map(|column| identifier(&column?)) + .collect::>()?; + indexes.push((identifier(&name)?, columns)); + } + PyTuple::new(py, create_index_sqls(&table, &indexes)) +} + +/// Return the content-addressed ID for one logical event. +#[pyfunction(name = "deterministic_microbatch_event_id")] +#[pyo3(signature = (**request))] +fn deterministic_event_id(request: Option<&Bound<'_, PyDict>>) -> PyResult { + let request = Request::new("deterministic_microbatch_event_id", request)?; + let scope: Bound<'_, PyAny> = request.get("scope")?; + let field = |name: &str| -> PyResult { scope.getattr(name)?.extract() }; + let optional = + |name: &str| -> PyResult> { optional_text(&scope.getattr(name)?) }; + let identity = MicrobatchEventIdentity { + scope_kind: field("scope_kind")?, + scope_key: field("scope_key")?, + model_name: field("model_name")?, + target_database: optional("target_database")?, + target_schema: optional("target_schema")?, + target_name: field("target_name")?, + physical_generation_id: field("physical_generation_id")?, + record_type: request.get("record_type")?.getattr("value")?.extract()?, + partition_start: optional_text(&request.get("partition_start")?)?, + partition_end: optional_text(&request.get("partition_end")?)?, + completion_reason: request.get("completion_reason")?.extract()?, + }; + microbatch_event_id(&identity).map_err(emit_error) +} + +pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { + module.add_function(wrap_pyfunction!(insert_many, module)?)?; + module.add_function(wrap_pyfunction!(existing_ids, module)?)?; + module.add_function(wrap_pyfunction!(read_scope, module)?)?; + module.add_function(wrap_pyfunction!(create_table, module)?)?; + module.add_function(wrap_pyfunction!(create_indexes, module)?)?; + module.add_function(wrap_pyfunction!(deterministic_event_id, module)?) +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/mod.rs b/crates/sqlbuild-python/src/bindings/_helpers/mod.rs index 7163558ad6..3ba2d852ea 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/mod.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/mod.rs @@ -53,6 +53,7 @@ pub(crate) mod lineage; pub(crate) mod load; pub(crate) mod macro_bridge; pub(crate) mod manifest; +pub(crate) mod microbatches; pub(crate) mod model_config; pub(crate) mod native_threads; pub(crate) mod output_capture; diff --git a/crates/sqlbuild-python/src/bindings/main/register.rs b/crates/sqlbuild-python/src/bindings/main/register.rs index 8fce78a9ae..dbd73bd745 100644 --- a/crates/sqlbuild-python/src/bindings/main/register.rs +++ b/crates/sqlbuild-python/src/bindings/main/register.rs @@ -7,7 +7,7 @@ use crate::bindings::_helpers::{ analysis, analysis_session, attachments, auditing, authored_values, boundary, cache, cli, compile_attachment, compile_command, compile_driver, compile_outputs, compiled_project, contracts, cursor, dag_artifact, diagnostics, discovery, exported_constants, fingerprints, - frontier, graph, lineage, macro_bridge, manifest, model_config, plan_output, + frontier, graph, lineage, macro_bridge, manifest, microbatches, model_config, plan_output, planner_registration, project_assembly, records, refactor, references, render, rules, scopes, semantic_checks, semantic_shapes, sql_test_glue, sqltext, state_rows, type_system, }; @@ -54,6 +54,7 @@ fn register_project_stages(module: &Bound<'_, PyModule>) -> PyResult<()> { cursor::registration::register(module)?; exported_constants::registration::register(module)?; state_rows::registration::register(module)?; + microbatches::registration::register(module)?; diagnostics::console_records::register(module)?; diagnostics::redaction::register(module)?; diagnostics::trackers::register(module)?; diff --git a/crates/sqlbuild-state/src/events/main/microbatch_event_id.rs b/crates/sqlbuild-state/src/events/main/microbatch_event_id.rs new file mode 100644 index 0000000000..bd71669b91 --- /dev/null +++ b/crates/sqlbuild-state/src/events/main/microbatch_event_id.rs @@ -0,0 +1,25 @@ +//! `microbatch_event_id`: the content-addressed ID of one logical microbatch event. + +use sqlbuild_core::json::errors::JsonEmitError; +use sqlbuild_core::json::models::JsonValue; + +use crate::events::_helpers::identity::{json_digest, optional_text, text}; +use crate::microbatches::models::MicrobatchEventIdentity; + +/// SHA-256 of the compact, ASCII-escaped JSON array of the identity fields. +pub fn microbatch_event_id(identity: &MicrobatchEventIdentity) -> Result { + let value = JsonValue::Array(vec![ + text(&identity.scope_kind), + text(&identity.scope_key), + text(&identity.model_name), + optional_text(identity.target_database.as_deref()), + optional_text(identity.target_schema.as_deref()), + text(&identity.target_name), + text(&identity.physical_generation_id), + text(&identity.record_type), + optional_text(identity.partition_start.as_deref()), + optional_text(identity.partition_end.as_deref()), + text(&identity.completion_reason), + ]); + json_digest(&value, false) +} diff --git a/crates/sqlbuild-state/src/events/main/mod.rs b/crates/sqlbuild-state/src/events/main/mod.rs index 9b543bf5fb..459f576db9 100644 --- a/crates/sqlbuild-state/src/events/main/mod.rs +++ b/crates/sqlbuild-state/src/events/main/mod.rs @@ -7,6 +7,7 @@ pub mod event_exists_sql; pub mod event_insert_sql; pub mod event_read_sql; pub mod grants_json; +pub mod microbatch_event_id; pub mod migration_event_id; pub mod newest_column_event; pub mod newest_fingerprint; diff --git a/crates/sqlbuild-state/src/lib.rs b/crates/sqlbuild-state/src/lib.rs index 22bec10215..571e27e44c 100644 --- a/crates/sqlbuild-state/src/lib.rs +++ b/crates/sqlbuild-state/src/lib.rs @@ -5,4 +5,5 @@ pub mod events; pub mod execution_history; pub mod freshness; +pub mod microbatches; pub mod state_tables; diff --git a/crates/sqlbuild-state/src/microbatches/_helpers/coverage.rs b/crates/sqlbuild-state/src/microbatches/_helpers/coverage.rs new file mode 100644 index 0000000000..82e057fa81 --- /dev/null +++ b/crates/sqlbuild-state/src/microbatches/_helpers/coverage.rs @@ -0,0 +1,146 @@ +//! The boundary sweep and gap classification behind microbatch coverage projection. + +use std::cmp::Ordering; +use std::collections::{BinaryHeap, HashSet}; + +use crate::microbatches::_helpers::ordering::{key_cmp, lt, lte, sorted_by_bound}; +use crate::microbatches::models::{BoundInterval, CompletionFact, CursorBound, TextInterval}; + +/// One swept segment between adjacent boundaries and its latest active completion. +pub(crate) struct Segment { + pub(crate) start: CursorBound, + pub(crate) end: CursorBound, + pub(crate) completion: usize, +} + +fn text_interval(start: &CursorBound, end: &CursorBound) -> TextInterval { + TextInterval { + start: start.text.clone(), + end: end.text.clone(), + } +} + +/// Every distinct bound text, sorted by key. +pub(crate) fn ordered_boundaries( + completions: &[CompletionFact], + expected: &[BoundInterval], +) -> Vec { + let mut boundaries: Vec = Vec::new(); + let mut seen: HashSet<&str> = HashSet::new(); + let intervals = completions + .iter() + .map(|completion| &completion.interval) + .chain(expected); + for interval in intervals { + for bound in [&interval.start, &interval.end] { + if seen.insert(bound.text.as_str()) { + boundaries.push(bound.clone()); + } + } + } + boundaries.sort_by(|left, right| { + key_cmp(&left.key, &right.key).then_with(|| left.text.cmp(&right.text)) + }); + boundaries +} + +/// Sweep adjacent boundaries, keeping the most recently created active completion. +pub(crate) fn latest_completion_segments( + completions: &[CompletionFact], + boundaries: &[CursorBound], +) -> Vec { + let mut latest_order: Vec = (0..completions.len()).collect(); + latest_order.sort_by(|left, right| { + let (left, right) = (&completions[*left], &completions[*right]); + left.created + .partial_cmp(&right.created) + .unwrap_or(Ordering::Equal) + .then_with(|| left.event_id.cmp(&right.event_id)) + }); + let mut rank: Vec = vec![0; completions.len()]; + for (position, index) in latest_order.into_iter().enumerate() { + rank[index] = position; + } + let mut by_start: Vec = (0..completions.len()).collect(); + by_start.sort_by(|left, right| { + key_cmp( + &completions[*left].interval.start.key, + &completions[*right].interval.start.key, + ) + }); + let mut active: BinaryHeap<(usize, usize)> = BinaryHeap::new(); + let mut next: usize = 0; + let mut segments: Vec = Vec::new(); + for pair in boundaries.windows(2) { + let (start, end) = (&pair[0], &pair[1]); + while next < by_start.len() && lte(&completions[by_start[next]].interval.start, start) { + active.push((rank[by_start[next]], by_start[next])); + next += 1; + } + while let Some((_, top)) = active.peek() { + if lte(end, &completions[*top].interval.end) { + break; + } + active.pop(); + } + if let Some((_, top)) = active.peek() { + segments.push(Segment { + start: start.clone(), + end: end.clone(), + completion: *top, + }); + } + } + segments +} + +/// The uncovered parts of each expected interval, split into known-missing and unaccounted. +pub(crate) fn classify_gaps( + expected: &[BoundInterval], + accounted: &[BoundInterval], + latest_ordinary_end: Option<&CursorBound>, +) -> (Vec, Vec) { + let ordered_expected: Vec = sorted_by_bound(expected, |item| &item.start); + let ordered_accounted: Vec = sorted_by_bound(accounted, |item| &item.start); + let mut known_missing: Vec = Vec::new(); + let mut unaccounted: Vec = Vec::new(); + let mut accounted_index: usize = 0; + for expected in &ordered_expected { + while accounted_index < ordered_accounted.len() + && lte(&ordered_accounted[accounted_index].end, &expected.start) + { + accounted_index += 1; + } + let mut frontier: &CursorBound = &expected.start; + let mut gaps: Vec<(&CursorBound, &CursorBound)> = Vec::new(); + for interval in &ordered_accounted[accounted_index..] { + if !lt(&interval.start, &expected.end) { + break; + } + if lt(frontier, &interval.start) { + gaps.push((frontier, &interval.start)); + } + if lt(frontier, &interval.end) { + frontier = &interval.end; + } + if lte(&expected.end, frontier) { + break; + } + } + if lt(frontier, &expected.end) { + gaps.push((frontier, &expected.end)); + } + for (start, end) in gaps { + let accounted_before: bool = accounted_index > 0 || lt(&expected.start, start); + let known: bool = + accounted_before && latest_ordinary_end.is_some_and(|latest| lte(end, latest)); + let gap: TextInterval = text_interval(start, end); + if known { + known_missing.push(gap); + } else { + unaccounted.push(gap); + } + } + } + (known_missing, unaccounted) +} diff --git a/crates/sqlbuild-state/src/microbatches/_helpers/mod.rs b/crates/sqlbuild-state/src/microbatches/_helpers/mod.rs new file mode 100644 index 0000000000..262e03e5b4 --- /dev/null +++ b/crates/sqlbuild-state/src/microbatches/_helpers/mod.rs @@ -0,0 +1,2 @@ +pub(crate) mod coverage; +pub(crate) mod ordering; diff --git a/crates/sqlbuild-state/src/microbatches/_helpers/ordering.rs b/crates/sqlbuild-state/src/microbatches/_helpers/ordering.rs new file mode 100644 index 0000000000..1bc1a17c6e --- /dev/null +++ b/crates/sqlbuild-state/src/microbatches/_helpers/ordering.rs @@ -0,0 +1,47 @@ +//! Cursor bound comparisons and stable sorts by bound key. + +use std::cmp::Ordering; + +use sqlbuild_cursor::algebra::models::SortKey; + +use crate::microbatches::models::CursorBound; + +/// How two keys of one cursor type order; integers and moments never meet in one history. +pub(crate) fn key_cmp(left: &SortKey, right: &SortKey) -> Ordering { + match (left, right) { + (SortKey::Integer(left), SortKey::Integer(right)) => left.cmp(right), + (SortKey::Moment(left), SortKey::Moment(right)) => left.cmp(right), + (SortKey::Integer(_), SortKey::Moment(_)) => Ordering::Less, + (SortKey::Moment(_), SortKey::Integer(_)) => Ordering::Greater, + } +} + +pub(crate) fn lt(left: &CursorBound, right: &CursorBound) -> bool { + key_cmp(&left.key, &right.key) == Ordering::Less +} + +pub(crate) fn lte(left: &CursorBound, right: &CursorBound) -> bool { + key_cmp(&left.key, &right.key) != Ordering::Greater +} + +/// `items` stably sorted by the key `bound` selects. +pub(crate) fn sorted_by_bound(items: &[T], bound: impl Fn(&T) -> &CursorBound) -> Vec { + let mut sorted: Vec = items.to_vec(); + sorted.sort_by(|left, right| key_cmp(&bound(left).key, &bound(right).key)); + sorted +} + +/// The first non-empty bound holding the smallest (`wanted = Less`) or largest key. +pub(crate) fn extreme<'a>( + bounds: impl Iterator, + wanted: Ordering, +) -> Option<&'a CursorBound> { + let mut best: Option<&CursorBound> = None; + for bound in bounds.filter(|bound| !bound.text.is_empty()) { + match best { + Some(current) if key_cmp(&bound.key, ¤t.key) != wanted => {} + _ => best = Some(bound), + } + } + best +} diff --git a/crates/sqlbuild-state/src/microbatches/main/create_index_sqls.rs b/crates/sqlbuild-state/src/microbatches/main/create_index_sqls.rs new file mode 100644 index 0000000000..dc96267040 --- /dev/null +++ b/crates/sqlbuild-state/src/microbatches/main/create_index_sqls.rs @@ -0,0 +1,15 @@ +//! `create_index_sqls`: the non-unique scope and provenance indexes of the state table. + +/// One `CREATE INDEX IF NOT EXISTS` per rendered index name and its rendered columns. +#[must_use] +pub fn create_index_sqls(table: &str, indexes: &[(String, Vec)]) -> Vec { + indexes + .iter() + .map(|(name, columns)| { + format!( + "CREATE INDEX IF NOT EXISTS {name} ON {table} ({})", + columns.join(", ") + ) + }) + .collect() +} diff --git a/crates/sqlbuild-state/src/microbatches/main/existing_event_ids_sql.rs b/crates/sqlbuild-state/src/microbatches/main/existing_event_ids_sql.rs new file mode 100644 index 0000000000..3306463c69 --- /dev/null +++ b/crates/sqlbuild-state/src/microbatches/main/existing_event_ids_sql.rs @@ -0,0 +1,10 @@ +//! `existing_event_ids_sql`: the lookup of event IDs already stored before a bulk insert. + +/// `SELECT event_id FROM
WHERE event_id IN ()`. +#[must_use] +pub fn existing_event_ids_sql(table: &str, event_id_literals: &[String]) -> String { + format!( + "SELECT event_id FROM {table} WHERE event_id IN ({})", + event_id_literals.join(", ") + ) +} diff --git a/crates/sqlbuild-state/src/microbatches/main/insert_many_sql.rs b/crates/sqlbuild-state/src/microbatches/main/insert_many_sql.rs new file mode 100644 index 0000000000..044afc896c --- /dev/null +++ b/crates/sqlbuild-state/src/microbatches/main/insert_many_sql.rs @@ -0,0 +1,24 @@ +//! `insert_many_sql`: one guarded set-based insert of events into one state table. + +/// Rows selected under the first row's column aliases, inserted where the event ID is new. +#[must_use] +pub fn insert_many_sql(table: &str, columns: &[String], rows: &[Vec]) -> String { + let mut selections: Vec = Vec::with_capacity(rows.len()); + for (index, row) in rows.iter().enumerate() { + if index > 0 { + selections.push(format!("SELECT {}", row.join(", "))); + continue; + } + let mut aliased: Vec = Vec::with_capacity(row.len()); + for (literal, column) in row.iter().zip(columns) { + aliased.push(format!("{literal} AS {column}")); + } + selections.push(format!("SELECT {}", aliased.join(", "))); + } + let columns: String = columns.join(", "); + format!( + "INSERT INTO {table} ({columns}) SELECT {columns} FROM ({}) AS incoming \ +WHERE NOT EXISTS (SELECT 1 FROM {table} AS existing WHERE existing.event_id = incoming.event_id)", + selections.join(" UNION ALL ") + ) +} diff --git a/crates/sqlbuild-state/src/microbatches/main/latest_requirement.rs b/crates/sqlbuild-state/src/microbatches/main/latest_requirement.rs new file mode 100644 index 0000000000..f64d75c6a2 --- /dev/null +++ b/crates/sqlbuild-state/src/microbatches/main/latest_requirement.rs @@ -0,0 +1,29 @@ +//! `latest_requirement`: the newest replay requirement created for the compiled version. + +use std::cmp::Ordering; + +use crate::microbatches::models::RequirementFact; + +/// The index of the first event ordering last by creation time then event ID, if any. +#[must_use] +pub fn latest_requirement(events: &[RequirementFact], current: &str) -> Option { + let mut best: Option = None; + for (index, event) in events.iter().enumerate() { + if !event.is_requirement || event.required_model_version_hash.as_deref() != Some(current) { + continue; + } + let newer: bool = best.is_none_or(|best| { + let held: &RequirementFact = &events[best]; + event + .created + .partial_cmp(&held.created) + .unwrap_or(Ordering::Equal) + .then_with(|| event.event_id.cmp(&held.event_id)) + == Ordering::Greater + }); + if newer { + best = Some(index); + } + } + best +} diff --git a/crates/sqlbuild-state/src/microbatches/main/mod.rs b/crates/sqlbuild-state/src/microbatches/main/mod.rs new file mode 100644 index 0000000000..bfe1dfee3e --- /dev/null +++ b/crates/sqlbuild-state/src/microbatches/main/mod.rs @@ -0,0 +1,7 @@ +pub mod create_index_sqls; +pub mod existing_event_ids_sql; +pub mod insert_many_sql; +pub mod latest_requirement; +pub mod project_coverage; +pub mod project_replay; +pub mod read_scope_sql; diff --git a/crates/sqlbuild-state/src/microbatches/main/project_coverage.rs b/crates/sqlbuild-state/src/microbatches/main/project_coverage.rs new file mode 100644 index 0000000000..5c33f44b8b --- /dev/null +++ b/crates/sqlbuild-state/src/microbatches/main/project_coverage.rs @@ -0,0 +1,73 @@ +//! `project_coverage`: fold completion facts and classify uncovered expected intervals. + +use std::cmp::Ordering; + +use crate::microbatches::_helpers::coverage::{ + Segment, classify_gaps, latest_completion_segments, ordered_boundaries, +}; +use crate::microbatches::_helpers::ordering::{extreme, lt, lte, sorted_by_bound}; +use crate::microbatches::models::{ + BoundInterval, CompletionFact, CoverageProjection, CoverageSegment, CursorBound, TextInterval, +}; + +/// Latest-completion segments, the contiguous frontier and the known-missing and unaccounted gaps. +#[must_use] +pub fn project_coverage( + completions: &[CompletionFact], + expected: &[BoundInterval], +) -> CoverageProjection { + let boundaries: Vec = ordered_boundaries(completions, expected); + let segments: Vec = latest_completion_segments(completions, &boundaries); + let accounted: Vec = segments + .iter() + .map(|segment| BoundInterval { + start: segment.start.clone(), + end: segment.end.clone(), + }) + .collect(); + let mut frontier: Option<&CursorBound> = extreme( + expected.iter().map(|interval| &interval.start), + Ordering::Less, + ); + let ordered_accounted: Vec = sorted_by_bound(&accounted, |item| &item.start); + if let Some(mut current) = frontier.filter(|_| !accounted.is_empty()) { + for interval in &ordered_accounted { + if !lte(&interval.start, current) { + break; + } + if lt(current, &interval.end) { + current = &interval.end; + } + } + frontier = Some(current); + } + let latest_ordinary_end: Option<&CursorBound> = extreme( + completions + .iter() + .filter(|completion| completion.ordinary) + .map(|completion| &completion.interval.end), + Ordering::Greater, + ); + let (known_missing, unaccounted) = classify_gaps(expected, &accounted, latest_ordinary_end); + let text = |segment: &Segment| TextInterval { + start: segment.start.text.clone(), + end: segment.end.text.clone(), + }; + CoverageProjection { + unknown_fingerprints: segments + .iter() + .filter(|segment| completions[segment.completion].unknown_fingerprint) + .map(text) + .collect(), + contiguous_frontier: frontier.map(|bound| bound.text.clone()), + known_missing, + unaccounted, + segments: segments + .iter() + .map(|segment| CoverageSegment { + interval: text(segment), + completion: segment.completion, + }) + .collect(), + } +} diff --git a/crates/sqlbuild-state/src/microbatches/main/project_replay.rs b/crates/sqlbuild-state/src/microbatches/main/project_replay.rs new file mode 100644 index 0000000000..8f8dc559ca --- /dev/null +++ b/crates/sqlbuild-state/src/microbatches/main/project_replay.rs @@ -0,0 +1,71 @@ +//! `project_replay`: derive a replay requirement's completion without mutable status. + +use crate::microbatches::_helpers::ordering::{lt, lte, sorted_by_bound}; +use crate::microbatches::models::{ + BoundInterval, CoveragePiece, CursorBound, ReplayProjection, ReplayState, ReplayVersions, +}; + +fn fully_covered(expected: &BoundInterval, pieces: &[&CoveragePiece]) -> bool { + let mut frontier: &CursorBound = &expected.start; + for piece in sorted_by_bound(pieces, |piece| &piece.interval.start) { + if lte(&piece.interval.end, frontier) { + continue; + } + if lt(frontier, &piece.interval.start) { + return false; + } + frontier = &piece.interval.end; + if lte(&expected.end, frontier) { + return true; + } + } + false +} + +/// Superseded for another version; otherwise which expected intervals are missing or unverified. +#[must_use] +pub fn project_replay( + versions: ReplayVersions<'_>, + expected: &[BoundInterval], + coverage: &[CoveragePiece], +) -> ReplayProjection { + if versions.required != Some(versions.current) { + return ReplayProjection { + state: ReplayState::Superseded, + missing: Vec::new(), + unknown_fingerprints: Vec::new(), + }; + } + let mut missing: Vec = Vec::new(); + let mut unknown: Vec = Vec::new(); + for (index, interval) in expected.iter().enumerate() { + let pieces: Vec<&CoveragePiece> = coverage + .iter() + .filter(|piece| { + lt(&piece.interval.start, &interval.end) && lt(&interval.start, &piece.interval.end) + }) + .collect(); + if !fully_covered(interval, &pieces) { + missing.push(index); + } else if pieces.iter().any(|piece| piece.unknown_fingerprint) { + unknown.push(index); + } else if pieces + .iter() + .any(|piece| piece.model_version_hash.as_deref() != Some(versions.current)) + { + missing.push(index); + } + } + let state: ReplayState = if !missing.is_empty() { + ReplayState::Incomplete + } else if !unknown.is_empty() { + ReplayState::CompleteWithUnknownFingerprints + } else { + ReplayState::VerifiedComplete + }; + ReplayProjection { + state, + missing, + unknown_fingerprints: unknown, + } +} diff --git a/crates/sqlbuild-state/src/microbatches/main/read_scope_sql.rs b/crates/sqlbuild-state/src/microbatches/main/read_scope_sql.rs new file mode 100644 index 0000000000..2b27e75d96 --- /dev/null +++ b/crates/sqlbuild-state/src/microbatches/main/read_scope_sql.rs @@ -0,0 +1,19 @@ +//! `read_scope_sql`: the ordered read of one scope's stored events. + +use crate::microbatches::models::ScopeRead; + +/// Every column of one scope's events, optionally one generation, by creation then event ID. +#[must_use] +pub fn read_scope_sql(table: &str, read: &ScopeRead<'_>) -> String { + let generation: String = read + .generation + .map(|literal| format!("AND physical_generation_id = {literal} ")) + .unwrap_or_default(); + format!( + "SELECT {} FROM {table} WHERE scope_kind = {} AND scope_key = {} {generation}\ +ORDER BY created_at, event_id", + read.columns.join(", "), + read.scope_kind, + read.scope_key + ) +} diff --git a/crates/sqlbuild-state/src/microbatches/mod.rs b/crates/sqlbuild-state/src/microbatches/mod.rs new file mode 100644 index 0000000000..e6e5a52eb4 --- /dev/null +++ b/crates/sqlbuild-state/src/microbatches/mod.rs @@ -0,0 +1,6 @@ +pub(crate) mod _helpers; +pub mod main; +pub mod models; + +#[cfg(test)] +mod tests; diff --git a/crates/sqlbuild-state/src/microbatches/models.rs b/crates/sqlbuild-state/src/microbatches/models.rs new file mode 100644 index 0000000000..dad4315763 --- /dev/null +++ b/crates/sqlbuild-state/src/microbatches/models.rs @@ -0,0 +1,121 @@ +//! Append-only microbatch facts, cursor bounds and the coverage derived from them. + +use sqlbuild_cursor::algebra::models::SortKey; + +/// One raw cursor bound and the key it sorts and compares by. +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct CursorBound { + pub text: String, + pub key: SortKey, +} + +/// A half-open interval between two cursor bounds. +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct BoundInterval { + pub start: CursorBound, + pub end: CursorBound, +} + +/// A half-open interval as its raw bound texts. +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct TextInterval { + pub start: String, + pub end: String, +} + +/// One partition or synthetic completion that names both partition bounds. +#[derive(Clone, Debug, PartialEq)] +pub struct CompletionFact { + pub interval: BoundInterval, + /// Whether this is an ordinary partition completion rather than a synthetic one. + pub ordinary: bool, + pub unknown_fingerprint: bool, + /// `created_at.timestamp()`. + pub created: f64, + pub event_id: String, +} + +/// One interval and the index of the latest completion accounting for it. +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct CoverageSegment { + pub interval: TextInterval, + pub completion: usize, +} + +/// Physical and fingerprint coverage derived from completion facts. +#[derive(Clone, Debug, Default, PartialEq, Eq)] +pub struct CoverageProjection { + pub segments: Vec, + pub contiguous_frontier: Option, + pub known_missing: Vec, + pub unaccounted: Vec, + pub unknown_fingerprints: Vec, +} + +/// One projected coverage interval as a replay requirement reads it. +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct CoveragePiece { + pub interval: BoundInterval, + pub unknown_fingerprint: bool, + pub model_version_hash: Option, +} + +/// The versions one replay requirement is checked against. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub struct ReplayVersions<'a> { + pub required: Option<&'a str>, + pub current: &'a str, +} + +/// The derived state of one replay requirement. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum ReplayState { + Incomplete, + VerifiedComplete, + CompleteWithUnknownFingerprints, + Superseded, +} + +/// A replay requirement's state and the expected intervals still missing or unverified. +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct ReplayProjection { + pub state: ReplayState, + /// Indexes into the expected intervals. + pub missing: Vec, + pub unknown_fingerprints: Vec, +} + +/// One replay requirement event as the latest-requirement selection reads it. +#[derive(Clone, Debug, PartialEq)] +pub struct RequirementFact { + pub is_requirement: bool, + pub required_model_version_hash: Option, + pub created: f64, + pub event_id: String, +} + +/// The fields of a microbatch event's content-addressed identity, in identity order. +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct MicrobatchEventIdentity { + pub scope_kind: String, + pub scope_key: String, + pub model_name: String, + pub target_database: Option, + pub target_schema: Option, + pub target_name: String, + pub physical_generation_id: String, + pub record_type: String, + pub partition_start: Option, + pub partition_end: Option, + pub completion_reason: String, +} + +/// A microbatch state table read: its selected columns and scope literals. +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct ScopeRead<'a> { + pub columns: &'a [String], + pub scope_kind: &'a str, + pub scope_key: &'a str, + /// The generation literal, or `None` for every generation. + pub generation: Option<&'a str>, +} diff --git a/crates/sqlbuild-state/src/microbatches/tests.rs b/crates/sqlbuild-state/src/microbatches/tests.rs new file mode 100644 index 0000000000..e9d7907f38 --- /dev/null +++ b/crates/sqlbuild-state/src/microbatches/tests.rs @@ -0,0 +1,8 @@ +#[path = "tests/helpers.rs"] +mod helpers; +#[path = "tests/test_projection.rs"] +mod projection; +#[path = "tests/test_statements.rs"] +mod statements; +#[path = "tests/test_types.rs"] +mod test_types; diff --git a/crates/sqlbuild-state/src/microbatches/tests/helpers.rs b/crates/sqlbuild-state/src/microbatches/tests/helpers.rs new file mode 100644 index 0000000000..0f4d610246 --- /dev/null +++ b/crates/sqlbuild-state/src/microbatches/tests/helpers.rs @@ -0,0 +1,69 @@ +use sqlbuild_cursor::algebra::models::SortKey; + +use crate::microbatches::models::{ + BoundInterval, CompletionFact, CursorBound, MicrobatchEventIdentity, TextInterval, +}; + +/// An integer cursor bound. +pub(crate) fn bound(value: i128) -> CursorBound { + CursorBound { + text: value.to_string(), + key: SortKey::Integer(value), + } +} + +/// An integer interval `[start, end)`. +pub(crate) fn interval(start: i128, end: i128) -> BoundInterval { + BoundInterval { + start: bound(start), + end: bound(end), + } +} + +/// A known-fingerprint ordinary completion created `created` seconds in. +pub(crate) fn completion( + event_id: &str, + (start, end): (i128, i128), + created: f64, +) -> CompletionFact { + CompletionFact { + interval: interval(start, end), + ordinary: true, + unknown_fingerprint: false, + created, + event_id: event_id.to_owned(), + } +} + +/// The identity of an ordinary completion of `orders` over `[0, 1)`. +pub(crate) fn completion_identity() -> MicrobatchEventIdentity { + MicrobatchEventIdentity { + scope_kind: "direct_logical".to_owned(), + scope_key: "duckdb:main.orders".to_owned(), + model_name: "orders".to_owned(), + target_database: None, + target_schema: Some("main".to_owned()), + target_name: "orders".to_owned(), + physical_generation_id: "gen-1".to_owned(), + record_type: "partition_completion".to_owned(), + partition_start: Some("0".to_owned()), + partition_end: Some("1".to_owned()), + completion_reason: "initial".to_owned(), + } +} + +/// Interval texts as owned pairs. +pub(crate) fn texts(intervals: &[TextInterval]) -> Vec<(String, String)> { + intervals + .iter() + .map(|interval| (interval.start.clone(), interval.end.clone())) + .collect() +} + +/// Borrowed pairs as owned pairs. +pub(crate) fn owned(pairs: &[(&str, &str)]) -> Vec<(String, String)> { + pairs + .iter() + .map(|(start, end)| ((*start).to_owned(), (*end).to_owned())) + .collect() +} diff --git a/crates/sqlbuild-state/src/microbatches/tests/test_projection.rs b/crates/sqlbuild-state/src/microbatches/tests/test_projection.rs new file mode 100644 index 0000000000..dcd9767232 --- /dev/null +++ b/crates/sqlbuild-state/src/microbatches/tests/test_projection.rs @@ -0,0 +1,133 @@ +use crate::microbatches::main::project_coverage::project_coverage; +use crate::microbatches::main::project_replay::project_replay; +use crate::microbatches::models::{CoveragePiece, ReplayState, ReplayVersions}; +use crate::microbatches::tests::helpers::{completion, interval, owned, texts}; +use crate::microbatches::tests::test_types::{CoverageTestCase, ReplayTestCase}; + +#[test] +fn given_completion_history_when_projecting_then_latest_completion_and_gaps_are_derived() { + let test_cases = [ + CoverageTestCase { + description: "newer overlapping completion wins and a middle gap is known missing", + completions: vec![ + completion("a", (0, 2), 1.0), + completion("b", (1, 2), 2.0), + completion("c", (3, 4), 3.0), + ], + expected: vec![ + interval(0, 1), + interval(1, 2), + interval(2, 3), + interval(3, 4), + ], + expected_segments: vec![("0", "1", 0), ("1", "2", 1), ("3", "4", 2)], + expected_frontier: Some("2"), + expected_known_missing: vec![("2", "3")], + expected_unaccounted: Vec::new(), + }, + CoverageTestCase { + description: "no history leaves every expected interval unaccounted", + completions: Vec::new(), + expected: vec![interval(5, 6)], + expected_segments: Vec::new(), + expected_frontier: Some("5"), + expected_known_missing: Vec::new(), + expected_unaccounted: vec![("5", "6")], + }, + ]; + + for test_case in test_cases { + let projection = project_coverage(&test_case.completions, &test_case.expected); + let segments: Vec<(String, String, usize)> = projection + .segments + .iter() + .map(|segment| { + ( + segment.interval.start.clone(), + segment.interval.end.clone(), + segment.completion, + ) + }) + .collect(); + let expected_segments: Vec<(String, String, usize)> = test_case + .expected_segments + .iter() + .map(|(start, end, index)| ((*start).to_owned(), (*end).to_owned(), *index)) + .collect(); + assert_eq!(segments, expected_segments, "{}", test_case.description); + assert_eq!( + projection.contiguous_frontier.as_deref(), + test_case.expected_frontier, + "{}", + test_case.description + ); + assert_eq!( + texts(&projection.known_missing), + owned(&test_case.expected_known_missing), + "{}", + test_case.description + ); + assert_eq!( + texts(&projection.unaccounted), + owned(&test_case.expected_unaccounted), + "{}", + test_case.description + ); + } +} + +#[test] +fn given_replay_requirement_when_projecting_then_state_follows_coverage_versions() { + let piece = |start: i128, end: i128, version: &str, unknown: bool| CoveragePiece { + interval: interval(start, end), + unknown_fingerprint: unknown, + model_version_hash: Some(version.to_owned()), + }; + let test_cases = [ + ReplayTestCase { + description: "other version is superseded", + required: Some("F1"), + coverage: Vec::new(), + expected_state: ReplayState::Superseded, + expected_missing: Vec::new(), + }, + ReplayTestCase { + description: "current version everywhere is verified", + required: Some("F2"), + coverage: vec![piece(0, 2, "F2", false)], + expected_state: ReplayState::VerifiedComplete, + expected_missing: Vec::new(), + }, + ReplayTestCase { + description: "old version piece is missing", + required: Some("F2"), + coverage: vec![piece(0, 1, "F2", false), piece(1, 2, "F1", false)], + expected_state: ReplayState::Incomplete, + expected_missing: vec![1], + }, + ReplayTestCase { + description: "unknown fingerprint completes with unknown", + required: Some("F2"), + coverage: vec![piece(0, 2, "F2", true)], + expected_state: ReplayState::CompleteWithUnknownFingerprints, + expected_missing: Vec::new(), + }, + ]; + + for test_case in test_cases { + let projection = project_replay( + ReplayVersions { + required: test_case.required, + current: "F2", + }, + &[interval(0, 1), interval(1, 2)], + &test_case.coverage, + ); + assert_eq!( + (projection.state, projection.missing), + (test_case.expected_state, test_case.expected_missing), + "{}", + test_case.description + ); + } +} diff --git a/crates/sqlbuild-state/src/microbatches/tests/test_statements.rs b/crates/sqlbuild-state/src/microbatches/tests/test_statements.rs new file mode 100644 index 0000000000..af30110414 --- /dev/null +++ b/crates/sqlbuild-state/src/microbatches/tests/test_statements.rs @@ -0,0 +1,47 @@ +use crate::events::main::microbatch_event_id::microbatch_event_id; +use crate::microbatches::main::insert_many_sql::insert_many_sql; +use crate::microbatches::tests::helpers::completion_identity; +use crate::microbatches::tests::test_types::{EventIdTestCase, InsertManyTestCase}; + +#[test] +fn given_event_identity_when_hashing_then_matches_python_json_digest() { + let test_cases = [EventIdTestCase { + description: "ordinary completion", + identity: completion_identity(), + expected_id: "86cf4c901147bf7e0bde6673b0725aed47d013211f9ccef8bb84f77fca1454ae", + }]; + + for test_case in test_cases { + assert_eq!( + microbatch_event_id(&test_case.identity).ok().as_deref(), + Some(test_case.expected_id), + "{}", + test_case.description + ); + } +} + +#[test] +fn given_rows_when_rendering_bulk_insert_then_first_row_carries_aliases() { + let test_cases = [InsertManyTestCase { + description: "two rows", + rows: vec![ + vec!["'a'".to_owned(), "1".to_owned()], + vec!["'b'".to_owned(), "2".to_owned()], + ], + expected_sql: "INSERT INTO s.t (event_id, n) SELECT event_id, n FROM (SELECT 'a' AS event_id, 1 AS n UNION ALL SELECT 'b', 2) AS incoming WHERE NOT EXISTS (SELECT 1 FROM s.t AS existing WHERE existing.event_id = incoming.event_id)", + }]; + + for test_case in test_cases { + assert_eq!( + insert_many_sql( + "s.t", + &["event_id".to_owned(), "n".to_owned()], + &test_case.rows + ), + test_case.expected_sql, + "{}", + test_case.description + ); + } +} diff --git a/crates/sqlbuild-state/src/microbatches/tests/test_types.rs b/crates/sqlbuild-state/src/microbatches/tests/test_types.rs new file mode 100644 index 0000000000..10d82e23da --- /dev/null +++ b/crates/sqlbuild-state/src/microbatches/tests/test_types.rs @@ -0,0 +1,33 @@ +use crate::microbatches::models::{ + BoundInterval, CompletionFact, CoveragePiece, MicrobatchEventIdentity, ReplayState, +}; + +pub(crate) struct CoverageTestCase { + pub(crate) description: &'static str, + pub(crate) completions: Vec, + pub(crate) expected: Vec, + pub(crate) expected_segments: Vec<(&'static str, &'static str, usize)>, + pub(crate) expected_frontier: Option<&'static str>, + pub(crate) expected_known_missing: Vec<(&'static str, &'static str)>, + pub(crate) expected_unaccounted: Vec<(&'static str, &'static str)>, +} + +pub(crate) struct ReplayTestCase { + pub(crate) description: &'static str, + pub(crate) required: Option<&'static str>, + pub(crate) coverage: Vec, + pub(crate) expected_state: ReplayState, + pub(crate) expected_missing: Vec, +} + +pub(crate) struct EventIdTestCase { + pub(crate) description: &'static str, + pub(crate) identity: MicrobatchEventIdentity, + pub(crate) expected_id: &'static str, +} + +pub(crate) struct InsertManyTestCase { + pub(crate) description: &'static str, + pub(crate) rows: Vec>, + pub(crate) expected_sql: &'static str, +} diff --git a/fensu.toml b/fensu.toml index 81e90ca63c..0093a15632 100644 --- a/fensu.toml +++ b/fensu.toml @@ -36,7 +36,12 @@ reason = "Native bindings own lint entry points, expansion preparation and compi [[targets.python.rule_exceptions]] rule = "FFR309" path = "src/sqlbuild/cli/compile_reuse/__init__.py" -reason = "Native bindings own the compile reuse check, replay and launch records; this package declares the reuse models and constants and keeps the provider settings, runtime identity and startup inputs the native code reads from the interpreter." +reason = "Native bindings own the compile reuse check, runtime identity, replay and launch records; this package declares the reuse models and keeps the provider settings and startup inputs the native code reads from the interpreter." + +[[targets.python.rule_exceptions]] +rule = "FFR309" +path = "src/sqlbuild/microbatches/__init__.py" +reason = "Native bindings own microbatch coverage and replay projection, event IDs and state SQL; this package keeps the executor's direct event store, the event row codec and the event, scope and projection models it consumes." [[targets.python.rule_exceptions]] rule = "FFR309" diff --git a/python-boundary.toml b/python-boundary.toml index 67a120c464..3c45cc5aa4 100644 --- a/python-boundary.toml +++ b/python-boundary.toml @@ -47,7 +47,7 @@ note = "Decorators that run when user modules are imported." [[stays]] path = "microbatches" reason = "executor" -note = "Microbatch state store the executor and adapters use." +note = "Rule 1: the executor's direct-mode event store (warehouse I/O and retries), the event row codec and the event, scope and projection shapes it consumes. Coverage and replay projection, event IDs and state SQL are native (sqlbuild-state microbatches)." [[stays]] path = "rules" diff --git a/src/sqlbuild/_native.pyi b/src/sqlbuild/_native.pyi index 777f965a47..77803b9818 100644 --- a/src/sqlbuild/_native.pyi +++ b/src/sqlbuild/_native.pyi @@ -113,7 +113,14 @@ from sqlbuild.executor.testing.models import ( SqlTestExecutionResult, StepResult, ) -from sqlbuild.microbatches.models import MicrobatchInterval +from sqlbuild.microbatches.models import ( + MicrobatchCoverageProjection, + MicrobatchEvent, + MicrobatchInterval, + MicrobatchScope, + ReplayRequirementProjection, +) +from sqlbuild.microbatches.types import MicrobatchRecordType from sqlbuild.provider.classes.container import ProviderContainer from sqlbuild.provider.classes.provider import Provider from sqlbuild.provider.classes.session import ProviderSession @@ -5025,6 +5032,54 @@ def normalize_sql_value( def render_state_sql_literal(*, value: object | None, declared_type: StateSqlValueType) -> str: ... def format_diagnostics_console_record(record: logging.LogRecord, *, use_color: bool) -> str: ... def interpreter_identity() -> tuple[str, str, str, str]: ... +def project_microbatch_coverage( + *, + events: tuple[MicrobatchEvent, ...], + expected_intervals: tuple[MicrobatchInterval, ...], + cursor_type: str, +) -> MicrobatchCoverageProjection: ... +def project_replay_requirement( + *, + requirement: MicrobatchEvent, + current_model_version_hash: str, + expected_intervals: tuple[MicrobatchInterval, ...], + coverage: MicrobatchCoverageProjection, + cursor_type: str, +) -> ReplayRequirementProjection: ... +def latest_active_replay_requirement( + *, events: tuple[MicrobatchEvent, ...], current_model_version_hash: str +) -> MicrobatchEvent | None: ... +def deterministic_microbatch_event_id( + *, + scope: MicrobatchScope, + record_type: MicrobatchRecordType, + partition_start: str | None, + partition_end: str | None, + completion_reason: str, +) -> str: ... +def build_microbatch_insert_many_sql( + *, events: tuple[MicrobatchEvent, ...], render_qualified_name: Callable[..., str | None] +) -> str: ... +def build_microbatch_existing_event_ids_sql( + *, events: tuple[MicrobatchEvent, ...], render_qualified_name: Callable[..., str | None] +) -> str: ... +def build_microbatch_read_scope_sql( + *, scope: MicrobatchScope, render_qualified_name: Callable[..., str | None] +) -> str: ... +def build_microbatch_create_table_sql( + *, + database: str | None, + schema: str, + render_qualified_name: Callable[..., str | None], + render_framework_type: Callable[[FrameworkType], str], +) -> str: ... +def build_microbatch_create_index_sqls( + *, + database: str | None, + schema: str, + render_identifier: Callable[[str], str], + render_qualified_name: Callable[..., str | None], +) -> tuple[str, ...]: ... def render_state_table_create_sql( *, qualified_name: str, diff --git a/src/sqlbuild/adapter/contract/classes/base_adapter.py b/src/sqlbuild/adapter/contract/classes/base_adapter.py index cc5a230ce8..f915886a37 100644 --- a/src/sqlbuild/adapter/contract/classes/base_adapter.py +++ b/src/sqlbuild/adapter/contract/classes/base_adapter.py @@ -2124,9 +2124,9 @@ def render_create_fingerprint_table_sql( def render_create_microbatch_state_table_sql(self, *, database: str | None, schema: str) -> str: """Render DDL that creates the direct microbatch state table when missing.""" - from sqlbuild.microbatches.main.create_table_sql import build_create_table_sql + from sqlbuild._native import build_microbatch_create_table_sql - return build_create_table_sql( + return build_microbatch_create_table_sql( database=database, schema=schema, render_qualified_name=self.render_qualified_name, diff --git a/src/sqlbuild/adapter/contract/classes/microbatch.py b/src/sqlbuild/adapter/contract/classes/microbatch.py index c9766c5fbe..cc55246bc8 100644 --- a/src/sqlbuild/adapter/contract/classes/microbatch.py +++ b/src/sqlbuild/adapter/contract/classes/microbatch.py @@ -39,10 +39,10 @@ def physical_relation_generation( def render_create_microbatch_state_table_sql(self, *, database: str | None, schema: str) -> str: """Render portable DDL for the direct microbatch state table.""" - from sqlbuild.microbatches.main.create_table_sql import build_create_table_sql + from sqlbuild._native import build_microbatch_create_table_sql adapter: Any = cast(Any, self) - return build_create_table_sql( + return build_microbatch_create_table_sql( database=database, schema=schema, render_qualified_name=adapter.render_qualified_name, diff --git a/src/sqlbuild/adapters/duckdb/classes/duckdb_adapter.py b/src/sqlbuild/adapters/duckdb/classes/duckdb_adapter.py index 971c445f7d..7294fd39c3 100644 --- a/src/sqlbuild/adapters/duckdb/classes/duckdb_adapter.py +++ b/src/sqlbuild/adapters/duckdb/classes/duckdb_adapter.py @@ -62,9 +62,9 @@ def physical_relation_generation( def render_create_microbatch_state_index_sqls( self, *, database: str | None, schema: str ) -> tuple[str, ...]: - from sqlbuild.microbatches.main.create_index_sqls import build_create_index_sqls + from sqlbuild._native import build_microbatch_create_index_sqls - return build_create_index_sqls( + return build_microbatch_create_index_sqls( database=database, schema=schema, render_identifier=self.render_identifier, diff --git a/src/sqlbuild/adapters/postgres/classes/postgres_adapter.py b/src/sqlbuild/adapters/postgres/classes/postgres_adapter.py index 4514800e9f..793d5fef84 100644 --- a/src/sqlbuild/adapters/postgres/classes/postgres_adapter.py +++ b/src/sqlbuild/adapters/postgres/classes/postgres_adapter.py @@ -1539,9 +1539,9 @@ def render_create_fingerprint_index_sqls( def render_create_microbatch_state_index_sqls( self, *, database: str | None, schema: str ) -> tuple[str, ...]: - from sqlbuild.microbatches.main.create_index_sqls import build_create_index_sqls + from sqlbuild._native import build_microbatch_create_index_sqls - return build_create_index_sqls( + return build_microbatch_create_index_sqls( database=database, schema=schema, render_identifier=self.render_identifier, diff --git a/src/sqlbuild/adapters/sqlserver/classes/sqlserver_adapter.py b/src/sqlbuild/adapters/sqlserver/classes/sqlserver_adapter.py index c6768c3aa6..af95fd3413 100644 --- a/src/sqlbuild/adapters/sqlserver/classes/sqlserver_adapter.py +++ b/src/sqlbuild/adapters/sqlserver/classes/sqlserver_adapter.py @@ -361,10 +361,10 @@ def render_create_fingerprint_table_sql( ) def render_create_microbatch_state_table_sql(self, *, database: str | None, schema: str) -> str: + from sqlbuild._native import build_microbatch_create_table_sql from sqlbuild.microbatches.constants import MICROBATCH_TABLE_NAME - from sqlbuild.microbatches.main.create_table_sql import build_create_table_sql - create_sql: str = build_create_table_sql( + create_sql: str = build_microbatch_create_table_sql( database=database, schema=schema, render_qualified_name=self.render_qualified_name, diff --git a/src/sqlbuild/executor/run/_helpers/materializations/microbatch.py b/src/sqlbuild/executor/run/_helpers/materializations/microbatch.py index b0b733a938..27f888d74a 100644 --- a/src/sqlbuild/executor/run/_helpers/materializations/microbatch.py +++ b/src/sqlbuild/executor/run/_helpers/materializations/microbatch.py @@ -30,13 +30,17 @@ aligned_timestamp_bounds, compute_query_hash, cursor_bound_display, + deterministic_microbatch_event_id, empty_cursor_window, fit_auxiliary_relation_name, future_cursor_cap_warning, inclusive_cursor_end, + latest_active_replay_requirement, maximum_start_cap_warning, merge_microbatch_intervals, microbatch_limit_warning, + project_microbatch_coverage, + project_replay_requirement, resolve_effective_microbatch_batch_size, ) from sqlbuild._native import cursor_cap_from_end as cap_from_end @@ -126,14 +130,6 @@ MICROBATCH_REPLAY_GENERATION_PREFIX, ) from sqlbuild.microbatches.exceptions import MicrobatchStateError -from sqlbuild.microbatches.main.deterministic_event_id import ( - deterministic_microbatch_event_id, -) -from sqlbuild.microbatches.main.latest_replay_requirement import ( - latest_active_replay_requirement, -) -from sqlbuild.microbatches.main.project_coverage import project_microbatch_coverage -from sqlbuild.microbatches.main.project_replay import project_replay_requirement from sqlbuild.microbatches.models import ( MicrobatchCoverageProjection, MicrobatchEvent, diff --git a/src/sqlbuild/microbatches/_helpers/projection.py b/src/sqlbuild/microbatches/_helpers/projection.py deleted file mode 100644 index 26b2a14173..0000000000 --- a/src/sqlbuild/microbatches/_helpers/projection.py +++ /dev/null @@ -1,358 +0,0 @@ -"""Internal interval algebra for append-only microbatch history.""" - -from __future__ import annotations - -import heapq -from datetime import datetime - -from sqlbuild._native import cursor_compare as compare -from sqlbuild._native import cursor_max_bound as max_bound -from sqlbuild._native import cursor_min_bound as min_bound -from sqlbuild._native import cursor_parse as parse -from sqlbuild._native import cursor_sort_key -from sqlbuild.microbatches.models import ( - MicrobatchCoverageProjection, - MicrobatchEvent, - MicrobatchInterval, - ProjectedMicrobatchInterval, - ReplayRequirementProjection, -) -from sqlbuild.microbatches.types import ( - MicrobatchFingerprintStatus, - MicrobatchRecordType, - ReplayRequirementState, -) - - -def project_microbatch_coverage( - *, - events: tuple[MicrobatchEvent, ...], - expected_intervals: tuple[MicrobatchInterval, ...], - cursor_type: str, -) -> MicrobatchCoverageProjection: - """Fold completion facts and classify uncovered expected intervals.""" - - completions: tuple[MicrobatchEvent, ...] = tuple( - event - for event in events - if event.record_type - in { - MicrobatchRecordType.PARTITION_COMPLETION, - MicrobatchRecordType.SYNTHETIC_COMPLETION, - } - and event.partition_start is not None - and event.partition_end is not None - ) - boundaries: set[str] = set() - for event in completions: - if event.partition_start is not None: - boundaries.add(event.partition_start) - if event.partition_end is not None: - boundaries.add(event.partition_end) - boundaries.update(interval.start for interval in expected_intervals) - boundaries.update(interval.end for interval in expected_intervals) - ordered_boundaries: tuple[str, ...] = tuple( - sorted(boundaries, key=lambda value: cursor_sort_key(raw=value, cursor_type=cursor_type)) - ) - projected: list[ProjectedMicrobatchInterval] = [] - for start, end, latest in _latest_completion_segments( - completions=completions, - ordered_boundaries=ordered_boundaries, - cursor_type=cursor_type, - ): - projected.append( - ProjectedMicrobatchInterval( - start=start, - end=end, - fingerprint_status=latest.fingerprint_status, - model_version_hash=latest.model_version_hash, - record_type=latest.record_type, - completion_type=latest.completion_type, - event_id=latest.event_id, - ) - ) - - accounted: list[MicrobatchInterval] = [ - MicrobatchInterval(start=interval.start, end=interval.end) for interval in projected - ] - contiguous_frontier: str | None = _minimum_bound( - values=tuple(interval.start for interval in expected_intervals), - cursor_type=cursor_type, - ) - if accounted and contiguous_frontier is not None: - ordered_accounted: list[MicrobatchInterval] = sorted( - accounted, - key=lambda interval: cursor_sort_key(raw=interval.start, cursor_type=cursor_type), - ) - for interval in ordered_accounted: - if _lte(left=interval.start, right=contiguous_frontier, cursor_type=cursor_type): - if _lt(left=contiguous_frontier, right=interval.end, cursor_type=cursor_type): - contiguous_frontier = interval.end - else: - break - - latest_ordinary_end: str | None = _maximum_bound( - values=tuple( - event.partition_end or "" - for event in completions - if event.record_type == MicrobatchRecordType.PARTITION_COMPLETION - ), - cursor_type=cursor_type, - ) - known_missing, unaccounted = _classify_uncovered_expected_intervals( - expected_intervals=expected_intervals, - accounted=tuple(accounted), - latest_ordinary_end=latest_ordinary_end, - cursor_type=cursor_type, - ) - - return MicrobatchCoverageProjection( - intervals=tuple(projected), - contiguous_frontier=contiguous_frontier, - known_missing=tuple(known_missing), - unaccounted=tuple(unaccounted), - unknown_fingerprints=tuple( - MicrobatchInterval(start=interval.start, end=interval.end) - for interval in projected - if interval.fingerprint_status == MicrobatchFingerprintStatus.UNKNOWN - ), - ) - - -def _latest_completion_segments( - *, - completions: tuple[MicrobatchEvent, ...], - ordered_boundaries: tuple[str, ...], - cursor_type: str, -) -> tuple[tuple[str, str, MicrobatchEvent], ...]: - """Sweep interval boundaries while retaining the latest active completion.""" - - latest_order: tuple[MicrobatchEvent, ...] = tuple( - sorted( - completions, - key=lambda event: (_created_sort_value(event.created_at), event.event_id), - ) - ) - latest_rank_by_identity: dict[int, int] = { - id(event): rank for rank, event in enumerate(latest_order) - } - by_start: tuple[MicrobatchEvent, ...] = tuple( - sorted( - completions, - key=lambda event: cursor_sort_key( - raw=event.partition_start or "", cursor_type=cursor_type - ), - ) - ) - active: list[tuple[int, MicrobatchEvent]] = [] - next_event_index: int = 0 - segments: list[tuple[str, str, MicrobatchEvent]] = [] - for boundary_index in range(len(ordered_boundaries) - 1): - start: str = ordered_boundaries[boundary_index] - end: str = ordered_boundaries[boundary_index + 1] - while next_event_index < len(by_start) and _lte( - left=by_start[next_event_index].partition_start or "", - right=start, - cursor_type=cursor_type, - ): - event: MicrobatchEvent = by_start[next_event_index] - heapq.heappush(active, (-latest_rank_by_identity[id(event)], event)) - next_event_index += 1 - while active and not _lte( - left=end, - right=active[0][1].partition_end or "", - cursor_type=cursor_type, - ): - heapq.heappop(active) - if active: - segments.append((start, end, active[0][1])) - return tuple(segments) - - -def _created_sort_value(value: datetime) -> float: - return value.timestamp() - - -def _classify_uncovered_expected_intervals( - *, - expected_intervals: tuple[MicrobatchInterval, ...], - accounted: tuple[MicrobatchInterval, ...], - latest_ordinary_end: str | None, - cursor_type: str, -) -> tuple[tuple[MicrobatchInterval, ...], tuple[MicrobatchInterval, ...]]: - """Classify sorted expected intervals without rescanning prior coverage.""" - - ordered_expected: tuple[MicrobatchInterval, ...] = tuple( - sorted( - expected_intervals, - key=lambda interval: cursor_sort_key(raw=interval.start, cursor_type=cursor_type), - ) - ) - ordered_accounted: tuple[MicrobatchInterval, ...] = tuple( - sorted( - accounted, - key=lambda interval: cursor_sort_key(raw=interval.start, cursor_type=cursor_type), - ) - ) - known_missing: list[MicrobatchInterval] = [] - unaccounted: list[MicrobatchInterval] = [] - accounted_index: int = 0 - for expected in ordered_expected: - while accounted_index < len(ordered_accounted) and _lte( - left=ordered_accounted[accounted_index].end, - right=expected.start, - cursor_type=cursor_type, - ): - accounted_index += 1 - frontier: str = expected.start - scan_index: int = accounted_index - gaps: list[MicrobatchInterval] = [] - while scan_index < len(ordered_accounted): - interval: MicrobatchInterval = ordered_accounted[scan_index] - if not _lt(left=interval.start, right=expected.end, cursor_type=cursor_type): - break - if _lt(left=frontier, right=interval.start, cursor_type=cursor_type): - gaps.append(MicrobatchInterval(start=frontier, end=interval.start)) - if _lt(left=frontier, right=interval.end, cursor_type=cursor_type): - frontier = interval.end - if _lte(left=expected.end, right=frontier, cursor_type=cursor_type): - break - scan_index += 1 - if _lt(left=frontier, right=expected.end, cursor_type=cursor_type): - gaps.append(MicrobatchInterval(start=frontier, end=expected.end)) - for gap in gaps: - has_accounted_before: bool = accounted_index > 0 or _lt( - left=expected.start, right=gap.start, cursor_type=cursor_type - ) - if ( - has_accounted_before - and latest_ordinary_end is not None - and _lte( - left=gap.end, - right=latest_ordinary_end, - cursor_type=cursor_type, - ) - ): - known_missing.append(gap) - else: - unaccounted.append(gap) - return tuple(known_missing), tuple(unaccounted) - - -def project_replay_requirement( - *, - requirement: MicrobatchEvent, - current_model_version_hash: str, - expected_intervals: tuple[MicrobatchInterval, ...], - coverage: MicrobatchCoverageProjection, - cursor_type: str, -) -> ReplayRequirementProjection: - """Derive replay completion without writing mutable requirement status.""" - - if requirement.required_model_version_hash != current_model_version_hash: - return ReplayRequirementProjection( - requirement=requirement, - state=ReplayRequirementState.SUPERSEDED, - ) - missing: list[MicrobatchInterval] = [] - unknown: list[MicrobatchInterval] = [] - for expected in expected_intervals: - pieces: tuple[ProjectedMicrobatchInterval, ...] = tuple( - interval - for interval in coverage.intervals - if _lt(left=interval.start, right=expected.end, cursor_type=cursor_type) - and _lt(left=expected.start, right=interval.end, cursor_type=cursor_type) - ) - if not _is_fully_covered( - expected=expected, - accounted=tuple( - MicrobatchInterval(start=piece.start, end=piece.end) for piece in pieces - ), - cursor_type=cursor_type, - ): - missing.append(expected) - continue - if any(piece.fingerprint_status == MicrobatchFingerprintStatus.UNKNOWN for piece in pieces): - unknown.append(expected) - continue - if any(piece.model_version_hash != current_model_version_hash for piece in pieces): - missing.append(expected) - state: ReplayRequirementState - if missing: - state = ReplayRequirementState.INCOMPLETE - elif unknown: - state = ReplayRequirementState.COMPLETE_WITH_UNKNOWN_FINGERPRINTS - else: - state = ReplayRequirementState.VERIFIED_COMPLETE - return ReplayRequirementProjection( - requirement=requirement, - state=state, - missing=tuple(missing), - unknown_fingerprints=tuple(unknown), - ) - - -def latest_active_replay_requirement( - *, events: tuple[MicrobatchEvent, ...], current_model_version_hash: str -) -> MicrobatchEvent | None: - """Return only the latest requirement created for the currently compiled version.""" - - requirements: tuple[MicrobatchEvent, ...] = tuple( - event - for event in events - if event.record_type == MicrobatchRecordType.REPLAY_REQUIREMENT - and event.required_model_version_hash == current_model_version_hash - ) - if not requirements: - return None - return max(requirements, key=lambda event: (event.created_at, event.event_id)) - - -def _is_fully_covered( - *, - expected: MicrobatchInterval, - accounted: tuple[MicrobatchInterval, ...] | list[MicrobatchInterval], - cursor_type: str, -) -> bool: - frontier: str = expected.start - for interval in sorted( - accounted, key=lambda item: cursor_sort_key(raw=item.start, cursor_type=cursor_type) - ): - if _lte(left=interval.end, right=frontier, cursor_type=cursor_type): - continue - if _lt(left=frontier, right=interval.start, cursor_type=cursor_type): - return False - frontier = interval.end - if _lte(left=expected.end, right=frontier, cursor_type=cursor_type): - return True - return False - - -def _maximum_bound(*, values: tuple[str, ...], cursor_type: str) -> str | None: - non_empty: tuple[str, ...] = tuple(value for value in values if value) - return max_bound(values=non_empty, cursor_type=cursor_type) if non_empty else None - - -def _minimum_bound(*, values: tuple[str, ...], cursor_type: str) -> str | None: - non_empty: tuple[str, ...] = tuple(value for value in values if value) - return min_bound(values=non_empty, cursor_type=cursor_type) if non_empty else None - - -def _lt(*, left: str, right: str, cursor_type: str) -> bool: - return ( - compare( - left=parse(raw=left, cursor_type=cursor_type), - right=parse(raw=right, cursor_type=cursor_type), - ) - < 0 - ) - - -def _lte(*, left: str, right: str, cursor_type: str) -> bool: - return ( - compare( - left=parse(raw=left, cursor_type=cursor_type), - right=parse(raw=right, cursor_type=cursor_type), - ) - <= 0 - ) diff --git a/src/sqlbuild/microbatches/_helpers/sql.py b/src/sqlbuild/microbatches/_helpers/sql.py deleted file mode 100644 index c5c63fc6d6..0000000000 --- a/src/sqlbuild/microbatches/_helpers/sql.py +++ /dev/null @@ -1,170 +0,0 @@ -"""Internal portable SQL for direct-mode microbatch history.""" - -from __future__ import annotations - -from collections.abc import Callable - -from sqlbuild._native import StateSqlValueType, render_state_sql_literal -from sqlbuild.adapter.contract.types import FrameworkType -from sqlbuild.microbatches.classes.event_codec import MicrobatchEventCodec -from sqlbuild.microbatches.constants import ( - MICROBATCH_COLUMN_TYPES, - MICROBATCH_COLUMNS, - MICROBATCH_GENERATION_WILDCARD, - MICROBATCH_TABLE_NAME, -) -from sqlbuild.microbatches.exceptions import MicrobatchStateError -from sqlbuild.microbatches.models import MicrobatchEvent, MicrobatchScope - - -def build_create_table_sql( - *, - database: str | None, - schema: str, - render_qualified_name: Callable[..., str | None], - render_framework_type: Callable[[FrameworkType], str], -) -> str: - table: str = _qualified_table( - database=database, schema=schema, render_qualified_name=render_qualified_name - ) - text_type: str = render_framework_type(FrameworkType.STRING) - timestamp_type: str = render_framework_type(FrameworkType.TIMESTAMP) - definitions: list[str] = [] - for column in MICROBATCH_COLUMNS: - declared_type: StateSqlValueType = MICROBATCH_COLUMN_TYPES[column] - if declared_type == StateSqlValueType.TIMESTAMP: - column_type: str = timestamp_type - elif declared_type == StateSqlValueType.INTEGER: - column_type = "BIGINT" - else: - column_type = text_type - definitions.append(f"{column} {column_type}") - return f"CREATE TABLE IF NOT EXISTS {table} ({', '.join(definitions)})" - - -def build_insert_sql( - *, event: MicrobatchEvent, render_qualified_name: Callable[..., str | None] -) -> str: - table: str = _qualified_table( - database=event.scope.target_database, - schema=_required_schema(event.scope), - render_qualified_name=render_qualified_name, - ) - values: tuple[object | None, ...] = MicrobatchEventCodec.values(event) - literals: str = ", ".join( - _column_literal(column=column, value=value) - for column, value in zip(MICROBATCH_COLUMNS, values, strict=True) - ) - return ( - f"INSERT INTO {table} ({', '.join(MICROBATCH_COLUMNS)}) SELECT " - f"{literals} " - f"WHERE NOT EXISTS (SELECT 1 FROM {table} " - f"WHERE event_id = {_column_literal(column='event_id', value=event.event_id)})" - ) - - -def build_insert_many_sql( - *, events: tuple[MicrobatchEvent, ...], render_qualified_name: Callable[..., str | None] -) -> str: - """Build one guarded set-based insert for events in the same state table.""" - - if not events: - raise MicrobatchStateError("bulk microbatch event insert requires at least one event") - first: MicrobatchEvent = events[0] - table: str = _qualified_table( - database=first.scope.target_database, - schema=_required_schema(first.scope), - render_qualified_name=render_qualified_name, - ) - selections: list[str] = [] - for index, event in enumerate(events): - values: tuple[object | None, ...] = MicrobatchEventCodec.values(event) - aliases: str = ( - "" - if index > 0 - else " " - + ", ".join( - f"{_column_literal(column=column, value=value)} AS {column}" - for column, value in zip(MICROBATCH_COLUMNS, values, strict=True) - ) - ) - selections.append( - "SELECT " - + ", ".join( - _column_literal(column=column, value=value) - for column, value in zip(MICROBATCH_COLUMNS, values, strict=True) - ) - if index > 0 - else f"SELECT{aliases}" - ) - columns: str = ", ".join(MICROBATCH_COLUMNS) - return ( - f"INSERT INTO {table} ({columns}) SELECT {columns} FROM " - f"({' UNION ALL '.join(selections)}) AS incoming " - f"WHERE NOT EXISTS (SELECT 1 FROM {table} AS existing " - "WHERE existing.event_id = incoming.event_id)" - ) - - -def build_existing_event_ids_sql( - *, events: tuple[MicrobatchEvent, ...], render_qualified_name: Callable[..., str | None] -) -> str: - """Build one lookup for event IDs already present before a bulk insert.""" - - if not events: - raise MicrobatchStateError("microbatch event lookup requires at least one event") - first: MicrobatchEvent = events[0] - table: str = _qualified_table( - database=first.scope.target_database, - schema=_required_schema(first.scope), - render_qualified_name=render_qualified_name, - ) - event_ids: str = ", ".join( - _column_literal(column="event_id", value=event.event_id) for event in events - ) - return f"SELECT event_id FROM {table} WHERE event_id IN ({event_ids})" - - -def build_read_scope_sql( - *, scope: MicrobatchScope, render_qualified_name: Callable[..., str | None] -) -> str: - table: str = _qualified_table( - database=scope.target_database, - schema=_required_schema(scope), - render_qualified_name=render_qualified_name, - ) - generation_predicate: str = ( - "" - if scope.physical_generation_id == MICROBATCH_GENERATION_WILDCARD - else "AND physical_generation_id = " - + _column_literal(column="physical_generation_id", value=scope.physical_generation_id) - + " " - ) - return ( - f"SELECT {', '.join(MICROBATCH_COLUMNS)} FROM {table} " - f"WHERE scope_kind = {_column_literal(column='scope_kind', value=scope.scope_kind)} " - f"AND scope_key = {_column_literal(column='scope_key', value=scope.scope_key)} " - f"{generation_predicate}" - "ORDER BY created_at, event_id" - ) - - -def _qualified_table( - *, database: str | None, schema: str, render_qualified_name: Callable[..., str | None] -) -> str: - table: str | None = render_qualified_name( - database=database, schema=schema, name=MICROBATCH_TABLE_NAME - ) - if table is None: - raise MicrobatchStateError("microbatch history requires a target schema") - return table - - -def _required_schema(scope: MicrobatchScope) -> str: - if scope.target_schema is None: - raise MicrobatchStateError("microbatch history requires a target schema") - return scope.target_schema - - -def _column_literal(*, column: str, value: object | None) -> str: - return render_state_sql_literal(value=value, declared_type=MICROBATCH_COLUMN_TYPES[column]) diff --git a/src/sqlbuild/microbatches/classes/direct_store.py b/src/sqlbuild/microbatches/classes/direct_store.py index ed3e6f5f9b..1fb05f220f 100644 --- a/src/sqlbuild/microbatches/classes/direct_store.py +++ b/src/sqlbuild/microbatches/classes/direct_store.py @@ -6,13 +6,14 @@ from dataclasses import replace from typing import Any -from sqlbuild._native import CompiledRelationLocation, ModelPlanEntry -from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter -from sqlbuild.microbatches._helpers.sql import ( - build_existing_event_ids_sql, - build_insert_many_sql, - build_read_scope_sql, +from sqlbuild._native import ( + CompiledRelationLocation, + ModelPlanEntry, + build_microbatch_existing_event_ids_sql, + build_microbatch_insert_many_sql, + build_microbatch_read_scope_sql, ) +from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter from sqlbuild.microbatches.classes.event_codec import MicrobatchEventCodec from sqlbuild.microbatches.constants import ( DIRECT_MICROBATCH_SCOPE_KIND, @@ -54,7 +55,7 @@ def write_many(self, events: tuple[MicrobatchEvent, ...]) -> MicrobatchWriteResu ] cursor: Any = self._adapter.execute( connection=self._connection, - sql=build_existing_event_ids_sql( + sql=build_microbatch_existing_event_ids_sql( events=chunk, render_qualified_name=self._adapter.render_qualified_name, ), @@ -68,7 +69,7 @@ def write_many(self, events: tuple[MicrobatchEvent, ...]) -> MicrobatchWriteResu if missing: self._adapter.execute( connection=self._connection, - sql=build_insert_many_sql( + sql=build_microbatch_insert_many_sql( events=missing, render_qualified_name=self._adapter.render_qualified_name, ), @@ -108,7 +109,7 @@ def read_scope_history(self, scope: MicrobatchScope) -> tuple[MicrobatchEvent, . self._initialize(database=scope.target_database, schema=schema) cursor: Any = self._adapter.execute( connection=self._connection, - sql=build_read_scope_sql( + sql=build_microbatch_read_scope_sql( scope=scope, render_qualified_name=self._adapter.render_qualified_name ), ) diff --git a/src/sqlbuild/microbatches/main/__init__.py b/src/sqlbuild/microbatches/main/__init__.py deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/src/sqlbuild/microbatches/main/create_index_sqls.py b/src/sqlbuild/microbatches/main/create_index_sqls.py deleted file mode 100644 index a81d0fcc08..0000000000 --- a/src/sqlbuild/microbatches/main/create_index_sqls.py +++ /dev/null @@ -1,33 +0,0 @@ -"""Public direct microbatch-state index DDL rendering entrypoint.""" - -from __future__ import annotations - -from collections.abc import Callable - -from sqlbuild.microbatches.constants import ( - MICROBATCH_DIRECT_INDEXES, - MICROBATCH_TABLE_NAME, -) - - -def build_create_index_sqls( - *, - database: str | None, - schema: str, - render_identifier: Callable[[str], str], - render_qualified_name: Callable[..., str | None], -) -> tuple[str, ...]: - """Build non-unique scope and provenance indexes for indexed warehouses.""" - - table: str = ( - render_qualified_name(database=database, schema=schema, name=MICROBATCH_TABLE_NAME) - or f"{schema}.{MICROBATCH_TABLE_NAME}" - ) - statements: list[str] = [] - for index_name, columns in MICROBATCH_DIRECT_INDEXES.items(): - rendered_columns: tuple[str, ...] = tuple(render_identifier(column) for column in columns) - statements.append( - f"CREATE INDEX IF NOT EXISTS {render_identifier(index_name)} ON {table} " - f"({', '.join(rendered_columns)})" - ) - return tuple(statements) diff --git a/src/sqlbuild/microbatches/main/create_table_sql.py b/src/sqlbuild/microbatches/main/create_table_sql.py deleted file mode 100644 index 16d2e658f8..0000000000 --- a/src/sqlbuild/microbatches/main/create_table_sql.py +++ /dev/null @@ -1,25 +0,0 @@ -"""Public direct microbatch-state DDL rendering entrypoint.""" - -from __future__ import annotations - -from collections.abc import Callable - -from sqlbuild.adapter.contract.types import FrameworkType -from sqlbuild.microbatches._helpers.sql import build_create_table_sql as _build_create_table_sql - - -def build_create_table_sql( - *, - database: str | None, - schema: str, - render_qualified_name: Callable[..., str | None], - render_framework_type: Callable[[FrameworkType], str], -) -> str: - """Build direct microbatch-state DDL through the public entrypoint.""" - - return _build_create_table_sql( - database=database, - schema=schema, - render_qualified_name=render_qualified_name, - render_framework_type=render_framework_type, - ) diff --git a/src/sqlbuild/microbatches/main/deterministic_event_id.py b/src/sqlbuild/microbatches/main/deterministic_event_id.py deleted file mode 100644 index 361f9cefeb..0000000000 --- a/src/sqlbuild/microbatches/main/deterministic_event_id.py +++ /dev/null @@ -1,36 +0,0 @@ -"""Build stable identifiers for immutable logical microbatch events.""" - -from __future__ import annotations - -import hashlib -import json - -from sqlbuild.microbatches.models import MicrobatchScope -from sqlbuild.microbatches.types import MicrobatchRecordType - - -def deterministic_microbatch_event_id( - *, - scope: MicrobatchScope, - record_type: MicrobatchRecordType, - partition_start: str | None, - partition_end: str | None, - completion_reason: str, -) -> str: - """Return the content-addressed ID for one logical event.""" - - identity: tuple[object, ...] = ( - scope.scope_kind, - scope.scope_key, - scope.model_name, - scope.target_database, - scope.target_schema, - scope.target_name, - scope.physical_generation_id, - record_type.value, - partition_start, - partition_end, - completion_reason, - ) - encoded: str = json.dumps(identity, separators=(",", ":"), ensure_ascii=True) - return hashlib.sha256(encoded.encode()).hexdigest() diff --git a/src/sqlbuild/microbatches/main/latest_replay_requirement.py b/src/sqlbuild/microbatches/main/latest_replay_requirement.py deleted file mode 100644 index f1e5a9f331..0000000000 --- a/src/sqlbuild/microbatches/main/latest_replay_requirement.py +++ /dev/null @@ -1,16 +0,0 @@ -"""Public active replay-requirement selection entrypoint.""" - -from sqlbuild.microbatches._helpers.projection import ( - latest_active_replay_requirement as _latest_active_replay_requirement, -) -from sqlbuild.microbatches.models import MicrobatchEvent - - -def latest_active_replay_requirement( - *, events: tuple[MicrobatchEvent, ...], current_model_version_hash: str -) -> MicrobatchEvent | None: - """Return the latest requirement for the currently expected model version.""" - - return _latest_active_replay_requirement( - events=events, current_model_version_hash=current_model_version_hash - ) diff --git a/src/sqlbuild/microbatches/main/project_coverage.py b/src/sqlbuild/microbatches/main/project_coverage.py deleted file mode 100644 index a60b4d0fbf..0000000000 --- a/src/sqlbuild/microbatches/main/project_coverage.py +++ /dev/null @@ -1,25 +0,0 @@ -"""Public physical-coverage projection entrypoint.""" - -from sqlbuild.microbatches._helpers.projection import ( - project_microbatch_coverage as _project_microbatch_coverage, -) -from sqlbuild.microbatches.models import ( - MicrobatchCoverageProjection, - MicrobatchEvent, - MicrobatchInterval, -) - - -def project_microbatch_coverage( - *, - events: tuple[MicrobatchEvent, ...], - expected_intervals: tuple[MicrobatchInterval, ...], - cursor_type: str, -) -> MicrobatchCoverageProjection: - """Project physical and fingerprint coverage for expected intervals.""" - - return _project_microbatch_coverage( - events=events, - expected_intervals=expected_intervals, - cursor_type=cursor_type, - ) diff --git a/src/sqlbuild/microbatches/main/project_replay.py b/src/sqlbuild/microbatches/main/project_replay.py deleted file mode 100644 index 02f5dc2daa..0000000000 --- a/src/sqlbuild/microbatches/main/project_replay.py +++ /dev/null @@ -1,30 +0,0 @@ -"""Public replay-requirement projection entrypoint.""" - -from sqlbuild.microbatches._helpers.projection import ( - project_replay_requirement as _project_replay_requirement, -) -from sqlbuild.microbatches.models import ( - MicrobatchCoverageProjection, - MicrobatchEvent, - MicrobatchInterval, - ReplayRequirementProjection, -) - - -def project_replay_requirement( - *, - requirement: MicrobatchEvent, - current_model_version_hash: str, - expected_intervals: tuple[MicrobatchInterval, ...], - coverage: MicrobatchCoverageProjection, - cursor_type: str, -) -> ReplayRequirementProjection: - """Project derived completion state for one replay requirement.""" - - return _project_replay_requirement( - requirement=requirement, - current_model_version_hash=current_model_version_hash, - expected_intervals=expected_intervals, - coverage=coverage, - cursor_type=cursor_type, - ) diff --git a/tests/integration/src/sqlbuild/microbatches/classes/helpers.py b/tests/integration/src/sqlbuild/microbatches/classes/helpers.py index c7a59c4964..2e04e6a956 100644 --- a/tests/integration/src/sqlbuild/microbatches/classes/helpers.py +++ b/tests/integration/src/sqlbuild/microbatches/classes/helpers.py @@ -5,11 +5,9 @@ from datetime import UTC, datetime from typing import Any +from sqlbuild._native import deterministic_microbatch_event_id from sqlbuild.microbatches.classes.event_codec import MicrobatchEventCodec from sqlbuild.microbatches.constants import MICROBATCH_COLUMNS -from sqlbuild.microbatches.main.deterministic_event_id import ( - deterministic_microbatch_event_id, -) from sqlbuild.microbatches.models import MicrobatchEvent, MicrobatchScope from sqlbuild.microbatches.types import ( MicrobatchCompletionType, diff --git a/tests/integration/src/sqlbuild/microbatches/classes/test_retired_columns.py b/tests/integration/src/sqlbuild/microbatches/classes/test_retired_columns.py index 359c5c12e5..13e0d3681b 100644 --- a/tests/integration/src/sqlbuild/microbatches/classes/test_retired_columns.py +++ b/tests/integration/src/sqlbuild/microbatches/classes/test_retired_columns.py @@ -7,11 +7,13 @@ import pytest +from sqlbuild._native import ( + deterministic_microbatch_event_id, + project_microbatch_coverage, + project_replay_requirement, +) from sqlbuild.adapters.duckdb.classes.duckdb_adapter import DuckDbAdapter from sqlbuild.microbatches.classes.direct_store import DirectMicrobatchEventStore -from sqlbuild.microbatches.main.deterministic_event_id import deterministic_microbatch_event_id -from sqlbuild.microbatches.main.project_coverage import project_microbatch_coverage -from sqlbuild.microbatches.main.project_replay import project_replay_requirement from sqlbuild.microbatches.models import ( MicrobatchCoverageProjection, MicrobatchEvent, diff --git a/tests/unit/src/sqlbuild/microbatches/test_event_codec.py b/tests/unit/src/sqlbuild/microbatches/test_event_codec.py index db947e387c..7a70e45777 100644 --- a/tests/unit/src/sqlbuild/microbatches/test_event_codec.py +++ b/tests/unit/src/sqlbuild/microbatches/test_event_codec.py @@ -4,8 +4,8 @@ import pytest +from sqlbuild._native import project_microbatch_coverage from sqlbuild.microbatches.classes.event_codec import MicrobatchEventCodec -from sqlbuild.microbatches.main.project_coverage import project_microbatch_coverage from sqlbuild.microbatches.models import ( MicrobatchCoverageProjection, MicrobatchEvent, diff --git a/tests/unit/src/sqlbuild/microbatches/test_projection.py b/tests/unit/src/sqlbuild/microbatches/test_projection.py index 3b7f438193..98932a28c3 100644 --- a/tests/unit/src/sqlbuild/microbatches/test_projection.py +++ b/tests/unit/src/sqlbuild/microbatches/test_projection.py @@ -4,8 +4,7 @@ import pytest -from sqlbuild.microbatches.main.project_coverage import project_microbatch_coverage -from sqlbuild.microbatches.main.project_replay import project_replay_requirement +from sqlbuild._native import project_microbatch_coverage, project_replay_requirement from sqlbuild.microbatches.models import ( MicrobatchCoverageProjection, MicrobatchInterval, diff --git a/tests/unit/src/sqlbuild/microbatches/test_sql.py b/tests/unit/src/sqlbuild/microbatches/test_sql.py index 07050ec121..081005fdf6 100644 --- a/tests/unit/src/sqlbuild/microbatches/test_sql.py +++ b/tests/unit/src/sqlbuild/microbatches/test_sql.py @@ -6,7 +6,13 @@ import pytest -from sqlbuild._native import CursorType +from sqlbuild._native import ( + CursorType, + build_microbatch_create_table_sql, + build_microbatch_insert_many_sql, + build_microbatch_read_scope_sql, + deterministic_microbatch_event_id, +) from sqlbuild._native import cursor_parse as parse from sqlbuild._native import cursor_render as render from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter @@ -16,16 +22,7 @@ from sqlbuild.adapters.postgres.classes.postgres_adapter import PostgresAdapter from sqlbuild.adapters.snowflake.classes.snowflake_adapter import SnowflakeAdapter from sqlbuild.adapters.sqlserver.classes.sqlserver_adapter import SqlServerAdapter -from sqlbuild.microbatches._helpers.sql import ( - build_create_table_sql, - build_insert_many_sql, - build_insert_sql, - build_read_scope_sql, -) from sqlbuild.microbatches.constants import MICROBATCH_COLUMNS, MICROBATCH_TABLE_NAME -from sqlbuild.microbatches.main.deterministic_event_id import ( - deterministic_microbatch_event_id, -) from sqlbuild.microbatches.models import MicrobatchEvent, MicrobatchScope from sqlbuild.microbatches.types import MicrobatchRecordType from tests.unit.src.sqlbuild.microbatches._test_types import ( @@ -115,7 +112,7 @@ def test_given_builtin_adapter_when_building_state_ddl_then_all_columns_are_decl test_case: MicrobatchDdlAdapterTestCase, ) -> None: adapter: BaseAdapter = test_case.adapter - sql: str = build_create_table_sql( + sql: str = build_microbatch_create_table_sql( database="warehouse", schema="analytics", render_qualified_name=adapter.render_qualified_name, @@ -177,8 +174,8 @@ def test_given_completion_when_building_insert_then_provenance_and_numeric_rows_ ) -> None: event: MicrobatchEvent = completion_for_sql() - sql: str = build_insert_sql( - event=event, render_qualified_name=DuckDbAdapter().render_qualified_name + sql: str = build_microbatch_insert_many_sql( + events=(event,), render_qualified_name=DuckDbAdapter().render_qualified_name ) assert "INSERT INTO analytics._sqlbuild_microbatches" in sql @@ -190,22 +187,39 @@ def test_given_completion_when_building_insert_then_provenance_and_numeric_rows_ assert sql == ( "INSERT INTO analytics._sqlbuild_microbatches (event_id, record_type, scope_kind, " "scope_key, model_name, target_database, target_schema, target_name, " - "physical_generation_id, " - "origin_run_id, origin_run_started_at, execution_run_id, execution_run_started_at, " - "run_type, completion_type, run_start, run_end, partition_start, partition_end, " - "batch_size, cursor_column, cursor_type, cursor_grain, model_version_hash, " - "definition_hash, fingerprint_status, replay_requirement_id, " - "required_model_version_hash, previous_model_version_hash, replay_policy, " - "rows_affected, completed_at, coverage_source, observed_row_count, observed_at, " - "synthetic_reason, unaccounted_policy, created_at) SELECT 'event-1', " - "'partition_completion', 'direct_logical', 'duckdb:analytics.orders', 'orders', NULL, " - "'analytics', 'orders', '*', 'origin-run', CAST(NULL AS TIMESTAMP), " - "'execution-run', CAST(NULL AS TIMESTAMP), 'replay_on_change', 'recovery', '0', '1', " - "'0', '1', '1', 'batch_id', 'integer', NULL, 'F2', 'fingerprint''definition', " - "'known', NULL, NULL, NULL, NULL, CAST(0 AS BIGINT), CAST(NULL AS TIMESTAMP), NULL, " - "CAST(NULL AS BIGINT), CAST(NULL AS TIMESTAMP), NULL, NULL, " - "CAST('2026-01-01T00:00:00+00:00' AS TIMESTAMP) WHERE NOT EXISTS (SELECT 1 FROM " - "analytics._sqlbuild_microbatches WHERE event_id = 'event-1')" + "physical_generation_id, origin_run_id, origin_run_started_at, execution_run_id, " + "execution_run_started_at, run_type, completion_type, run_start, run_end, " + "partition_start, partition_end, batch_size, cursor_column, cursor_type, " + "cursor_grain, model_version_hash, definition_hash, fingerprint_status, " + "replay_requirement_id, required_model_version_hash, previous_model_version_hash, " + "replay_policy, rows_affected, completed_at, coverage_source, observed_row_count, " + "observed_at, synthetic_reason, unaccounted_policy, created_at) SELECT event_id, " + "record_type, scope_kind, scope_key, model_name, target_database, target_schema, " + "target_name, physical_generation_id, origin_run_id, origin_run_started_at, " + "execution_run_id, execution_run_started_at, run_type, completion_type, run_start, " + "run_end, partition_start, partition_end, batch_size, cursor_column, cursor_type, " + "cursor_grain, model_version_hash, definition_hash, fingerprint_status, " + "replay_requirement_id, required_model_version_hash, previous_model_version_hash, " + "replay_policy, rows_affected, completed_at, coverage_source, observed_row_count, " + "observed_at, synthetic_reason, unaccounted_policy, created_at FROM (SELECT 'event-1'" + " AS event_id, 'partition_completion' AS record_type, 'direct_logical' AS scope_kind," + " 'duckdb:analytics.orders' AS scope_key, 'orders' AS model_name, NULL AS " + "target_database, 'analytics' AS target_schema, 'orders' AS target_name, '*' AS " + "physical_generation_id, 'origin-run' AS origin_run_id, CAST(NULL AS TIMESTAMP) AS " + "origin_run_started_at, 'execution-run' AS execution_run_id, CAST(NULL AS TIMESTAMP) " + "AS execution_run_started_at, 'replay_on_change' AS run_type, 'recovery' AS " + "completion_type, '0' AS run_start, '1' AS run_end, '0' AS partition_start, '1' AS " + "partition_end, '1' AS batch_size, 'batch_id' AS cursor_column, 'integer' AS " + "cursor_type, NULL AS cursor_grain, 'F2' AS model_version_hash, " + "'fingerprint''definition' AS definition_hash, 'known' AS fingerprint_status, NULL AS" + " replay_requirement_id, NULL AS required_model_version_hash, NULL AS " + "previous_model_version_hash, NULL AS replay_policy, CAST(0 AS BIGINT) AS " + "rows_affected, CAST(NULL AS TIMESTAMP) AS completed_at, NULL AS coverage_source, " + "CAST(NULL AS BIGINT) AS observed_row_count, CAST(NULL AS TIMESTAMP) AS observed_at, " + "NULL AS synthetic_reason, NULL AS unaccounted_policy, " + "CAST('2026-01-01T00:00:00+00:00' AS TIMESTAMP) AS created_at) AS incoming WHERE NOT " + "EXISTS (SELECT 1 FROM analytics._sqlbuild_microbatches AS existing WHERE " + "existing.event_id = incoming.event_id)" ) assert len((sql,)) == test_case.expected_statement_count @@ -249,10 +263,10 @@ def test_given_builtin_adapter_when_rendering_direct_event_inserts_then_typed_co ) -> None: event: MicrobatchEvent = completion_for_sql() - single_sql: str = build_insert_sql( - event=event, render_qualified_name=test_case.adapter.render_qualified_name + single_sql: str = build_microbatch_insert_many_sql( + events=(event,), render_qualified_name=test_case.adapter.render_qualified_name ) - bulk_sql: str = build_insert_many_sql( + bulk_sql: str = build_microbatch_insert_many_sql( events=(event, replace(event, event_id="second-event")), render_qualified_name=test_case.adapter.render_qualified_name, ) @@ -275,10 +289,10 @@ def test_given_wildcard_and_concrete_generations_when_reading_then_filter_is_opt ) -> None: scope: MicrobatchScope = completion_for_sql().scope - wildcard_sql: str = build_read_scope_sql( + wildcard_sql: str = build_microbatch_read_scope_sql( scope=scope, render_qualified_name=DuckDbAdapter().render_qualified_name ) - concrete_sql: str = build_read_scope_sql( + concrete_sql: str = build_microbatch_read_scope_sql( scope=MicrobatchScope( scope_kind=scope.scope_kind, scope_key=scope.scope_key, From 9d24bb9faf9760cb1614bc60cb52d8657816efb3 Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Thu, 15 Oct 2026 21:53:02 +0100 Subject: [PATCH 30/48] refactor: validate lifecycle events against a native event catalogue --- crates/sqlbuild-core/src/lib.rs | 1 + .../src/lifecycle_events/_helpers/envelope.rs | 20 + .../src/lifecycle_events/_helpers/fields.rs | 123 ++ .../src/lifecycle_events/_helpers/mod.rs | 3 + .../src/lifecycle_events/_helpers/payloads.rs | 247 ++++ .../src/lifecycle_events/constants.rs | 1066 +++++++++++++++++ .../src/lifecycle_events/main/catalog.rs | 27 + .../src/lifecycle_events/main/mod.rs | 2 + .../main/validate_known_event.rs | 60 + .../sqlbuild-core/src/lifecycle_events/mod.rs | 8 + .../src/lifecycle_events/models.rs | 37 + .../src/lifecycle_events/tests.rs | 6 + .../src/lifecycle_events/tests/helpers.rs | 33 + .../src/lifecycle_events/tests/test_types.rs | 7 + .../lifecycle_events/tests/test_validate.rs | 62 + .../src/lifecycle_events/types.rs | 10 + .../src/bindings/_helpers/diagnostics/mod.rs | 1 + .../_helpers/diagnostics/registration.rs | 11 + .../_helpers/event_exporting/constants.rs | 1 - .../_helpers/event_exporting/policy.rs | 18 +- .../_helpers/event_exporting/values.rs | 18 +- .../_helpers/lifecycle_events/catalog.rs | 83 ++ .../bindings/_helpers/lifecycle_events/mod.rs | 4 + .../_helpers/lifecycle_events/payloads.rs | 157 +++ .../_helpers/lifecycle_events/registration.rs | 10 + .../_helpers/lifecycle_events/validation.rs | 112 ++ .../src/bindings/_helpers/mod.rs | 1 + .../src/bindings/main/register.rs | 12 +- python-boundary.toml | 2 +- src/sqlbuild/_native.pyi | 25 + .../run/_helpers/execution/promotion.py | 6 +- .../runtime/observability/_helpers/factory.py | 2 +- .../observability/_helpers/identity.py | 4 +- .../observability/_helpers/observability.py | 27 +- .../observability/_helpers/validation.py | 410 ------- .../classes/operation_lifecycle.py | 26 +- .../classes/resource_attempt_lifecycle.py | 7 +- .../runtime/observability/constants.py | 526 +------- .../main/canonicalize_operation_adapter.py | 2 +- src/sqlbuild/runtime/observability/models.py | 92 +- .../runtime/event_exporting/helpers.py | 29 + .../runtime/event_exporting/test_policy.py | 6 +- .../runtime/observability/test_catalog.py | 19 +- .../runtime/observability/test_models.py | 4 +- 44 files changed, 2247 insertions(+), 1080 deletions(-) create mode 100644 crates/sqlbuild-core/src/lifecycle_events/_helpers/envelope.rs create mode 100644 crates/sqlbuild-core/src/lifecycle_events/_helpers/fields.rs create mode 100644 crates/sqlbuild-core/src/lifecycle_events/_helpers/mod.rs create mode 100644 crates/sqlbuild-core/src/lifecycle_events/_helpers/payloads.rs create mode 100644 crates/sqlbuild-core/src/lifecycle_events/constants.rs create mode 100644 crates/sqlbuild-core/src/lifecycle_events/main/catalog.rs create mode 100644 crates/sqlbuild-core/src/lifecycle_events/main/mod.rs create mode 100644 crates/sqlbuild-core/src/lifecycle_events/main/validate_known_event.rs create mode 100644 crates/sqlbuild-core/src/lifecycle_events/mod.rs create mode 100644 crates/sqlbuild-core/src/lifecycle_events/models.rs create mode 100644 crates/sqlbuild-core/src/lifecycle_events/tests.rs create mode 100644 crates/sqlbuild-core/src/lifecycle_events/tests/helpers.rs create mode 100644 crates/sqlbuild-core/src/lifecycle_events/tests/test_types.rs create mode 100644 crates/sqlbuild-core/src/lifecycle_events/tests/test_validate.rs create mode 100644 crates/sqlbuild-core/src/lifecycle_events/types.rs create mode 100644 crates/sqlbuild-python/src/bindings/_helpers/diagnostics/registration.rs create mode 100644 crates/sqlbuild-python/src/bindings/_helpers/lifecycle_events/catalog.rs create mode 100644 crates/sqlbuild-python/src/bindings/_helpers/lifecycle_events/mod.rs create mode 100644 crates/sqlbuild-python/src/bindings/_helpers/lifecycle_events/payloads.rs create mode 100644 crates/sqlbuild-python/src/bindings/_helpers/lifecycle_events/registration.rs create mode 100644 crates/sqlbuild-python/src/bindings/_helpers/lifecycle_events/validation.rs delete mode 100644 src/sqlbuild/runtime/observability/_helpers/validation.py create mode 100644 tests/unit/src/sqlbuild/runtime/event_exporting/helpers.py diff --git a/crates/sqlbuild-core/src/lib.rs b/crates/sqlbuild-core/src/lib.rs index 7f9bf5d3a5..320d1e8996 100644 --- a/crates/sqlbuild-core/src/lib.rs +++ b/crates/sqlbuild-core/src/lib.rs @@ -33,6 +33,7 @@ pub mod init_command; pub mod integration_results; pub mod janitor_text; pub mod json; +pub mod lifecycle_events; pub mod load_command; pub mod migration_values; pub mod models; diff --git a/crates/sqlbuild-core/src/lifecycle_events/_helpers/envelope.rs b/crates/sqlbuild-core/src/lifecycle_events/_helpers/envelope.rs new file mode 100644 index 0000000000..d6366eb7b5 --- /dev/null +++ b/crates/sqlbuild-core/src/lifecycle_events/_helpers/envelope.rs @@ -0,0 +1,20 @@ +//! The catalogue definition and correlations of one event envelope. + +use crate::lifecycle_events::main::catalog::catalog; +use crate::lifecycle_events::models::{EventDefinition, KnownEventFacts}; + +/// Whether the named correlation field is set on the event. +pub(crate) fn correlation_set(event: &KnownEventFacts, name: &str) -> bool { + event + .correlations + .iter() + .any(|(field, set)| *field == name && *set) +} + +/// The catalogue definition of the event's type and schema version, if catalogued. +pub(crate) fn definition(event: &KnownEventFacts) -> Option { + let entries: Vec = catalog(event.schema_version)?; + entries + .into_iter() + .find(|entry| entry.event_type == event.event_type) +} diff --git a/crates/sqlbuild-core/src/lifecycle_events/_helpers/fields.rs b/crates/sqlbuild-core/src/lifecycle_events/_helpers/fields.rs new file mode 100644 index 0000000000..6ef184878e --- /dev/null +++ b/crates/sqlbuild-core/src/lifecycle_events/_helpers/fields.rs @@ -0,0 +1,123 @@ +//! Payload value predicates and the per-field type contract. + +use crate::json::main::dumps::dumps; +use crate::json::models::{JsonDialect, JsonInteger, JsonValue, StdlibJsonOptions}; +use crate::lifecycle_events::constants::{ + BOOLEAN_PAYLOAD_FIELDS, DURATION_MS_FIELD, EXIT_CODE_FIELD, FINITE_NUMBER_PAYLOAD_FIELDS, + JSON_PAYLOAD_FIELDS, MAX_METADATA_BYTES, METADATA_FIELD, MINIMUM_DURATION_MS, + NONNEGATIVE_INTEGER_PAYLOAD_FIELDS, STRING_PAYLOAD_FIELDS, +}; +use crate::lifecycle_events::models::PayloadValue; +use crate::text::main::python_str_repr::python_str_repr; + +/// `payload.get(name)`, treating a JSON null like an absent field. +pub(crate) fn get<'a>( + payload: &'a [(String, PayloadValue)], + name: &str, +) -> Option<&'a PayloadValue> { + payload + .iter() + .find(|(key, _)| key == name) + .map(|(_, value)| value) + .filter(|value| !matches!(value, PayloadValue::Null)) +} + +/// The integer a value holds, excluding booleans. +pub(crate) fn integer(value: Option<&PayloadValue>) -> Option { + match value { + Some(PayloadValue::Integer { value, .. }) => Some(*value), + _ => None, + } +} + +/// Whether a value is text listed in `allowed`. +pub(crate) fn catalogued(value: Option<&PayloadValue>, allowed: &[&str]) -> bool { + matches!(value, Some(PayloadValue::Text(text)) if allowed.contains(&text.as_str())) +} + +fn finite_number(value: &PayloadValue) -> Option { + match value { + PayloadValue::Integer { value, .. } => Some(*value as f64), + PayloadValue::Float(number) if number.is_finite() => Some(*number), + _ => None, + } +} + +fn json(value: &PayloadValue) -> JsonValue { + match value { + PayloadValue::Null => JsonValue::Null, + PayloadValue::Bool(flag) => JsonValue::Bool(*flag), + PayloadValue::Integer { text, .. } => { + JsonInteger::parse(text).map_or(JsonValue::Null, JsonValue::Integer) + } + PayloadValue::Float(number) => JsonValue::Float(*number), + PayloadValue::Text(text) => JsonValue::String(text.clone()), + PayloadValue::Array(items) => JsonValue::Array(items.iter().map(json).collect()), + PayloadValue::Object(entries) => JsonValue::Object( + entries + .iter() + .map(|(key, item)| (key.clone(), json(item))) + .collect(), + ), + } +} + +/// The byte length of `json.dumps(value, sort_keys=True, separators=(",", ":"))`. +fn encoded_length(value: &PayloadValue) -> usize { + let options = StdlibJsonOptions { + sort_keys: true, + allow_nan: false, + ..StdlibJsonOptions::new(None).with_separators(",", ":") + }; + dumps(&json(value), &JsonDialect::Stdlib(options)).map_or(0, |text| text.len()) +} + +/// The type contract of one payload field, as the first failing message. +pub(crate) fn field_problem(name: &str, value: &PayloadValue) -> Option { + let quoted: String = python_str_repr(name); + if STRING_PAYLOAD_FIELDS.contains(&name) { + let valid: bool = matches!(value, PayloadValue::Text(text) if !text.is_empty()); + return (!valid).then(|| format!("payload field {quoted} must be a non-empty string")); + } + if NONNEGATIVE_INTEGER_PAYLOAD_FIELDS.contains(&name) { + let valid: bool = integer(Some(value)).is_some_and(|number| number >= 0); + return (!valid).then(|| { + format!("payload field {quoted} must be a nonnegative integer excluding bool") + }); + } + if FINITE_NUMBER_PAYLOAD_FIELDS.contains(&name) { + return finite_number(value) + .is_none() + .then(|| format!("payload field {quoted} must be a finite number excluding bool")); + } + if BOOLEAN_PAYLOAD_FIELDS.contains(&name) { + return (!matches!(value, PayloadValue::Bool(_))) + .then(|| format!("payload field {quoted} must be a boolean")); + } + if JSON_PAYLOAD_FIELDS.contains(&name) { + return None; + } + if name == DURATION_MS_FIELD { + let valid: bool = finite_number(value).is_some_and(|number| number >= MINIMUM_DURATION_MS); + return (!valid).then(|| { + "payload field 'duration_ms' must be a nonnegative finite number excluding bool" + .to_owned() + }); + } + if name == EXIT_CODE_FIELD { + return integer(Some(value)) + .is_none() + .then(|| "payload field 'exit_code' must be an integer excluding bool".to_owned()); + } + if name == METADATA_FIELD { + if !matches!(value, PayloadValue::Object(_)) { + return Some("payload field 'metadata' must be a JSON object".to_owned()); + } + if encoded_length(value) > MAX_METADATA_BYTES { + return Some(format!( + "payload field 'metadata' must encode to at most {MAX_METADATA_BYTES} bytes" + )); + } + } + None +} diff --git a/crates/sqlbuild-core/src/lifecycle_events/_helpers/mod.rs b/crates/sqlbuild-core/src/lifecycle_events/_helpers/mod.rs new file mode 100644 index 0000000000..7c28c938e5 --- /dev/null +++ b/crates/sqlbuild-core/src/lifecycle_events/_helpers/mod.rs @@ -0,0 +1,3 @@ +pub(crate) mod envelope; +pub(crate) mod fields; +pub(crate) mod payloads; diff --git a/crates/sqlbuild-core/src/lifecycle_events/_helpers/payloads.rs b/crates/sqlbuild-core/src/lifecycle_events/_helpers/payloads.rs new file mode 100644 index 0000000000..a653721d5a --- /dev/null +++ b/crates/sqlbuild-core/src/lifecycle_events/_helpers/payloads.rs @@ -0,0 +1,247 @@ +//! Event-family payload rules: statements, operations, microbatches, skips, retries and runs. + +use std::collections::BTreeSet; + +use crate::lifecycle_events::_helpers::fields::field_problem; +use crate::lifecycle_events::_helpers::fields::{catalogued, get, integer}; +use crate::lifecycle_events::constants::{ + AUDIT_RUN_COUNT_FIELDS, CONFIGURED_CONCURRENCY_FIELD, FORBIDDEN_STATEMENT_PAYLOAD_FIELDS, + HOOK_NAME_MAX_TAIL, HOOK_PHASES, HOOK_TYPES, METADATA_FIELD, MICROBATCH_EVENT_PREFIX, + MICROBATCH_KINDS, MICROBATCH_RUN_TYPES, MICROBATCH_STRATEGIES, OPERATION_ADAPTERS, + OPERATION_EVENT_PREFIX, OPERATION_KINDS, OPERATION_METADATA_FIELDS, OPERATION_NAMES, + OPERATION_PHASES, OPERATION_SCOPES, OPERATION_STRATEGIES, OPERATION_TARGET_KINDS, + RESOURCE_ATTEMPT_SKIPPED_EVENT, RESOURCE_SKIP_CODES, RESOURCE_SKIP_MODES, + RETRY_SCHEDULED_EVENT, RUN_STARTED_EVENT, RUN_TERMINALS, STATEMENT_EVENT_PREFIX, +}; +use crate::lifecycle_events::models::{KnownEventFacts, PayloadValue}; + +type Payload = [(String, PayloadValue)]; + +/// Every forbidden SQL or parameter key anywhere in a statement payload. +pub(crate) fn forbidden_statement_fields(value: &PayloadValue) -> BTreeSet { + let mut found: BTreeSet = BTreeSet::new(); + match value { + PayloadValue::Object(entries) => { + for (key, item) in entries { + if FORBIDDEN_STATEMENT_PAYLOAD_FIELDS.contains(&key.as_str()) { + found.insert(key.clone()); + } + found.extend(forbidden_statement_fields(item)); + } + } + PayloadValue::Array(items) => { + for item in items { + found.extend(forbidden_statement_fields(item)); + } + } + _ => {} + } + found +} + +fn safe_hook_name(name: &str) -> bool { + let mut characters = name.chars(); + let Some(first) = characters.next() else { + return false; + }; + (first.is_ascii_alphabetic() || first == '_') + && name.len() <= HOOK_NAME_MAX_TAIL + 1 + && characters.all(|character| character.is_ascii_alphanumeric() || character == '_') +} + +fn uncatalogued(payload: &Payload, name: &str, allowed: &[&str]) -> bool { + get(payload, name).is_some_and(|value| !catalogued(Some(value), allowed)) +} + +/// Catalogued operation dimensions, hook identity and allowlisted metadata counts. +pub(crate) fn operation_problem(payload: &Payload) -> Option { + if uncatalogued(payload, "operation_kind", &OPERATION_KINDS) { + return Some("operation_kind must be a catalogued value".to_owned()); + } + if uncatalogued(payload, "operation_name", &OPERATION_NAMES) { + return Some("operation_name must be a catalogued value".to_owned()); + } + let dimensions: [(&str, &[&str]); 5] = [ + ("phase", &OPERATION_PHASES), + ("strategy", &OPERATION_STRATEGIES), + ("target_kind", &OPERATION_TARGET_KINDS), + ("adapter", &OPERATION_ADAPTERS), + ("scope", &OPERATION_SCOPES), + ]; + for (name, allowed) in dimensions { + if uncatalogued(payload, name, allowed) { + return Some(format!("{name} must be a catalogued value")); + } + } + let hook_fields = ["hook_phase", "hook_type", "hook_index", "hook_name"]; + if hook_fields.iter().any(|name| get(payload, name).is_some()) { + if !catalogued(get(payload, "hook_phase"), &HOOK_PHASES) { + return Some("hook_phase must be a catalogued value".to_owned()); + } + if !catalogued(get(payload, "hook_type"), &HOOK_TYPES) { + return Some("hook_type must be a catalogued value".to_owned()); + } + if !integer(get(payload, "hook_index")).is_some_and(|index| index >= 0) { + return Some("hook_index must be a nonnegative integer".to_owned()); + } + let unsafe_name: bool = match get(payload, "hook_name") { + None => false, + Some(PayloadValue::Text(name)) => !safe_hook_name(name), + Some(_) => true, + }; + if unsafe_name { + return Some("hook_name must be a non-empty safe token".to_owned()); + } + } + let entries: &Vec<(String, PayloadValue)> = match get(payload, METADATA_FIELD)? { + PayloadValue::Object(entries) => entries, + _ => return Some("operation metadata must be a JSON object".to_owned()), + }; + if entries + .iter() + .any(|(key, _)| !OPERATION_METADATA_FIELDS.contains(&key.as_str())) + { + return Some("operation metadata contains a non-allowlisted field".to_owned()); + } + entries + .iter() + .any(|(_, value)| !integer(Some(value)).is_some_and(|count| count >= 0)) + .then(|| "operation metadata values must be nonnegative integers excluding bool".to_owned()) +} + +/// Positive, ordered batch positions and catalogued microbatch dimensions. +pub(crate) fn microbatch_problem(payload: &Payload) -> Option { + let Some(index) = integer(get(payload, "batch_index")).filter(|index| *index >= 1) else { + return Some("batch_index must be a positive integer".to_owned()); + }; + let Some(count) = integer(get(payload, "batch_count")).filter(|count| *count >= 1) else { + return Some("batch_count must be a positive integer".to_owned()); + }; + if index > count { + return Some("batch_index must not exceed batch_count".to_owned()); + } + let dimensions: [(&str, &[&str]); 3] = [ + ("microbatch_strategy", &MICROBATCH_STRATEGIES), + ("microbatch_run_type", &MICROBATCH_RUN_TYPES), + ("batch_kind", &MICROBATCH_KINDS), + ]; + dimensions + .into_iter() + .find(|(name, allowed)| !catalogued(get(payload, name), allowed)) + .map(|(name, _)| format!("{name} must be a catalogued value")) +} + +/// A catalogued skip code and, when present, skip mode. +pub(crate) fn skip_problem(payload: &Payload) -> Option { + if !catalogued(get(payload, "skip_code"), &RESOURCE_SKIP_CODES) { + return Some("skip_code must be a catalogued value".to_owned()); + } + uncatalogued(payload, "skip_mode", &RESOURCE_SKIP_MODES) + .then(|| "skip_mode must be a catalogued value".to_owned()) +} + +/// A positive failed attempt followed by the next attempt number. +pub(crate) fn retry_problem(payload: &Payload) -> Option { + let Some(failed) = integer(get(payload, "failed_attempt_number")).filter(|number| *number > 0) + else { + return Some("failed_attempt_number must be a positive integer".to_owned()); + }; + (integer(get(payload, "next_attempt_number")) != Some(failed + 1)) + .then(|| "next_attempt_number must equal failed_attempt_number + 1".to_owned()) +} + +fn run_count(payload: &Payload, name: &str) -> Result { + integer(get(payload, name)) + .filter(|count| *count >= 0) + .ok_or_else(|| format!("{name} must be a nonnegative integer excluding bool")) +} + +/// Worker counts of a started run and audit totals of a finished run. +pub(crate) fn run_problem(event_type: &str, payload: &Payload) -> Option { + let present = |name: &str| payload.iter().any(|(key, _)| key == name); + if event_type == RUN_STARTED_EVENT && present(CONFIGURED_CONCURRENCY_FIELD) { + let Some(configured) = + integer(get(payload, CONFIGURED_CONCURRENCY_FIELD)).filter(|count| *count >= 1) + else { + return Some( + "configured_concurrency must be a positive integer excluding bool".to_owned(), + ); + }; + let Some(selected) = integer(get(payload, "selected_count")).filter(|count| *count >= 0) + else { + return Some("selected_count must be a nonnegative integer excluding bool".to_owned()); + }; + let Some(workers) = integer(get(payload, "worker_count")) else { + return Some("worker_count must be an integer excluding bool".to_owned()); + }; + if workers != configured.min(selected) { + return Some( + "worker_count must equal min(configured_concurrency, selected_count)".to_owned(), + ); + } + } + if !RUN_TERMINALS.contains(&event_type) + || !AUDIT_RUN_COUNT_FIELDS.iter().any(|name| present(name)) + { + return None; + } + if !AUDIT_RUN_COUNT_FIELDS.iter().all(|name| present(name)) { + return Some( + "audit run terminals must include pass_count, warn_count, and fail_count together" + .to_owned(), + ); + } + let counts = || -> Result, String> { + let pass: i128 = run_count(payload, "pass_count")?; + let warn: i128 = run_count(payload, "warn_count")?; + let fail: i128 = run_count(payload, "fail_count")?; + let succeeded: i128 = run_count(payload, "succeeded_count")?; + let failed: i128 = run_count(payload, "failed_count")?; + if succeeded != pass + warn { + return Ok(Some( + "succeeded_count must equal pass_count + warn_count".to_owned(), + )); + } + Ok((failed != fail).then(|| "failed_count must equal fail_count".to_owned())) + }; + counts().unwrap_or_else(Some) +} + +/// The first statement, operation, microbatch, skip, retry, field or run violation. +pub(crate) fn family_problem(event: &KnownEventFacts) -> Option { + let event_type: &str = &event.event_type; + if event_type.starts_with(STATEMENT_EVENT_PREFIX) { + let forbidden: BTreeSet = + forbidden_statement_fields(&PayloadValue::Object(event.payload.clone())); + if !forbidden.is_empty() { + let names: Vec = forbidden.into_iter().collect(); + return Some(format!( + "statement lifecycle payload must not contain SQL or parameter values; \ +forbidden field(s): {}", + names.join(", ") + )); + } + } + let checks: [(bool, fn(&[(String, PayloadValue)]) -> Option); 4] = [ + ( + event_type.starts_with(OPERATION_EVENT_PREFIX), + operation_problem, + ), + ( + event_type.starts_with(MICROBATCH_EVENT_PREFIX), + microbatch_problem, + ), + (event_type == RESOURCE_ATTEMPT_SKIPPED_EVENT, skip_problem), + (event_type == RETRY_SCHEDULED_EVENT, retry_problem), + ]; + for (applies, check) in checks { + if let Some(problem) = applies.then(|| check(&event.payload)).flatten() { + return Some(problem); + } + } + event + .payload + .iter() + .find_map(|(name, value)| field_problem(name, value)) + .or_else(|| run_problem(event_type, &event.payload)) +} diff --git a/crates/sqlbuild-core/src/lifecycle_events/constants.rs b/crates/sqlbuild-core/src/lifecycle_events/constants.rs new file mode 100644 index 0000000000..f7ad1aa621 --- /dev/null +++ b/crates/sqlbuild-core/src/lifecycle_events/constants.rs @@ -0,0 +1,1066 @@ +//! The versioned lifecycle event catalogue and the payload value contracts it validates. + +use crate::lifecycle_events::types::CatalogEntry; + +/// The lifecycle event schema version SQLBuild produces. +pub const CURRENT_LIFECYCLE_EVENT_SCHEMA_VERSION: i64 = 2; +/// Every lifecycle event schema version with a catalogue, oldest first. +pub const LIFECYCLE_EVENT_SCHEMA_VERSIONS: [i64; 2] = [1, 2]; +pub(crate) const MAX_METADATA_BYTES: usize = 4096; +/// The smallest duration a payload may carry. +pub(crate) const MINIMUM_DURATION_MS: f64 = 0.0; +/// The most characters a hook name may hold after its first. +pub(crate) const HOOK_NAME_MAX_TAIL: usize = 127; +pub(crate) const DURATION_MS_FIELD: &str = "duration_ms"; +pub(crate) const EXIT_CODE_FIELD: &str = "exit_code"; +pub(crate) const METADATA_FIELD: &str = "metadata"; +pub(crate) const CONFIGURED_CONCURRENCY_FIELD: &str = "configured_concurrency"; +pub(crate) const RUN_STARTED_EVENT: &str = "run_started"; +pub(crate) const STATEMENT_EVENT_PREFIX: &str = "statement_"; +pub(crate) const OPERATION_EVENT_PREFIX: &str = "operation_"; +pub(crate) const MICROBATCH_EVENT_PREFIX: &str = "microbatch_"; +pub(crate) const RESOURCE_ATTEMPT_SKIPPED_EVENT: &str = "resource_attempt_skipped"; +pub(crate) const RETRY_SCHEDULED_EVENT: &str = "retry_scheduled"; +pub(crate) const RUN_TERMINALS: [&str; 2] = ["run_completed", "run_failed"]; +pub(crate) const AUDIT_RUN_COUNT_FIELDS: [&str; 3] = ["fail_count", "pass_count", "warn_count"]; +pub(crate) const MICROBATCH_STRATEGIES: [&str; 2] = ["rolling_window", "watermark"]; +pub(crate) const MICROBATCH_RUN_TYPES: [&str; 3] = ["backfill", "normal", "replay_on_change"]; +pub(crate) const MICROBATCH_KINDS: [&str; 2] = ["ordinary", "recovery"]; +pub const OPERATION_TARGET_KINDS: [&str; 7] = [ + "audit", + "namespace", + "relation", + "source", + "sql_test", + "staging_relation", + "state_batch", +]; +pub const OPERATION_SCOPES: [&str; 7] = [ + "delta", + "end", + "final", + "model", + "seed", + "source", + "standalone", +]; +pub const OPERATION_PHASES: [&str; 11] = [ + "apply", + "assert", + "convert", + "create", + "evaluate", + "inspect", + "observe", + "promote", + "publish", + "reconcile", + "setup", +]; +pub(crate) const FORBIDDEN_STATEMENT_PAYLOAD_FIELDS: [&str; 8] = [ + "bindings", + "full_sql", + "parameter_values", + "parameters", + "params", + "query_sql", + "sql", + "statement_sql", +]; +pub(crate) const STRING_PAYLOAD_FIELDS: [&str; 51] = [ + "adapter", + "attached_column_name", + "attached_target_kind", + "attached_target_name", + "attachment_kind", + "audit_definition_name", + "audit_description", + "audit_name", + "batch_kind", + "binding_key", + "command", + "configured_batch_size", + "cursor_end_exclusive", + "cursor_start", + "definition_fingerprint", + "effective_batch_size", + "error_code", + "error_type", + "evaluation_mode", + "evidence_error", + "evidence_sql", + "executed_sql", + "execution_error", + "execution_fingerprint", + "hook_name", + "hook_phase", + "hook_type", + "intent", + "job_id", + "measurement_sql", + "microbatch_run_type", + "microbatch_strategy", + "operation_kind", + "operation_name", + "outcome", + "planned_cursor_end_exclusive", + "planned_cursor_start", + "query_id", + "resource_kind", + "resource_name", + "result_id", + "run_kind", + "run_scope_phase", + "sample_unit", + "scope", + "severity", + "sql_digest", + "statement_kind", + "target_database", + "target_name", + "target_schema", +]; +pub(crate) const NONNEGATIVE_INTEGER_PAYLOAD_FIELDS: [&str; 25] = [ + "affected_rows", + "attempt_number", + "batch_count", + "batch_index", + "batch_size", + "configured_concurrency", + "delay_ms", + "evidence_count", + "fail_count", + "failed_attempt_number", + "failed_count", + "hook_index", + "minimum_samples", + "next_attempt_number", + "pass_count", + "process_id", + "row_count", + "sample_count", + "selected_count", + "signal_number", + "skipped_count", + "succeeded_count", + "violation_count", + "warn_count", + "worker_count", +]; +pub(crate) const FINITE_NUMBER_PAYLOAD_FIELDS: [&str; 1] = ["measured_value"]; +pub(crate) const BOOLEAN_PAYLOAD_FIELDS: [&str; 2] = ["evidence_truncated", "reused"]; +pub(crate) const JSON_PAYLOAD_FIELDS: [&str; 2] = ["evidence", "thresholds"]; +pub const RESOURCE_SKIP_CODES: [&str; 4] = ["dependency", "explicit", "fan_in", "scheduler"]; +pub const RESOURCE_SKIP_MODES: [&str; 2] = ["hard", "soft"]; +pub const HOOK_PHASES: [&str; 2] = ["post_hooks", "pre_hooks"]; +pub const HOOK_TYPES: [&str; 2] = ["python", "sql"]; +pub const OPERATION_KINDS: [&str; 10] = [ + "clone", + "freshness", + "janitor", + "loader", + "project", + "python_node", + "quality", + "scenario", + "subprocess", + "warehouse", +]; +pub const OPERATION_NAMES: [&str; 55] = [ + "audit_evaluation", + "clone_execution", + "clone_finalization", + "clone_namespace_preparation", + "clone_relation_inspection", + "clone_relation_transfer", + "clone_retention_reconciliation", + "clone_state_connection", + "clone_state_inspection", + "clone_target_connection", + "dbt_command", + "discovery_declaration_parse", + "discovery_filesystem_walk", + "discovery_project_assembly", + "discovery_python_import", + "external_manifest_discovery", + "external_source_load", + "ingestr_command", + "janitor_candidate_planning", + "janitor_cleanup_action", + "janitor_execution", + "janitor_state_inspection", + "janitor_target_connection", + "janitor_warehouse_inspection", + "managed_source_load", + "project_compile", + "project_discovery", + "python_asset", + "python_check", + "python_hook", + "python_materialization", + "python_task", + "relation_promotion", + "retention_application", + "retention_inspection", + "runtime_schema_inspection", + "scenario_capture", + "scenario_cleanup", + "scenario_execution", + "scenario_relation_read", + "scenario_schema_inspection", + "scenario_snapshot_serialization", + "scenario_snapshot_write", + "scenario_target_connection", + "schema_synchronization", + "source_freshness_metadata_observation", + "source_freshness_publication", + "source_freshness_query_observation", + "sql_hook", + "sql_test_assertion", + "sql_test_planning", + "sql_test_setup", + "staging_creation", + "table_type_conversion", + "table_type_inspection", +]; +pub const OPERATION_METADATA_FIELDS: [&str; 9] = [ + "added_count", + "altered_count", + "attempt_number", + "byte_count", + "changed_count", + "item_count", + "removed_count", + "row_count", + "unknown_count", +]; +pub const OPERATION_STRATEGIES: [&str; 10] = [ + "adapter", + "append_new_columns", + "atomic_replace", + "atomic_swap", + "build_aside", + "column", + "create_new", + "rename", + "sql", + "sync_all_columns", +]; +pub const OPERATION_ADAPTERS: [&str; 8] = [ + "bigquery", + "custom", + "databricks", + "duckdb", + "motherduck", + "postgres", + "snowflake", + "sqlserver", +]; + +/// The schema version 1 catalogue in declaration order. +pub(crate) const LIFECYCLE_EVENT_CATALOG_V1: [CatalogEntry; 20] = [ + ("invocation_started", &[], &[], &["command"], false), + ( + "invocation_completed", + &[], + &[], + &["command", "duration_ms", "exit_code"], + true, + ), + ( + "invocation_failed", + &[], + &[], + &[ + "command", + "duration_ms", + "error_code", + "error_type", + "exit_code", + ], + true, + ), + ( + "run_started", + &["run_id"], + &[], + &[ + "configured_concurrency", + "run_kind", + "selected_count", + "worker_count", + ], + false, + ), + ( + "run_completed", + &["run_id"], + &[], + &[ + "duration_ms", + "fail_count", + "failed_count", + "pass_count", + "run_kind", + "skipped_count", + "succeeded_count", + "warn_count", + ], + true, + ), + ( + "run_failed", + &["run_id"], + &[], + &[ + "duration_ms", + "error_code", + "error_type", + "fail_count", + "failed_count", + "pass_count", + "run_kind", + "skipped_count", + "succeeded_count", + "warn_count", + ], + true, + ), + ( + "resource_attempt_started", + &["resource_attempt_id", "resource_id", "run_id"], + &[], + &["attempt_number", "resource_kind", "resource_name"], + false, + ), + ( + "resource_attempt_completed", + &["resource_attempt_id", "resource_id", "run_id"], + &[], + &[ + "attempt_number", + "duration_ms", + "resource_kind", + "resource_name", + ], + true, + ), + ( + "resource_attempt_failed", + &["resource_attempt_id", "resource_id", "run_id"], + &[], + &[ + "attempt_number", + "duration_ms", + "error_code", + "error_type", + "resource_kind", + "resource_name", + ], + true, + ), + ( + "resource_attempt_skipped", + &["resource_attempt_id", "resource_id", "run_id"], + &[ + "attempt_number", + "resource_kind", + "resource_name", + "skip_code", + ], + &[ + "attempt_number", + "duration_ms", + "resource_kind", + "resource_name", + "skip_code", + "skip_mode", + ], + true, + ), + ( + "operation_started", + &["operation_id"], + &[], + &[ + "adapter", + "hook_index", + "hook_name", + "hook_phase", + "hook_type", + "metadata", + "operation_kind", + "operation_name", + "phase", + "scope", + "strategy", + "target_kind", + ], + false, + ), + ( + "operation_completed", + &["operation_id"], + &[], + &[ + "adapter", + "duration_ms", + "exit_code", + "hook_index", + "hook_name", + "hook_phase", + "hook_type", + "metadata", + "operation_kind", + "operation_name", + "phase", + "process_id", + "scope", + "signal_number", + "strategy", + "target_kind", + ], + true, + ), + ( + "operation_failed", + &["operation_id"], + &[], + &[ + "adapter", + "duration_ms", + "error_code", + "error_type", + "exit_code", + "hook_index", + "hook_name", + "hook_phase", + "hook_type", + "metadata", + "operation_kind", + "operation_name", + "phase", + "process_id", + "scope", + "signal_number", + "strategy", + "target_kind", + ], + true, + ), + ( + "retry_scheduled", + &["resource_attempt_id", "resource_id", "run_id"], + &[ + "delay_ms", + "error_type", + "failed_attempt_number", + "next_attempt_number", + ], + &[ + "delay_ms", + "error_code", + "error_type", + "failed_attempt_number", + "next_attempt_number", + ], + false, + ), + ( + "audit_completed", + &["run_id"], + &[ + "attachment_kind", + "audit_definition_name", + "audit_name", + "evaluation_mode", + "outcome", + "run_scope_phase", + "severity", + ], + &[ + "attached_column_name", + "attached_target_kind", + "attached_target_name", + "attachment_kind", + "audit_definition_name", + "audit_description", + "audit_name", + "binding_key", + "definition_fingerprint", + "evaluation_mode", + "evidence", + "evidence_count", + "evidence_error", + "evidence_sql", + "evidence_truncated", + "executed_sql", + "execution_error", + "execution_fingerprint", + "measured_value", + "measurement_sql", + "minimum_samples", + "outcome", + "result_id", + "reused", + "run_scope_phase", + "sample_count", + "sample_unit", + "severity", + "sql_digest", + "target_database", + "target_name", + "target_schema", + "thresholds", + "violation_count", + ], + true, + ), + ( + "statement_started", + &["statement_id"], + &[], + &[ + "adapter", + "affected_rows", + "batch_size", + "intent", + "job_id", + "metadata", + "query_id", + "row_count", + "sql_digest", + "statement_kind", + ], + false, + ), + ( + "statement_submitted", + &["statement_id"], + &[], + &[ + "adapter", + "affected_rows", + "batch_size", + "intent", + "job_id", + "metadata", + "query_id", + "sql_digest", + "statement_kind", + ], + false, + ), + ( + "statement_heartbeat", + &["statement_id"], + &[], + &[ + "adapter", + "affected_rows", + "batch_size", + "duration_ms", + "intent", + "job_id", + "metadata", + "query_id", + "row_count", + "sql_digest", + "statement_kind", + ], + false, + ), + ( + "statement_completed", + &["statement_id"], + &[], + &[ + "adapter", + "affected_rows", + "batch_size", + "duration_ms", + "intent", + "job_id", + "metadata", + "query_id", + "row_count", + "sql_digest", + "statement_kind", + ], + true, + ), + ( + "statement_failed", + &["statement_id"], + &[], + &[ + "adapter", + "affected_rows", + "batch_size", + "duration_ms", + "error_code", + "error_type", + "intent", + "job_id", + "metadata", + "query_id", + "row_count", + "sql_digest", + "statement_kind", + ], + true, + ), +]; +/// The schema version 2 catalogue in declaration order. +pub(crate) const LIFECYCLE_EVENT_CATALOG_V2: [CatalogEntry; 23] = [ + ("invocation_started", &[], &[], &["command"], false), + ( + "invocation_completed", + &[], + &[], + &["command", "duration_ms", "exit_code"], + true, + ), + ( + "invocation_failed", + &[], + &[], + &[ + "command", + "duration_ms", + "error_code", + "error_type", + "exit_code", + ], + true, + ), + ( + "run_started", + &["run_id"], + &[], + &[ + "configured_concurrency", + "run_kind", + "selected_count", + "worker_count", + ], + false, + ), + ( + "run_completed", + &["run_id"], + &[], + &[ + "duration_ms", + "fail_count", + "failed_count", + "pass_count", + "run_kind", + "skipped_count", + "succeeded_count", + "warn_count", + ], + true, + ), + ( + "run_failed", + &["run_id"], + &[], + &[ + "duration_ms", + "error_code", + "error_type", + "fail_count", + "failed_count", + "pass_count", + "run_kind", + "skipped_count", + "succeeded_count", + "warn_count", + ], + true, + ), + ( + "resource_attempt_started", + &["resource_attempt_id", "resource_id", "run_id"], + &[], + &["attempt_number", "resource_kind", "resource_name"], + false, + ), + ( + "resource_attempt_completed", + &["resource_attempt_id", "resource_id", "run_id"], + &[], + &[ + "attempt_number", + "duration_ms", + "resource_kind", + "resource_name", + ], + true, + ), + ( + "resource_attempt_failed", + &["resource_attempt_id", "resource_id", "run_id"], + &[], + &[ + "attempt_number", + "duration_ms", + "error_code", + "error_type", + "resource_kind", + "resource_name", + ], + true, + ), + ( + "resource_attempt_skipped", + &["resource_attempt_id", "resource_id", "run_id"], + &[ + "attempt_number", + "resource_kind", + "resource_name", + "skip_code", + ], + &[ + "attempt_number", + "duration_ms", + "resource_kind", + "resource_name", + "skip_code", + "skip_mode", + ], + true, + ), + ( + "operation_started", + &["operation_id"], + &[], + &[ + "adapter", + "hook_index", + "hook_name", + "hook_phase", + "hook_type", + "metadata", + "operation_kind", + "operation_name", + "phase", + "scope", + "strategy", + "target_kind", + ], + false, + ), + ( + "operation_completed", + &["operation_id"], + &[], + &[ + "adapter", + "duration_ms", + "exit_code", + "hook_index", + "hook_name", + "hook_phase", + "hook_type", + "metadata", + "operation_kind", + "operation_name", + "phase", + "process_id", + "scope", + "signal_number", + "strategy", + "target_kind", + ], + true, + ), + ( + "operation_failed", + &["operation_id"], + &[], + &[ + "adapter", + "duration_ms", + "error_code", + "error_type", + "exit_code", + "hook_index", + "hook_name", + "hook_phase", + "hook_type", + "metadata", + "operation_kind", + "operation_name", + "phase", + "process_id", + "scope", + "signal_number", + "strategy", + "target_kind", + ], + true, + ), + ( + "retry_scheduled", + &["resource_attempt_id", "resource_id", "run_id"], + &[ + "delay_ms", + "error_type", + "failed_attempt_number", + "next_attempt_number", + ], + &[ + "delay_ms", + "error_code", + "error_type", + "failed_attempt_number", + "next_attempt_number", + ], + false, + ), + ( + "audit_completed", + &["run_id"], + &[ + "attachment_kind", + "audit_definition_name", + "audit_name", + "evaluation_mode", + "outcome", + "run_scope_phase", + "severity", + ], + &[ + "attached_column_name", + "attached_target_kind", + "attached_target_name", + "attachment_kind", + "audit_definition_name", + "audit_description", + "audit_name", + "binding_key", + "definition_fingerprint", + "evaluation_mode", + "evidence", + "evidence_count", + "evidence_error", + "evidence_sql", + "evidence_truncated", + "executed_sql", + "execution_error", + "execution_fingerprint", + "measured_value", + "measurement_sql", + "minimum_samples", + "outcome", + "result_id", + "reused", + "run_scope_phase", + "sample_count", + "sample_unit", + "severity", + "sql_digest", + "target_database", + "target_name", + "target_schema", + "thresholds", + "violation_count", + ], + true, + ), + ( + "statement_started", + &["statement_id"], + &[], + &[ + "adapter", + "affected_rows", + "batch_size", + "intent", + "job_id", + "metadata", + "query_id", + "row_count", + "sql_digest", + "statement_kind", + ], + false, + ), + ( + "statement_submitted", + &["statement_id"], + &[], + &[ + "adapter", + "affected_rows", + "batch_size", + "intent", + "job_id", + "metadata", + "query_id", + "sql_digest", + "statement_kind", + ], + false, + ), + ( + "statement_heartbeat", + &["statement_id"], + &[], + &[ + "adapter", + "affected_rows", + "batch_size", + "duration_ms", + "intent", + "job_id", + "metadata", + "query_id", + "row_count", + "sql_digest", + "statement_kind", + ], + false, + ), + ( + "statement_completed", + &["statement_id"], + &[], + &[ + "adapter", + "affected_rows", + "batch_size", + "duration_ms", + "intent", + "job_id", + "metadata", + "query_id", + "row_count", + "sql_digest", + "statement_kind", + ], + true, + ), + ( + "statement_failed", + &["statement_id"], + &[], + &[ + "adapter", + "affected_rows", + "batch_size", + "duration_ms", + "error_code", + "error_type", + "intent", + "job_id", + "metadata", + "query_id", + "row_count", + "sql_digest", + "statement_kind", + ], + true, + ), + ( + "microbatch_started", + &["resource_attempt_id", "resource_id", "run_id"], + &[ + "batch_count", + "batch_index", + "batch_kind", + "configured_batch_size", + "cursor_end_exclusive", + "cursor_start", + "effective_batch_size", + "microbatch_run_type", + "microbatch_strategy", + "planned_cursor_end_exclusive", + "planned_cursor_start", + ], + &[ + "batch_count", + "batch_index", + "batch_kind", + "configured_batch_size", + "cursor_end_exclusive", + "cursor_start", + "effective_batch_size", + "microbatch_run_type", + "microbatch_strategy", + "planned_cursor_end_exclusive", + "planned_cursor_start", + ], + false, + ), + ( + "microbatch_completed", + &["resource_attempt_id", "resource_id", "run_id"], + &[ + "batch_count", + "batch_index", + "batch_kind", + "configured_batch_size", + "cursor_end_exclusive", + "cursor_start", + "effective_batch_size", + "microbatch_run_type", + "microbatch_strategy", + "planned_cursor_end_exclusive", + "planned_cursor_start", + ], + &[ + "affected_rows", + "batch_count", + "batch_index", + "batch_kind", + "configured_batch_size", + "cursor_end_exclusive", + "cursor_start", + "duration_ms", + "effective_batch_size", + "microbatch_run_type", + "microbatch_strategy", + "planned_cursor_end_exclusive", + "planned_cursor_start", + ], + true, + ), + ( + "microbatch_failed", + &["resource_attempt_id", "resource_id", "run_id"], + &[ + "batch_count", + "batch_index", + "batch_kind", + "configured_batch_size", + "cursor_end_exclusive", + "cursor_start", + "effective_batch_size", + "error_type", + "microbatch_run_type", + "microbatch_strategy", + "planned_cursor_end_exclusive", + "planned_cursor_start", + ], + &[ + "batch_count", + "batch_index", + "batch_kind", + "configured_batch_size", + "cursor_end_exclusive", + "cursor_start", + "duration_ms", + "effective_batch_size", + "error_code", + "error_type", + "microbatch_run_type", + "microbatch_strategy", + "planned_cursor_end_exclusive", + "planned_cursor_start", + ], + true, + ), +]; diff --git a/crates/sqlbuild-core/src/lifecycle_events/main/catalog.rs b/crates/sqlbuild-core/src/lifecycle_events/main/catalog.rs new file mode 100644 index 0000000000..74d374d42a --- /dev/null +++ b/crates/sqlbuild-core/src/lifecycle_events/main/catalog.rs @@ -0,0 +1,27 @@ +//! `catalog`: the catalogued event definitions of one schema version, in declaration order. + +use crate::lifecycle_events::constants::{LIFECYCLE_EVENT_CATALOG_V1, LIFECYCLE_EVENT_CATALOG_V2}; +use crate::lifecycle_events::models::EventDefinition; +use crate::lifecycle_events::types::CatalogEntry; + +fn definition(entry: &CatalogEntry) -> EventDefinition { + let (event_type, required_correlations, required_payload, allowed_payload, terminal) = *entry; + EventDefinition { + event_type, + required_correlations, + required_payload, + allowed_payload, + terminal, + } +} + +/// Every definition of `schema_version`, or `None` for an uncatalogued version. +#[must_use] +pub fn catalog(schema_version: i64) -> Option> { + let entries: &[CatalogEntry] = match schema_version { + 1 => &LIFECYCLE_EVENT_CATALOG_V1, + 2 => &LIFECYCLE_EVENT_CATALOG_V2, + _ => return None, + }; + Some(entries.iter().map(definition).collect()) +} diff --git a/crates/sqlbuild-core/src/lifecycle_events/main/mod.rs b/crates/sqlbuild-core/src/lifecycle_events/main/mod.rs new file mode 100644 index 0000000000..e199b26b01 --- /dev/null +++ b/crates/sqlbuild-core/src/lifecycle_events/main/mod.rs @@ -0,0 +1,2 @@ +pub mod catalog; +pub mod validate_known_event; diff --git a/crates/sqlbuild-core/src/lifecycle_events/main/validate_known_event.rs b/crates/sqlbuild-core/src/lifecycle_events/main/validate_known_event.rs new file mode 100644 index 0000000000..311a971fd0 --- /dev/null +++ b/crates/sqlbuild-core/src/lifecycle_events/main/validate_known_event.rs @@ -0,0 +1,60 @@ +//! `validate_known_event`: correlations and safe payload fields of one catalogued event. + +use std::collections::BTreeSet; + +use crate::lifecycle_events::_helpers::envelope::{correlation_set, definition}; +use crate::lifecycle_events::_helpers::payloads::family_problem; +use crate::lifecycle_events::models::KnownEventFacts; +use crate::text::main::python_str_repr::python_str_repr; + +/// The first contract violation of a catalogued event, as its validation message. +pub fn validate_known_event(event: &KnownEventFacts) -> Result<(), String> { + let quoted: String = python_str_repr(&event.event_type); + let Some(definition) = definition(event) else { + return Err(format!( + "unknown lifecycle event_type {quoted}; decode it as an opaque envelope" + )); + }; + let missing: Vec<&str> = definition + .required_correlations + .iter() + .copied() + .filter(|name| !correlation_set(event, name)) + .collect(); + if !missing.is_empty() { + return Err(format!( + "known event {quoted} requires correlation field(s): {}", + missing.join(", ") + )); + } + let keys: BTreeSet<&str> = event.payload.iter().map(|(key, _)| key.as_str()).collect(); + let missing_payload: Vec<&str> = definition + .required_payload + .iter() + .copied() + .filter(|name| !keys.contains(name)) + .collect(); + if !missing_payload.is_empty() { + return Err(format!( + "known event {quoted} requires payload field(s): {}", + missing_payload.join(", ") + )); + } + let unexpected: Vec<&str> = keys + .iter() + .copied() + .filter(|key| !definition.allowed_payload.contains(key)) + .collect(); + if !unexpected.is_empty() { + let allowed: String = if definition.allowed_payload.is_empty() { + "none".to_owned() + } else { + definition.allowed_payload.join(", ") + }; + return Err(format!( + "payload field(s) not allowed for known event {quoted}: {}; allowed fields: {allowed}", + unexpected.join(", ") + )); + } + family_problem(event).map_or(Ok(()), Err) +} diff --git a/crates/sqlbuild-core/src/lifecycle_events/mod.rs b/crates/sqlbuild-core/src/lifecycle_events/mod.rs new file mode 100644 index 0000000000..5853224abe --- /dev/null +++ b/crates/sqlbuild-core/src/lifecycle_events/mod.rs @@ -0,0 +1,8 @@ +pub(crate) mod _helpers; +pub mod constants; +pub mod main; +pub mod models; +pub(crate) mod types; + +#[cfg(test)] +mod tests; diff --git a/crates/sqlbuild-core/src/lifecycle_events/models.rs b/crates/sqlbuild-core/src/lifecycle_events/models.rs new file mode 100644 index 0000000000..c39367f126 --- /dev/null +++ b/crates/sqlbuild-core/src/lifecycle_events/models.rs @@ -0,0 +1,37 @@ +//! Lifecycle event definitions and the JSON payload values validation reads. + +/// The validation contract of one catalogued lifecycle event type. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub struct EventDefinition { + pub event_type: &'static str, + pub required_correlations: &'static [&'static str], + pub required_payload: &'static [&'static str], + pub allowed_payload: &'static [&'static str], + pub terminal: bool, +} + +/// One frozen JSON payload value. +#[derive(Clone, Debug, PartialEq)] +pub enum PayloadValue { + Null, + Bool(bool), + /// An integer, saturated into `i128`, with its exact decimal text. + Integer { + value: i128, + text: String, + }, + Float(f64), + Text(String), + Array(Vec), + Object(Vec<(String, PayloadValue)>), +} + +/// The envelope facts of one lifecycle event that catalogue validation reads. +#[derive(Clone, Debug, PartialEq)] +pub struct KnownEventFacts { + pub schema_version: i64, + pub event_type: String, + /// Each correlation field name with whether it is set. + pub correlations: Vec<(&'static str, bool)>, + pub payload: Vec<(String, PayloadValue)>, +} diff --git a/crates/sqlbuild-core/src/lifecycle_events/tests.rs b/crates/sqlbuild-core/src/lifecycle_events/tests.rs new file mode 100644 index 0000000000..e1b877d89c --- /dev/null +++ b/crates/sqlbuild-core/src/lifecycle_events/tests.rs @@ -0,0 +1,6 @@ +#[path = "tests/helpers.rs"] +mod helpers; +#[path = "tests/test_types.rs"] +mod test_types; +#[path = "tests/test_validate.rs"] +mod validate; diff --git a/crates/sqlbuild-core/src/lifecycle_events/tests/helpers.rs b/crates/sqlbuild-core/src/lifecycle_events/tests/helpers.rs new file mode 100644 index 0000000000..7945ca3222 --- /dev/null +++ b/crates/sqlbuild-core/src/lifecycle_events/tests/helpers.rs @@ -0,0 +1,33 @@ +use crate::lifecycle_events::models::{KnownEventFacts, PayloadValue}; + +/// An integer payload value. +pub(crate) fn int(value: i128) -> PayloadValue { + PayloadValue::Integer { + value, + text: value.to_string(), + } +} + +/// A text payload value. +pub(crate) fn text(value: &str) -> PayloadValue { + PayloadValue::Text(value.to_owned()) +} + +/// A version 2 event of `event_type` with every correlation set. +pub(crate) fn event(event_type: &str, payload: Vec<(&str, PayloadValue)>) -> KnownEventFacts { + KnownEventFacts { + schema_version: 2, + event_type: event_type.to_owned(), + correlations: vec![ + ("run_id", true), + ("resource_id", true), + ("resource_attempt_id", true), + ("operation_id", true), + ("statement_id", true), + ], + payload: payload + .into_iter() + .map(|(key, value)| (key.to_owned(), value)) + .collect(), + } +} diff --git a/crates/sqlbuild-core/src/lifecycle_events/tests/test_types.rs b/crates/sqlbuild-core/src/lifecycle_events/tests/test_types.rs new file mode 100644 index 0000000000..34bc5cecc4 --- /dev/null +++ b/crates/sqlbuild-core/src/lifecycle_events/tests/test_types.rs @@ -0,0 +1,7 @@ +use crate::lifecycle_events::models::KnownEventFacts; + +pub(crate) struct ValidateTestCase { + pub(crate) description: &'static str, + pub(crate) event: KnownEventFacts, + pub(crate) expected_result: Result<(), &'static str>, +} diff --git a/crates/sqlbuild-core/src/lifecycle_events/tests/test_validate.rs b/crates/sqlbuild-core/src/lifecycle_events/tests/test_validate.rs new file mode 100644 index 0000000000..8d98a1c185 --- /dev/null +++ b/crates/sqlbuild-core/src/lifecycle_events/tests/test_validate.rs @@ -0,0 +1,62 @@ +use crate::lifecycle_events::main::validate_known_event::validate_known_event; +use crate::lifecycle_events::tests::helpers::{event, int, text}; +use crate::lifecycle_events::tests::test_types::ValidateTestCase; + +#[test] +fn given_lifecycle_event_when_validating_then_first_contract_violation_is_reported() { + let test_cases = [ + ValidateTestCase { + description: "unknown type", + event: event("custom", Vec::new()), + expected_result: Err( + "unknown lifecycle event_type 'custom'; decode it as an opaque envelope", + ), + }, + ValidateTestCase { + description: "unexpected payload lists allowed fields", + event: event("invocation_started", vec![("x", int(1))]), + expected_result: Err( + "payload field(s) not allowed for known event 'invocation_started': x; allowed fields: command", + ), + }, + ValidateTestCase { + description: "retry numbering", + event: event( + "retry_scheduled", + vec![ + ("failed_attempt_number", int(1)), + ("next_attempt_number", int(3)), + ("delay_ms", int(5)), + ("error_type", text("E")), + ], + ), + expected_result: Err("next_attempt_number must equal failed_attempt_number + 1"), + }, + ValidateTestCase { + description: "uncatalogued operation adapter", + event: event("operation_started", vec![("adapter", text("other"))]), + expected_result: Err("adapter must be a catalogued value"), + }, + ValidateTestCase { + description: "valid run start", + event: event( + "run_started", + vec![ + ("configured_concurrency", int(4)), + ("selected_count", int(2)), + ("worker_count", int(2)), + ], + ), + expected_result: Ok(()), + }, + ]; + + for test_case in test_cases { + assert_eq!( + validate_known_event(&test_case.event), + test_case.expected_result.map_err(str::to_owned), + "{}", + test_case.description + ); + } +} diff --git a/crates/sqlbuild-core/src/lifecycle_events/types.rs b/crates/sqlbuild-core/src/lifecycle_events/types.rs new file mode 100644 index 0000000000..3fcd3dd26c --- /dev/null +++ b/crates/sqlbuild-core/src/lifecycle_events/types.rs @@ -0,0 +1,10 @@ +//! The shape of one catalogue entry. + +/// One catalogued event: type, required correlations, required and allowed payload, terminal. +pub(crate) type CatalogEntry = ( + &'static str, + &'static [&'static str], + &'static [&'static str], + &'static [&'static str], + bool, +); diff --git a/crates/sqlbuild-python/src/bindings/_helpers/diagnostics/mod.rs b/crates/sqlbuild-python/src/bindings/_helpers/diagnostics/mod.rs index 2dd94ad479..6570df35c8 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/diagnostics/mod.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/diagnostics/mod.rs @@ -1,3 +1,4 @@ pub(crate) mod console_records; pub(crate) mod redaction; +pub(crate) mod registration; pub(crate) mod trackers; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/diagnostics/registration.rs b/crates/sqlbuild-python/src/bindings/_helpers/diagnostics/registration.rs new file mode 100644 index 0000000000..a0e1da2c15 --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/diagnostics/registration.rs @@ -0,0 +1,11 @@ +//! Register diagnostics console formatting, redaction and process trackers. + +use pyo3::prelude::{Bound, PyModule, PyResult}; + +use crate::bindings::_helpers::diagnostics::{console_records, redaction, trackers}; + +pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { + console_records::register(module)?; + redaction::register(module)?; + trackers::register(module) +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/event_exporting/constants.rs b/crates/sqlbuild-python/src/bindings/_helpers/event_exporting/constants.rs index 05c527f2fc..cd153535c8 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/event_exporting/constants.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/event_exporting/constants.rs @@ -2,7 +2,6 @@ pub(crate) const EXCEPTIONS_MODULE: &str = "sqlbuild.runtime.event_exporting.exceptions"; pub(crate) const SEVERITY_MODULE: &str = "sqlbuild._native"; -pub(crate) const CATALOG_MODULE: &str = "sqlbuild.runtime.observability.constants"; pub(crate) const INVOKE_LIFECYCLE_MODULE: &str = "sqlbuild.runtime.event_exporting.main._invoke_lifecycle_event_sink"; pub(crate) const RUN_OUTPUT_SINKS_MODULE: &str = diff --git a/crates/sqlbuild-python/src/bindings/_helpers/event_exporting/policy.rs b/crates/sqlbuild-python/src/bindings/_helpers/event_exporting/policy.rs index c5cf0b9ffd..8641e01e07 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/event_exporting/policy.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/event_exporting/policy.rs @@ -1,32 +1,28 @@ //! Lifecycle export policy, severity comparison and narrowing for Python callers. use pyo3::prelude::{ - Bound, PyAny, PyAnyMethods, PyModule, PyModuleMethods, PyResult, Python, pyfunction, - wrap_pyfunction, + Bound, PyAny, PyModule, PyModuleMethods, PyResult, Python, pyfunction, wrap_pyfunction, }; use pyo3::types::{PyDict, PyDictMethods}; use sqlbuild_core::event_export::main::severity_at_least::severity_at_least; use sqlbuild_core::event_export::main::stricter_severity::stricter_severity; use sqlbuild_core::event_export::models::{ExportPolicy, ExportSeverity}; +use sqlbuild_core::lifecycle_events::constants::CURRENT_LIFECYCLE_EVENT_SCHEMA_VERSION; +use sqlbuild_core::lifecycle_events::main::catalog::catalog; use crate::bindings::_helpers::event_exporting::values::{ export_policy, severity_from, severity_object, }; -const CATALOG_MODULE: &str = "sqlbuild.runtime.observability.constants"; - /// `{event_type: (kind, severity, priority)}` for every catalogued lifecycle event, in catalogue order. #[pyfunction] fn lifecycle_export_policy_catalog(py: Python<'_>) -> PyResult> { - let catalog: Bound<'_, PyAny> = py - .import(CATALOG_MODULE)? - .getattr("LIFECYCLE_EVENT_CATALOG")?; let policies: Bound<'_, PyDict> = PyDict::new(py); - for event_type in catalog.try_iter()? { - let event_type: String = event_type?.extract()?; - let policy: ExportPolicy = export_policy(py, &event_type)?; + for definition in catalog(CURRENT_LIFECYCLE_EVENT_SCHEMA_VERSION).unwrap_or_default() { + let event_type: &str = definition.event_type; + let policy: ExportPolicy = export_policy(py, event_type)?; policies.set_item( - &event_type, + event_type, ( policy.kind, severity_object(py, policy.severity)?, diff --git a/crates/sqlbuild-python/src/bindings/_helpers/event_exporting/values.rs b/crates/sqlbuild-python/src/bindings/_helpers/event_exporting/values.rs index cd2ee51981..c9a7c33915 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/event_exporting/values.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/event_exporting/values.rs @@ -9,11 +9,11 @@ use pyo3::types::{PyDict, PyDictMethods, PyString, PyTuple, PyTupleMethods}; use sqlbuild_core::event_export::main::lifecycle_export_policy::lifecycle_export_policy; use sqlbuild_core::event_export::main::sanitized_exception_type::sanitized_exception_type; use sqlbuild_core::event_export::models::{ExportPolicy, ExportSeverity, ExporterCounts}; +use sqlbuild_core::lifecycle_events::constants::CURRENT_LIFECYCLE_EVENT_SCHEMA_VERSION; +use sqlbuild_core::lifecycle_events::main::catalog::catalog; use crate::bindings::_helpers::compile_driver::python_calls::imported; -use crate::bindings::_helpers::event_exporting::constants::{ - CATALOG_MODULE, EXCEPTIONS_MODULE, SEVERITY_MODULE, -}; +use crate::bindings::_helpers::event_exporting::constants::{EXCEPTIONS_MODULE, SEVERITY_MODULE}; /// `BoundEventExporter` parameters, in positional order. const BOUND_EXPORTER_PARAMETERS: [&str; 5] = [ @@ -74,15 +74,13 @@ pub(crate) fn error_type_of(py: Python<'_>, error: &PyErr) -> String { /// Export dimensions of every catalogued lifecycle event type, derived once per interpreter. pub(crate) fn export_policies(py: Python<'_>) -> PyResult<&HashMap> { POLICIES.get_or_try_init(py, || { - let catalog: Bound<'_, PyAny> = imported(py, CATALOG_MODULE, "LIFECYCLE_EVENT_CATALOG")?; let mut policies: HashMap = HashMap::new(); - for entry in catalog.call_method0("items")?.try_iter()? { - let (event_type, definition): (String, Bound<'_, PyAny>) = entry?.extract()?; - let terminal: bool = definition.getattr("terminal")?.is_truthy()?; - let policy: ExportPolicy = lifecycle_export_policy(&event_type, terminal); - policies.insert(event_type, policy); + for definition in catalog(CURRENT_LIFECYCLE_EVENT_SCHEMA_VERSION).unwrap_or_default() { + let policy: ExportPolicy = + lifecycle_export_policy(definition.event_type, definition.terminal); + policies.insert(definition.event_type.to_owned(), policy); } - Ok(policies) + Ok::<_, pyo3::PyErr>(policies) }) } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/lifecycle_events/catalog.rs b/crates/sqlbuild-python/src/bindings/_helpers/lifecycle_events/catalog.rs new file mode 100644 index 0000000000..b31699fda1 --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/lifecycle_events/catalog.rs @@ -0,0 +1,83 @@ +//! The native lifecycle event catalogue: known types, terminality and catalogued values. + +use pyo3::exceptions::PyKeyError; +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyModule, PyModuleMethods, PyResult}; +use pyo3::types::{PyFrozenSet, PyString}; +use pyo3::{pyfunction, wrap_pyfunction}; +use sqlbuild_core::lifecycle_events::constants::{ + CURRENT_LIFECYCLE_EVENT_SCHEMA_VERSION, HOOK_PHASES, HOOK_TYPES, + LIFECYCLE_EVENT_SCHEMA_VERSIONS, OPERATION_ADAPTERS, OPERATION_KINDS, + OPERATION_METADATA_FIELDS, OPERATION_NAMES, OPERATION_PHASES, OPERATION_SCOPES, + OPERATION_STRATEGIES, OPERATION_TARGET_KINDS, RESOURCE_SKIP_CODES, RESOURCE_SKIP_MODES, +}; +use sqlbuild_core::lifecycle_events::main::catalog::catalog; +use sqlbuild_core::lifecycle_events::models::EventDefinition; + +/// The definition of `event_type` in the `schema_version` catalogue, if both are catalogued. +pub(crate) fn definition( + schema_version: &Bound<'_, PyAny>, + event_type: &Bound<'_, PyAny>, +) -> PyResult> { + let (Ok(version), Ok(name)) = ( + schema_version.extract::(), + event_type.downcast::(), + ) else { + return Ok(None); + }; + let name: String = name.extract()?; + Ok(catalog(version) + .and_then(|entries| entries.into_iter().find(|entry| entry.event_type == name))) +} + +/// Whether `event_type` is catalogued for `schema_version`. +#[pyfunction] +#[pyo3(signature = (*, schema_version, event_type))] +fn lifecycle_event_catalogued( + schema_version: &Bound<'_, PyAny>, + event_type: &Bound<'_, PyAny>, +) -> PyResult { + Ok(definition(schema_version, event_type)?.is_some()) +} + +/// Whether a catalogued event closes its correlated lifecycle scope; `KeyError` otherwise. +#[pyfunction] +#[pyo3(signature = (*, schema_version, event_type))] +fn lifecycle_event_terminal( + schema_version: &Bound<'_, PyAny>, + event_type: &Bound<'_, PyAny>, +) -> PyResult { + definition(schema_version, event_type)? + .map(|entry| entry.terminal) + .ok_or_else(|| PyKeyError::new_err(event_type.clone().unbind())) +} + +pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { + let py = module.py(); + module.add( + "CURRENT_LIFECYCLE_EVENT_SCHEMA_VERSION", + CURRENT_LIFECYCLE_EVENT_SCHEMA_VERSION, + )?; + module.add( + "LIFECYCLE_EVENT_SCHEMA_VERSIONS", + PyFrozenSet::new(py, LIFECYCLE_EVENT_SCHEMA_VERSIONS)?, + )?; + let sets: [(&str, &[&str]); 12] = [ + ("HOOK_PHASES", &HOOK_PHASES), + ("HOOK_TYPES", &HOOK_TYPES), + ("OPERATION_ADAPTERS", &OPERATION_ADAPTERS), + ("OPERATION_KINDS", &OPERATION_KINDS), + ("OPERATION_METADATA_FIELDS", &OPERATION_METADATA_FIELDS), + ("OPERATION_NAMES", &OPERATION_NAMES), + ("OPERATION_PHASES", &OPERATION_PHASES), + ("OPERATION_SCOPES", &OPERATION_SCOPES), + ("OPERATION_STRATEGIES", &OPERATION_STRATEGIES), + ("OPERATION_TARGET_KINDS", &OPERATION_TARGET_KINDS), + ("RESOURCE_SKIP_CODES", &RESOURCE_SKIP_CODES), + ("RESOURCE_SKIP_MODES", &RESOURCE_SKIP_MODES), + ]; + for (name, values) in sets { + module.add(name, PyFrozenSet::new(py, values)?)?; + } + module.add_function(wrap_pyfunction!(lifecycle_event_catalogued, module)?)?; + module.add_function(wrap_pyfunction!(lifecycle_event_terminal, module)?) +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/lifecycle_events/mod.rs b/crates/sqlbuild-python/src/bindings/_helpers/lifecycle_events/mod.rs new file mode 100644 index 0000000000..00d4e7017d --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/lifecycle_events/mod.rs @@ -0,0 +1,4 @@ +pub(crate) mod catalog; +pub(crate) mod payloads; +pub(crate) mod registration; +pub(crate) mod validation; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/lifecycle_events/payloads.rs b/crates/sqlbuild-python/src/bindings/_helpers/lifecycle_events/payloads.rs new file mode 100644 index 0000000000..17c0d9c764 --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/lifecycle_events/payloads.rs @@ -0,0 +1,157 @@ +//! Frozen JSON values read from Python into payload values, and JSON values frozen in place. + +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyErr, PyResult, Python}; +use pyo3::types::{PyBool, PyDict, PyDictMethods, PyFloat, PyInt, PyList, PyString, PyTuple}; +use sqlbuild_core::lifecycle_events::models::PayloadValue; + +use crate::bindings::_helpers::compile_driver::python_calls::imported; + +const EXCEPTIONS_MODULE: &str = "sqlbuild.runtime.observability.exceptions"; + +/// `ObservabilityValidationError(message)`. +pub(crate) fn validation_error(py: Python<'_>, message: &str) -> PyErr { + match imported(py, EXCEPTIONS_MODULE, "ObservabilityValidationError") + .and_then(|class| class.call1((message,))) + { + Ok(error) => PyErr::from_value(error), + Err(error) => error, + } +} + +/// Whether `value` is a `collections.abc.Mapping`. +pub(crate) fn abc_mapping(value: &Bound<'_, PyAny>) -> PyResult { + value.is_instance(&imported(value.py(), "collections.abc", "Mapping")?) +} + +/// One segment of a JSON value path: a key or a list index. +#[derive(Clone)] +pub(crate) enum PathSegment { + Key(String), + Index(usize), +} + +/// The root name, then `.key` and `[index]` segments. +fn rendered_path(path: &[PathSegment]) -> String { + let mut text: String = String::new(); + for (position, segment) in path.iter().enumerate() { + match segment { + PathSegment::Index(index) => text.push_str(&format!("[{index}]")), + PathSegment::Key(key) if position > 0 => text.push_str(&format!(".{key}")), + PathSegment::Key(key) => text.push_str(key), + } + } + text +} + +/// `path` followed by one more segment. +fn extended(path: &[PathSegment], segment: PathSegment) -> Vec { + let mut extended: Vec = path.to_vec(); + extended.push(segment); + extended +} + +fn scalar(value: &Bound<'_, PyAny>) -> bool { + value.is_none() || value.is_instance_of::() || value.is_instance_of::() +} + +/// Validate and recursively freeze a JSON-compatible value. +pub(crate) fn frozen<'py>( + value: &Bound<'py, PyAny>, + path: &[PathSegment], +) -> PyResult> { + let py: Python<'py> = value.py(); + if scalar(value) { + return Ok(value.clone()); + } + if value.is_instance_of::() { + if !value.extract::()?.is_finite() { + return Err(validation_error( + py, + &format!("{} must not contain NaN or infinity", rendered_path(path)), + )); + } + return Ok(value.clone()); + } + if abc_mapping(value)? { + let copied: Bound<'py, PyDict> = PyDict::new(py); + for item in value.call_method0("items")?.try_iter()? { + let (key, item): (Bound<'py, PyAny>, Bound<'py, PyAny>) = item?.extract()?; + let Ok(name) = key.downcast::() else { + return Err(validation_error( + py, + &format!("{} keys must be strings", rendered_path(path)), + )); + }; + if scalar(&item) { + copied.set_item(&key, item)?; + continue; + } + let nested: Vec = extended(path, PathSegment::Key(name.extract()?)); + copied.set_item(&key, frozen(&item, &nested)?)?; + } + return imported(py, "types", "MappingProxyType")?.call1((copied,)); + } + if value.is_instance_of::() || value.is_instance_of::() { + let mut items: Vec> = Vec::new(); + for (index, item) in value.try_iter()?.enumerate() { + let nested: Vec = extended(path, PathSegment::Index(index)); + items.push(frozen(&item?, &nested)?); + } + return Ok(PyTuple::new(py, items)?.into_any()); + } + let type_name: String = value.get_type().getattr("__name__")?.extract()?; + Err(validation_error( + py, + &format!( + "{} contains non-JSON value of type {type_name}", + rendered_path(path) + ), + )) +} + +/// A frozen JSON value as a payload value. +pub(crate) fn payload_value(value: &Bound<'_, PyAny>) -> PyResult { + if value.is_none() { + return Ok(PayloadValue::Null); + } + if value.is_instance_of::() { + return Ok(PayloadValue::Bool(value.is_truthy()?)); + } + if value.is_instance_of::() { + let saturated: i128 = match value.extract::() { + Ok(number) => number, + Err(_) if value.lt(0)? => i128::MIN, + Err(_) => i128::MAX, + }; + return Ok(PayloadValue::Integer { + value: saturated, + text: value.str()?.extract()?, + }); + } + if value.is_instance_of::() { + return Ok(PayloadValue::Float(value.extract()?)); + } + if value.is_instance_of::() { + return Ok(PayloadValue::Text(value.extract()?)); + } + if abc_mapping(value)? { + return Ok(PayloadValue::Object(object_entries(value)?)); + } + let items: Vec = value + .try_iter()? + .map(|item| payload_value(&item?)) + .collect::>()?; + Ok(PayloadValue::Array(items)) +} + +/// The entries of a frozen JSON object in iteration order. +pub(crate) fn object_entries(value: &Bound<'_, PyAny>) -> PyResult> { + value + .call_method0("items")? + .try_iter()? + .map(|item| { + let (key, item): (String, Bound<'_, PyAny>) = item?.extract()?; + Ok((key, payload_value(&item)?)) + }) + .collect() +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/lifecycle_events/registration.rs b/crates/sqlbuild-python/src/bindings/_helpers/lifecycle_events/registration.rs new file mode 100644 index 0000000000..11c2a473a2 --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/lifecycle_events/registration.rs @@ -0,0 +1,10 @@ +//! Register lifecycle event catalogue queries, constants and envelope validation. + +use pyo3::prelude::{Bound, PyModule, PyResult}; + +use crate::bindings::_helpers::lifecycle_events::{catalog, validation}; + +pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { + catalog::register(module)?; + validation::register(module) +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/lifecycle_events/validation.rs b/crates/sqlbuild-python/src/bindings/_helpers/lifecycle_events/validation.rs new file mode 100644 index 0000000000..2b4a55b45c --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/lifecycle_events/validation.rs @@ -0,0 +1,112 @@ +//! Envelope field validation, JSON freezing and catalogued event validation. + +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyModule, PyModuleMethods, PyResult}; +use pyo3::types::{PyBool, PyInt, PyString}; +use pyo3::{pyfunction, wrap_pyfunction}; +use sqlbuild_core::lifecycle_events::main::validate_known_event::validate_known_event; +use sqlbuild_core::lifecycle_events::models::KnownEventFacts; + +use crate::bindings::_helpers::compile_driver::python_calls::imported; +use crate::bindings::_helpers::lifecycle_events::payloads::{ + PathSegment, frozen, object_entries, validation_error, +}; + +const CORRELATION_FIELDS: [&str; 5] = [ + "run_id", + "resource_id", + "resource_attempt_id", + "operation_id", + "statement_id", +]; + +/// Validate the shared positive integer schema-version contract. +#[pyfunction(name = "validate_lifecycle_schema_version")] +#[pyo3(signature = (*, value))] +fn schema_version_check(value: &Bound<'_, PyAny>) -> PyResult<()> { + if value.is_instance_of::() || !value.is_instance_of::() || value.le(0)? { + return Err(validation_error( + value.py(), + "schema_version must be a positive integer excluding bool", + )); + } + Ok(()) +} + +/// Validate and recursively freeze a JSON-compatible value. +#[pyfunction(name = "freeze_lifecycle_json")] +#[pyo3(signature = (*, value, path))] +fn frozen_json<'py>(value: &Bound<'py, PyAny>, path: &str) -> PyResult> { + frozen(value, &[PathSegment::Key(path.to_owned())]) +} + +/// Validate one required non-empty string field. +#[pyfunction(name = "validate_lifecycle_required_text")] +#[pyo3(signature = (*, value, field_name))] +fn required_text_check(value: &Bound<'_, PyAny>, field_name: &str) -> PyResult<()> { + let valid: bool = value + .downcast::() + .is_ok_and(|text| text.len().is_ok_and(|length| length > 0)); + if !valid { + return Err(validation_error( + value.py(), + &format!("{field_name} must be a non-empty string"), + )); + } + Ok(()) +} + +/// Validate one optional non-empty string field. +#[pyfunction(name = "validate_lifecycle_optional_text")] +#[pyo3(signature = (*, value, field_name))] +fn optional_text_check(value: &Bound<'_, PyAny>, field_name: &str) -> PyResult<()> { + if value.is_none() { + return Ok(()); + } + required_text_check(value, field_name) +} + +/// Validate a timezone-aware UTC timestamp. +#[pyfunction(name = "validate_lifecycle_timestamp")] +#[pyo3(signature = (*, value))] +fn timestamp_check(value: &Bound<'_, PyAny>) -> PyResult<()> { + let py = value.py(); + let datetime: Bound<'_, PyAny> = imported(py, "datetime", "datetime")?; + if !value.is_instance(&datetime)? || value.getattr("tzinfo")?.is_none() { + return Err(validation_error( + py, + "occurred_at must be a timezone-aware UTC datetime", + )); + } + let utc_offset: Bound<'_, PyAny> = + imported(py, "datetime", "UTC")?.call_method1("utcoffset", (value,))?; + if value.call_method0("utcoffset")?.ne(utc_offset)? { + return Err(validation_error(py, "occurred_at must use UTC")); + } + Ok(()) +} + +/// Validate correlations and safe payload fields for a catalogued event. +#[pyfunction(name = "validate_known_lifecycle_event")] +#[pyo3(signature = (*, event))] +fn known_event_check(event: &Bound<'_, PyAny>) -> PyResult<()> { + let mut correlations: Vec<(&'static str, bool)> = Vec::new(); + for field in CORRELATION_FIELDS { + correlations.push((field, !event.getattr(field)?.is_none())); + } + let facts = KnownEventFacts { + schema_version: event.getattr("schema_version")?.extract()?, + event_type: event.getattr("event_type")?.extract()?, + correlations, + payload: object_entries(&event.getattr("payload")?)?, + }; + validate_known_event(&facts).map_err(|message| validation_error(event.py(), &message)) +} + +pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { + module.add_function(wrap_pyfunction!(schema_version_check, module)?)?; + module.add_function(wrap_pyfunction!(frozen_json, module)?)?; + module.add_function(wrap_pyfunction!(required_text_check, module)?)?; + module.add_function(wrap_pyfunction!(optional_text_check, module)?)?; + module.add_function(wrap_pyfunction!(timestamp_check, module)?)?; + module.add_function(wrap_pyfunction!(known_event_check, module)?) +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/mod.rs b/crates/sqlbuild-python/src/bindings/_helpers/mod.rs index 3ba2d852ea..9b7bb46cb6 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/mod.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/mod.rs @@ -49,6 +49,7 @@ pub(crate) mod graph; pub(crate) mod hand_built; pub(crate) mod init_command; pub(crate) mod janitor; +pub(crate) mod lifecycle_events; pub(crate) mod lineage; pub(crate) mod load; pub(crate) mod macro_bridge; diff --git a/crates/sqlbuild-python/src/bindings/main/register.rs b/crates/sqlbuild-python/src/bindings/main/register.rs index dbd73bd745..d47d397c40 100644 --- a/crates/sqlbuild-python/src/bindings/main/register.rs +++ b/crates/sqlbuild-python/src/bindings/main/register.rs @@ -7,9 +7,10 @@ use crate::bindings::_helpers::{ analysis, analysis_session, attachments, auditing, authored_values, boundary, cache, cli, compile_attachment, compile_command, compile_driver, compile_outputs, compiled_project, contracts, cursor, dag_artifact, diagnostics, discovery, exported_constants, fingerprints, - frontier, graph, lineage, macro_bridge, manifest, microbatches, model_config, plan_output, - planner_registration, project_assembly, records, refactor, references, render, rules, scopes, - semantic_checks, semantic_shapes, sql_test_glue, sqltext, state_rows, type_system, + frontier, graph, lifecycle_events, lineage, macro_bridge, manifest, microbatches, model_config, + plan_output, planner_registration, project_assembly, records, refactor, references, render, + rules, scopes, semantic_checks, semantic_shapes, sql_test_glue, sqltext, state_rows, + type_system, }; use crate::bindings::models; @@ -55,9 +56,8 @@ fn register_project_stages(module: &Bound<'_, PyModule>) -> PyResult<()> { exported_constants::registration::register(module)?; state_rows::registration::register(module)?; microbatches::registration::register(module)?; - diagnostics::console_records::register(module)?; - diagnostics::redaction::register(module)?; - diagnostics::trackers::register(module)?; + lifecycle_events::registration::register(module)?; + diagnostics::registration::register(module)?; analysis::sql_test_comparisons::register(module)?; Ok(()) } diff --git a/python-boundary.toml b/python-boundary.toml index 3c45cc5aa4..45d844c679 100644 --- a/python-boundary.toml +++ b/python-boundary.toml @@ -212,7 +212,7 @@ note = "Encodes the Python fact-view values user rules read, for their digests." [[stays]] path = "runtime/observability" reason = "public-api" -note = "Documented observability API used throughout the executor." +note = "Documented observability API used throughout the executor: lifecycle emitters, dispatcher, scopes and envelope shapes. The event catalogue, payload validation and JSON freezing are native (sqlbuild-core lifecycle_events)." [[stays]] path = "runtime/execution_history/__init__.py" diff --git a/src/sqlbuild/_native.pyi b/src/sqlbuild/_native.pyi index 77803b9818..ea273cb89b 100644 --- a/src/sqlbuild/_native.pyi +++ b/src/sqlbuild/_native.pyi @@ -158,6 +158,7 @@ from sqlbuild.runtime.execution_history.types import ( ) from sqlbuild.runtime.observability.classes.event_dispatcher import EventDispatcher from sqlbuild.runtime.observability.models import DiagnosticLog, LifecycleEvent +from sqlbuild.runtime.observability.types import JSONValue from sqlbuild.runtime.output_capture.models import ( BoundCommandOutputSink, CommandOutputCaptureSummary, @@ -5080,6 +5081,30 @@ def build_microbatch_create_index_sqls( render_identifier: Callable[[str], str], render_qualified_name: Callable[..., str | None], ) -> tuple[str, ...]: ... + +CURRENT_LIFECYCLE_EVENT_SCHEMA_VERSION: int +LIFECYCLE_EVENT_SCHEMA_VERSIONS: frozenset[int] +HOOK_PHASES: frozenset[str] +HOOK_TYPES: frozenset[str] +OPERATION_ADAPTERS: frozenset[str] +OPERATION_KINDS: frozenset[str] +OPERATION_METADATA_FIELDS: frozenset[str] +OPERATION_NAMES: frozenset[str] +OPERATION_PHASES: frozenset[str] +OPERATION_SCOPES: frozenset[str] +OPERATION_STRATEGIES: frozenset[str] +OPERATION_TARGET_KINDS: frozenset[str] +RESOURCE_SKIP_CODES: frozenset[str] +RESOURCE_SKIP_MODES: frozenset[str] + +def lifecycle_event_catalogued(*, schema_version: object, event_type: object) -> bool: ... +def lifecycle_event_terminal(*, schema_version: int, event_type: str) -> bool: ... +def validate_lifecycle_schema_version(*, value: object) -> None: ... +def freeze_lifecycle_json(*, value: object, path: str) -> JSONValue: ... +def validate_lifecycle_required_text(*, value: object, field_name: str) -> None: ... +def validate_lifecycle_optional_text(*, value: object, field_name: str) -> None: ... +def validate_lifecycle_timestamp(*, value: object) -> None: ... +def validate_known_lifecycle_event(*, event: LifecycleEvent) -> None: ... def render_state_table_create_sql( *, qualified_name: str, diff --git a/src/sqlbuild/executor/run/_helpers/execution/promotion.py b/src/sqlbuild/executor/run/_helpers/execution/promotion.py index c3c20f17d8..63b72639b1 100644 --- a/src/sqlbuild/executor/run/_helpers/execution/promotion.py +++ b/src/sqlbuild/executor/run/_helpers/execution/promotion.py @@ -4,6 +4,7 @@ from typing import Any +from sqlbuild._native import OPERATION_STRATEGIES from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter from sqlbuild.adapter.contract.classes.statement_recorder import StatementRecorder from sqlbuild.adapter.contract.types import PromotionStrategy @@ -14,10 +15,7 @@ OperationAttributes, OperationLifecycle, ) -from sqlbuild.runtime.observability.constants import ( - OPERATION_STRATEGIES, - RENAME_OPERATION_STRATEGY, -) +from sqlbuild.runtime.observability.constants import RENAME_OPERATION_STRATEGY from sqlbuild.runtime.observability.main.canonicalize_operation_adapter import ( canonicalize_operation_adapter, ) diff --git a/src/sqlbuild/runtime/observability/_helpers/factory.py b/src/sqlbuild/runtime/observability/_helpers/factory.py index b1bc0e5631..ea883f9fd1 100644 --- a/src/sqlbuild/runtime/observability/_helpers/factory.py +++ b/src/sqlbuild/runtime/observability/_helpers/factory.py @@ -7,11 +7,11 @@ from types import MappingProxyType from uuid import uuid4 +from sqlbuild._native import CURRENT_LIFECYCLE_EVENT_SCHEMA_VERSION from sqlbuild.runtime.observability._helpers.identity import ( current_execution_identity, current_invocation_external_context, ) -from sqlbuild.runtime.observability.constants import CURRENT_LIFECYCLE_EVENT_SCHEMA_VERSION from sqlbuild.runtime.observability.exceptions import ObservabilityValidationError from sqlbuild.runtime.observability.models import ExecutionIdentity, LifecycleEvent from sqlbuild.runtime.observability.types import JSONValue diff --git a/src/sqlbuild/runtime/observability/_helpers/identity.py b/src/sqlbuild/runtime/observability/_helpers/identity.py index 599e68933e..58f4436675 100644 --- a/src/sqlbuild/runtime/observability/_helpers/identity.py +++ b/src/sqlbuild/runtime/observability/_helpers/identity.py @@ -10,7 +10,7 @@ from typing import cast from uuid import uuid4 -from sqlbuild.runtime.observability._helpers.validation import freeze_json +from sqlbuild._native import freeze_lifecycle_json from sqlbuild.runtime.observability.exceptions import ObservabilityValidationError from sqlbuild.runtime.observability.models import ExecutionIdentity from sqlbuild.runtime.observability.types import JSONValue @@ -73,7 +73,7 @@ def invocation_external_context_scope( ) -> Iterator[Mapping[str, JSONValue]]: """Install validated integration context for lifecycle facts in one command.""" - frozen: JSONValue = freeze_json(value=external_context, path="external_context") + frozen: JSONValue = freeze_lifecycle_json(value=external_context, path="external_context") if not isinstance(frozen, Mapping): raise ObservabilityValidationError("external_context must be a JSON object") context: Mapping[str, JSONValue] = cast(Mapping[str, JSONValue], frozen) diff --git a/src/sqlbuild/runtime/observability/_helpers/observability.py b/src/sqlbuild/runtime/observability/_helpers/observability.py index 80c5c8cdf6..a2406ea77f 100644 --- a/src/sqlbuild/runtime/observability/_helpers/observability.py +++ b/src/sqlbuild/runtime/observability/_helpers/observability.py @@ -7,19 +7,21 @@ from datetime import datetime from typing import Any, cast -from sqlbuild.runtime.observability._helpers.validation import validate_schema_version +from sqlbuild._native import ( + lifecycle_event_catalogued, + lifecycle_event_terminal, + validate_lifecycle_schema_version, +) from sqlbuild.runtime.observability.constants import ( CURRENT_DIAGNOSTIC_LOG_SCHEMA_VERSION, DIAGNOSTIC_ENVELOPE_FIELDS, LIFECYCLE_ENVELOPE_FIELDS_BY_VERSION, - LIFECYCLE_EVENT_CATALOGS, LIFECYCLE_INVOCATION_METADATA_SCHEMA_VERSION, ) from sqlbuild.runtime.observability.exceptions import ObservabilityValidationError from sqlbuild.runtime.observability.models import ( DiagnosticLog, LifecycleEvent, - LifecycleEventDefinition, OpaqueLifecycleEvent, ) @@ -96,16 +98,9 @@ def lifecycle_event_from_json(raw_json: str) -> LifecycleEvent | OpaqueLifecycle data: dict[str, Any] = _loads_object(raw_json=raw_json, envelope_name="lifecycle event") event_type: object = data.get("event_type") schema_version_raw: Any = data.get("schema_version") - validate_schema_version(value=schema_version_raw) + validate_lifecycle_schema_version(value=schema_version_raw) schema_version: int = cast(int, schema_version_raw) - version_catalog: Mapping[str, LifecycleEventDefinition] | None = LIFECYCLE_EVENT_CATALOGS.get( - schema_version - ) - if ( - version_catalog is None - or not isinstance(event_type, str) - or event_type not in version_catalog - ): + if not lifecycle_event_catalogued(schema_version=schema_version, event_type=event_type): return OpaqueLifecycleEvent(raw=data) _reject_unknown_fields( data=data, @@ -119,7 +114,7 @@ def lifecycle_event_from_json(raw_json: str) -> LifecycleEvent | OpaqueLifecycle occurred_at: datetime = datetime.fromisoformat(occurred_at_raw.replace("Z", "+00:00")) return LifecycleEvent( event_id=data["event_id"], - event_type=event_type, + event_type=cast(str, event_type), schema_version=schema_version, producer=data["producer"], producer_version=data["producer_version"], @@ -176,7 +171,7 @@ def diagnostic_log_from_json(raw_json: str) -> DiagnosticLog: data: dict[str, Any] = _loads_object(raw_json=raw_json, envelope_name="diagnostic log") schema_version_raw: Any = data.get("schema_version") - validate_schema_version(value=schema_version_raw) + validate_lifecycle_schema_version(value=schema_version_raw) schema_version: int = cast(int, schema_version_raw) if schema_version == CURRENT_DIAGNOSTIC_LOG_SCHEMA_VERSION: _reject_unknown_fields( @@ -219,7 +214,9 @@ def diagnostic_log_from_json(raw_json: str) -> DiagnosticLog: def is_terminal_event(event: LifecycleEvent) -> bool: """Return whether the fact closes its correlated lifecycle scope.""" - return LIFECYCLE_EVENT_CATALOGS[event.schema_version][event.event_type].terminal + return lifecycle_event_terminal( + schema_version=event.schema_version, event_type=event.event_type + ) def validate_idempotent_duplicate(*, original: LifecycleEvent, duplicate: LifecycleEvent) -> None: diff --git a/src/sqlbuild/runtime/observability/_helpers/validation.py b/src/sqlbuild/runtime/observability/_helpers/validation.py deleted file mode 100644 index 1dac7d85e3..0000000000 --- a/src/sqlbuild/runtime/observability/_helpers/validation.py +++ /dev/null @@ -1,410 +0,0 @@ -"""Validation support for immutable observability envelopes.""" - -from __future__ import annotations - -import json -import math -import re -from collections.abc import Mapping -from datetime import UTC, datetime -from types import MappingProxyType - -from sqlbuild.runtime.observability.constants import ( - AUDIT_RUN_COUNT_FIELDS, - BOOLEAN_PAYLOAD_FIELDS, - CONFIGURED_CONCURRENCY_FIELD, - DURATION_MS_FIELD, - EXIT_CODE_FIELD, - FINITE_NUMBER_PAYLOAD_FIELDS, - FORBIDDEN_STATEMENT_PAYLOAD_FIELDS, - HOOK_PHASES, - HOOK_TYPES, - JSON_PAYLOAD_FIELDS, - LIFECYCLE_EVENT_CATALOGS, - MAX_METADATA_BYTES, - METADATA_FIELD, - MICROBATCH_EVENT_PREFIX, - MICROBATCH_KINDS, - MICROBATCH_RUN_TYPES, - MICROBATCH_STRATEGIES, - NONNEGATIVE_INTEGER_PAYLOAD_FIELDS, - OPERATION_ADAPTERS, - OPERATION_EVENT_PREFIX, - OPERATION_KINDS, - OPERATION_METADATA_FIELDS, - OPERATION_NAMES, - OPERATION_PHASES, - OPERATION_SCOPES, - OPERATION_STRATEGIES, - OPERATION_TARGET_KINDS, - RESOURCE_ATTEMPT_SKIPPED_EVENT, - RESOURCE_SKIP_CODES, - RESOURCE_SKIP_MODES, - RETRY_SCHEDULED_EVENT, - RUN_STARTED_EVENT, - RUN_TERMINALS, - STATEMENT_EVENT_PREFIX, - STRING_PAYLOAD_FIELDS, -) -from sqlbuild.runtime.observability.exceptions import ObservabilityValidationError -from sqlbuild.runtime.observability.models import LifecycleEvent, LifecycleEventDefinition -from sqlbuild.runtime.observability.types import JSONValue - -_HOOK_NAME_PATTERN: re.Pattern[str] = re.compile(r"^[A-Za-z_][A-Za-z0-9_]{0,127}$") - - -def validate_schema_version(*, value: object) -> None: - """Validate the shared positive integer schema-version contract.""" - - if isinstance(value, bool) or not isinstance(value, int) or value <= 0: - raise ObservabilityValidationError( - "schema_version must be a positive integer excluding bool" - ) - - -def freeze_json(*, value: object, path: str) -> JSONValue: - """Validate and recursively freeze a JSON-compatible value.""" - - return _freeze_json(value=value, path=(path,)) - - -def _freeze_json(*, value: object, path: tuple[str | int, ...]) -> JSONValue: - if value is None or isinstance(value, (str, bool, int)): - return value - if isinstance(value, float): - if not math.isfinite(value): - raise ObservabilityValidationError( - f"{_render_path(path)} must not contain NaN or infinity" - ) - return value - if isinstance(value, Mapping): - frozen: dict[str, JSONValue] = {} - for key, item in value.items(): - if not isinstance(key, str): - raise ObservabilityValidationError(f"{_render_path(path)} keys must be strings") - frozen[key] = ( - item - if item is None or isinstance(item, (str, bool, int)) - else _freeze_json(value=item, path=(*path, key)) - ) - return MappingProxyType(frozen) - if isinstance(value, (list, tuple)): - return tuple( - _freeze_json(value=item, path=(*path, index)) for index, item in enumerate(value) - ) - raise ObservabilityValidationError( - f"{_render_path(path)} contains non-JSON value of type {type(value).__name__}" - ) - - -def _render_path(path: tuple[str | int, ...]) -> str: - return "".join( - f"[{segment}]" if isinstance(segment, int) else f".{segment}" if index else segment - for index, segment in enumerate(path) - ) - - -def validate_required_text(*, value: object, field_name: str) -> None: - """Validate one required non-empty string field.""" - - if not isinstance(value, str) or not value: - raise ObservabilityValidationError(f"{field_name} must be a non-empty string") - - -def validate_optional_text(*, value: object, field_name: str) -> None: - """Validate one optional non-empty string field.""" - - if value is not None: - validate_required_text(value=value, field_name=field_name) - - -def validate_timestamp(*, value: datetime) -> None: - """Validate a timezone-aware UTC timestamp.""" - - if not isinstance(value, datetime) or value.tzinfo is None: - raise ObservabilityValidationError("occurred_at must be a timezone-aware UTC datetime") - if value.utcoffset() != UTC.utcoffset(value): - raise ObservabilityValidationError("occurred_at must use UTC") - - -def _forbidden_statement_fields(*, value: object) -> set[str]: - if isinstance(value, Mapping): - found: set[str] = { - key - for key in value - if isinstance(key, str) and key in FORBIDDEN_STATEMENT_PAYLOAD_FIELDS - } - for item in value.values(): - found.update(_forbidden_statement_fields(value=item)) - return found - if isinstance(value, (list, tuple)): - found = set() - for item in value: - found.update(_forbidden_statement_fields(value=item)) - return found - return set() - - -def _plain_json(*, value: object) -> object: - if isinstance(value, Mapping): - return {key: _plain_json(value=item) for key, item in value.items()} - if isinstance(value, (list, tuple)): - return [_plain_json(value=item) for item in value] - return value - - -def _validate_payload_field(*, field_name: str, value: object) -> None: - if field_name in STRING_PAYLOAD_FIELDS: - if not isinstance(value, str) or not value: - raise ObservabilityValidationError( - f"payload field {field_name!r} must be a non-empty string" - ) - return - if field_name in NONNEGATIVE_INTEGER_PAYLOAD_FIELDS: - if isinstance(value, bool) or not isinstance(value, int) or value < 0: - raise ObservabilityValidationError( - f"payload field {field_name!r} must be a nonnegative integer excluding bool" - ) - return - if field_name in FINITE_NUMBER_PAYLOAD_FIELDS: - if ( - isinstance(value, bool) - or not isinstance(value, (int, float)) - or not math.isfinite(value) - ): - raise ObservabilityValidationError( - f"payload field {field_name!r} must be a finite number excluding bool" - ) - return - if field_name in BOOLEAN_PAYLOAD_FIELDS: - if not isinstance(value, bool): - raise ObservabilityValidationError(f"payload field {field_name!r} must be a boolean") - return - if field_name in JSON_PAYLOAD_FIELDS: - return - if field_name == DURATION_MS_FIELD: - if ( - isinstance(value, bool) - or not isinstance(value, (int, float)) - or not math.isfinite(value) - or value < 0 - ): - raise ObservabilityValidationError( - "payload field 'duration_ms' must be a nonnegative finite number excluding bool" - ) - return - if field_name == EXIT_CODE_FIELD: - if isinstance(value, bool) or not isinstance(value, int): - raise ObservabilityValidationError( - "payload field 'exit_code' must be an integer excluding bool" - ) - return - if field_name == METADATA_FIELD: - if not isinstance(value, Mapping): - raise ObservabilityValidationError("payload field 'metadata' must be a JSON object") - encoded: bytes = json.dumps( - _plain_json(value=value), - ensure_ascii=True, - allow_nan=False, - separators=(",", ":"), - sort_keys=True, - ).encode("utf-8") - if len(encoded) > MAX_METADATA_BYTES: - raise ObservabilityValidationError( - f"payload field 'metadata' must encode to at most {MAX_METADATA_BYTES} bytes" - ) - - -def _validate_operation_payload(*, payload: Mapping[str, JSONValue]) -> None: - operation_kind: JSONValue | None = payload.get("operation_kind") - if operation_kind is not None and ( - not isinstance(operation_kind, str) or operation_kind not in OPERATION_KINDS - ): - raise ObservabilityValidationError("operation_kind must be a catalogued value") - operation_name: JSONValue | None = payload.get("operation_name") - if operation_name is not None and ( - not isinstance(operation_name, str) or operation_name not in OPERATION_NAMES - ): - raise ObservabilityValidationError("operation_name must be a catalogued value") - for field_name, allowed in ( - ("phase", OPERATION_PHASES), - ("strategy", OPERATION_STRATEGIES), - ("target_kind", OPERATION_TARGET_KINDS), - ("adapter", OPERATION_ADAPTERS), - ("scope", OPERATION_SCOPES), - ): - value: JSONValue | None = payload.get(field_name) - if value is not None and (not isinstance(value, str) or value not in allowed): - raise ObservabilityValidationError(f"{field_name} must be a catalogued value") - hook_phase: JSONValue | None = payload.get("hook_phase") - hook_type: JSONValue | None = payload.get("hook_type") - hook_index: JSONValue | None = payload.get("hook_index") - hook_name: JSONValue | None = payload.get("hook_name") - hook_fields_present: bool = any( - value is not None for value in (hook_phase, hook_type, hook_index, hook_name) - ) - if hook_fields_present: - if hook_phase not in HOOK_PHASES: - raise ObservabilityValidationError("hook_phase must be a catalogued value") - if hook_type not in HOOK_TYPES: - raise ObservabilityValidationError("hook_type must be a catalogued value") - if isinstance(hook_index, bool) or not isinstance(hook_index, int) or hook_index < 0: - raise ObservabilityValidationError("hook_index must be a nonnegative integer") - if hook_name is not None and ( - not isinstance(hook_name, str) or _HOOK_NAME_PATTERN.fullmatch(hook_name) is None - ): - raise ObservabilityValidationError("hook_name must be a non-empty safe token") - metadata: JSONValue | None = payload.get(METADATA_FIELD) - if metadata is None: - return - if not isinstance(metadata, Mapping): - raise ObservabilityValidationError("operation metadata must be a JSON object") - if set(metadata) - OPERATION_METADATA_FIELDS: - raise ObservabilityValidationError("operation metadata contains a non-allowlisted field") - for metadata_value in metadata.values(): - if ( - isinstance(metadata_value, bool) - or not isinstance(metadata_value, int) - or metadata_value < 0 - ): - raise ObservabilityValidationError( - "operation metadata values must be nonnegative integers excluding bool" - ) - - -def validate_known_lifecycle_event(*, event: LifecycleEvent) -> None: - """Validate correlations and safe payload fields for a catalogued event.""" - - version_catalog: Mapping[str, LifecycleEventDefinition] | None = LIFECYCLE_EVENT_CATALOGS.get( - event.schema_version - ) - definition: LifecycleEventDefinition | None = ( - version_catalog.get(event.event_type) if version_catalog is not None else None - ) - if definition is None: - raise ObservabilityValidationError( - f"unknown lifecycle event_type {event.event_type!r}; decode it as an opaque envelope" - ) - missing: list[str] = sorted( - field_name - for field_name in definition.required_correlations - if getattr(event, field_name) is None - ) - if missing: - raise ObservabilityValidationError( - f"known event {event.event_type!r} requires correlation field(s): {', '.join(missing)}" - ) - missing_payload: list[str] = sorted(definition.required_payload_fields - set(event.payload)) - if missing_payload: - raise ObservabilityValidationError( - f"known event {event.event_type!r} requires payload field(s): " - f"{', '.join(missing_payload)}" - ) - unexpected: list[str] = sorted(set(event.payload) - definition.allowed_payload_fields) - if unexpected: - allowed: str = ", ".join(sorted(definition.allowed_payload_fields)) or "none" - raise ObservabilityValidationError( - f"payload field(s) not allowed for known event {event.event_type!r}: " - f"{', '.join(unexpected)}; allowed fields: {allowed}" - ) - if event.event_type.startswith(STATEMENT_EVENT_PREFIX): - forbidden: list[str] = sorted(_forbidden_statement_fields(value=event.payload)) - if forbidden: - raise ObservabilityValidationError( - f"statement lifecycle payload must not contain SQL or parameter values; " - f"forbidden field(s): {', '.join(forbidden)}" - ) - if event.event_type.startswith(OPERATION_EVENT_PREFIX): - _validate_operation_payload(payload=event.payload) - if event.event_type.startswith(MICROBATCH_EVENT_PREFIX): - _validate_microbatch_payload(payload=event.payload) - if event.event_type == RESOURCE_ATTEMPT_SKIPPED_EVENT: - if event.payload.get("skip_code") not in RESOURCE_SKIP_CODES: - raise ObservabilityValidationError("skip_code must be a catalogued value") - skip_mode: JSONValue | None = event.payload.get("skip_mode") - if skip_mode is not None and skip_mode not in RESOURCE_SKIP_MODES: - raise ObservabilityValidationError("skip_mode must be a catalogued value") - if event.event_type == RETRY_SCHEDULED_EVENT: - failed_attempt: JSONValue | None = event.payload.get("failed_attempt_number") - next_attempt: JSONValue | None = event.payload.get("next_attempt_number") - if ( - not isinstance(failed_attempt, int) - or isinstance(failed_attempt, bool) - or failed_attempt <= 0 - ): - raise ObservabilityValidationError("failed_attempt_number must be a positive integer") - if ( - not isinstance(next_attempt, int) - or isinstance(next_attempt, bool) - or next_attempt != failed_attempt + 1 - ): - raise ObservabilityValidationError( - "next_attempt_number must equal failed_attempt_number + 1" - ) - for field_name, value in event.payload.items(): - _validate_payload_field(field_name=field_name, value=value) - _validate_run_payload(event=event) - - -def _validate_microbatch_payload(*, payload: Mapping[str, JSONValue]) -> None: - batch_index: JSONValue | None = payload.get("batch_index") - batch_count: JSONValue | None = payload.get("batch_count") - if isinstance(batch_index, bool) or not isinstance(batch_index, int) or batch_index < 1: - raise ObservabilityValidationError("batch_index must be a positive integer") - if isinstance(batch_count, bool) or not isinstance(batch_count, int) or batch_count < 1: - raise ObservabilityValidationError("batch_count must be a positive integer") - if batch_index > batch_count: - raise ObservabilityValidationError("batch_index must not exceed batch_count") - for field_name, allowed in ( - ("microbatch_strategy", MICROBATCH_STRATEGIES), - ("microbatch_run_type", MICROBATCH_RUN_TYPES), - ("batch_kind", MICROBATCH_KINDS), - ): - value: JSONValue | None = payload.get(field_name) - if not isinstance(value, str) or value not in allowed: - raise ObservabilityValidationError(f"{field_name} must be a catalogued value") - - -def _validate_run_payload(*, event: LifecycleEvent) -> None: - payload: Mapping[str, JSONValue] = event.payload - if event.event_type == RUN_STARTED_EVENT and CONFIGURED_CONCURRENCY_FIELD in payload: - configured: JSONValue = payload[CONFIGURED_CONCURRENCY_FIELD] - selected: JSONValue | None = payload.get("selected_count") - workers: JSONValue | None = payload.get("worker_count") - if not isinstance(configured, int) or isinstance(configured, bool) or configured < 1: - raise ObservabilityValidationError( - "configured_concurrency must be a positive integer excluding bool" - ) - if not isinstance(selected, int) or isinstance(selected, bool) or selected < 0: - raise ObservabilityValidationError( - "selected_count must be a nonnegative integer excluding bool" - ) - if not isinstance(workers, int) or isinstance(workers, bool): - raise ObservabilityValidationError("worker_count must be an integer excluding bool") - if workers != min(configured, selected): - raise ObservabilityValidationError( - "worker_count must equal min(configured_concurrency, selected_count)" - ) - if event.event_type in RUN_TERMINALS and AUDIT_RUN_COUNT_FIELDS & set(payload): - if not AUDIT_RUN_COUNT_FIELDS <= set(payload): - raise ObservabilityValidationError( - "audit run terminals must include pass_count, warn_count, and fail_count together" - ) - pass_count: int = _required_run_count(payload=payload, field_name="pass_count") - warn_count: int = _required_run_count(payload=payload, field_name="warn_count") - fail_count: int = _required_run_count(payload=payload, field_name="fail_count") - succeeded_count: int = _required_run_count(payload=payload, field_name="succeeded_count") - failed_count: int = _required_run_count(payload=payload, field_name="failed_count") - if succeeded_count != pass_count + warn_count: - raise ObservabilityValidationError("succeeded_count must equal pass_count + warn_count") - if failed_count != fail_count: - raise ObservabilityValidationError("failed_count must equal fail_count") - - -def _required_run_count(*, payload: Mapping[str, JSONValue], field_name: str) -> int: - value: JSONValue | None = payload.get(field_name) - if isinstance(value, bool) or not isinstance(value, int) or value < 0: - raise ObservabilityValidationError( - f"{field_name} must be a nonnegative integer excluding bool" - ) - return value diff --git a/src/sqlbuild/runtime/observability/classes/operation_lifecycle.py b/src/sqlbuild/runtime/observability/classes/operation_lifecycle.py index 32568b62cb..e071b3223b 100644 --- a/src/sqlbuild/runtime/observability/classes/operation_lifecycle.py +++ b/src/sqlbuild/runtime/observability/classes/operation_lifecycle.py @@ -10,18 +10,7 @@ from types import TracebackType from typing import Any -from sqlbuild.runtime.observability._helpers.dispatcher import ( - current_event_dispatcher, - dispatcher_scope, -) -from sqlbuild.runtime.observability._helpers.factory import create_lifecycle_event -from sqlbuild.runtime.observability._helpers.identity import ( - current_execution_identity, - invocation_scope, - operation_scope, -) -from sqlbuild.runtime.observability.classes.event_dispatcher import EventDispatcher -from sqlbuild.runtime.observability.constants import ( +from sqlbuild._native import ( HOOK_PHASES, HOOK_TYPES, OPERATION_ADAPTERS, @@ -32,8 +21,19 @@ OPERATION_SCOPES, OPERATION_STRATEGIES, OPERATION_TARGET_KINDS, - RETRY_SCHEDULED_EVENT, ) +from sqlbuild.runtime.observability._helpers.dispatcher import ( + current_event_dispatcher, + dispatcher_scope, +) +from sqlbuild.runtime.observability._helpers.factory import create_lifecycle_event +from sqlbuild.runtime.observability._helpers.identity import ( + current_execution_identity, + invocation_scope, + operation_scope, +) +from sqlbuild.runtime.observability.classes.event_dispatcher import EventDispatcher +from sqlbuild.runtime.observability.constants import RETRY_SCHEDULED_EVENT from sqlbuild.runtime.observability.exceptions import ObservabilityValidationError from sqlbuild.runtime.observability.models import ExecutionIdentity, OperationAttributes from sqlbuild.runtime.observability.types import JSONValue diff --git a/src/sqlbuild/runtime/observability/classes/resource_attempt_lifecycle.py b/src/sqlbuild/runtime/observability/classes/resource_attempt_lifecycle.py index e4dd7b4c2e..9b0bf9aa84 100644 --- a/src/sqlbuild/runtime/observability/classes/resource_attempt_lifecycle.py +++ b/src/sqlbuild/runtime/observability/classes/resource_attempt_lifecycle.py @@ -8,6 +8,7 @@ from types import TracebackType from uuid import uuid4 +from sqlbuild._native import RESOURCE_SKIP_CODES, RESOURCE_SKIP_MODES from sqlbuild.runtime.observability._helpers.dispatcher import ( current_event_dispatcher, dispatcher_scope, @@ -20,11 +21,7 @@ run_scope, ) from sqlbuild.runtime.observability.classes.event_dispatcher import EventDispatcher -from sqlbuild.runtime.observability.constants import ( - RESOURCE_ATTEMPT_SKIPPED_EVENT, - RESOURCE_SKIP_CODES, - RESOURCE_SKIP_MODES, -) +from sqlbuild.runtime.observability.constants import RESOURCE_ATTEMPT_SKIPPED_EVENT from sqlbuild.runtime.observability.exceptions import ObservabilityValidationError from sqlbuild.runtime.observability.models import ExecutionIdentity from sqlbuild.runtime.observability.types import JSONValue diff --git a/src/sqlbuild/runtime/observability/constants.py b/src/sqlbuild/runtime/observability/constants.py index d09e09cbca..61ddeeeb6b 100644 --- a/src/sqlbuild/runtime/observability/constants.py +++ b/src/sqlbuild/runtime/observability/constants.py @@ -1,332 +1,17 @@ -"""Versioned runtime observability catalogs and decision values.""" +"""Lifecycle envelope, diagnostic log and emitter constants; the event catalogue is native.""" from collections.abc import Mapping from types import MappingProxyType -from sqlbuild.runtime.observability.models import LifecycleEventDefinition - -CURRENT_LIFECYCLE_EVENT_SCHEMA_VERSION: int = 2 LIFECYCLE_INVOCATION_METADATA_SCHEMA_VERSION: int = 2 CURRENT_DIAGNOSTIC_LOG_SCHEMA_VERSION: int = 1 -DURATION_MS_FIELD: str = "duration_ms" -EXIT_CODE_FIELD: str = "exit_code" -PROCESS_ID_FIELD: str = "process_id" -SIGNAL_NUMBER_FIELD: str = "signal_number" -METADATA_FIELD: str = "metadata" -CONFIGURED_CONCURRENCY_FIELD: str = "configured_concurrency" -RUN_STARTED_EVENT: str = "run_started" -RUN_TERMINALS: frozenset[str] = frozenset({"run_completed", "run_failed"}) -AUDIT_RUN_COUNT_FIELDS: frozenset[str] = frozenset({"pass_count", "warn_count", "fail_count"}) -STATEMENT_EVENT_PREFIX: str = "statement_" -MAX_METADATA_BYTES: int = 4096 DIAGNOSTIC_SEVERITIES: frozenset[str] = frozenset( {"trace", "debug", "info", "warning", "error", "critical"} ) -ERROR_FIELDS: frozenset[str] = frozenset({"error_code", "error_type"}) -DURATION_FIELDS: frozenset[str] = frozenset({DURATION_MS_FIELD}) -RESOURCE_FIELDS: frozenset[str] = frozenset({"resource_kind", "resource_name", "attempt_number"}) -RESOURCE_SKIP_CODES: frozenset[str] = frozenset({"dependency", "explicit", "fan_in", "scheduler"}) -RESOURCE_SKIP_MODES: frozenset[str] = frozenset({"hard", "soft"}) RESOURCE_ATTEMPT_SKIPPED_EVENT: str = "resource_attempt_skipped" -RESOURCE_TERMINALS: frozenset[str] = frozenset( - {"resource_attempt_completed", "resource_attempt_failed", RESOURCE_ATTEMPT_SKIPPED_EVENT} -) -MICROBATCH_EVENT_PREFIX: str = "microbatch_" -MICROBATCH_FIELDS: frozenset[str] = frozenset( - { - "cursor_start", - "cursor_end_exclusive", - "planned_cursor_start", - "planned_cursor_end_exclusive", - "batch_index", - "batch_count", - "configured_batch_size", - "effective_batch_size", - "microbatch_strategy", - "microbatch_run_type", - "batch_kind", - } -) -MICROBATCH_STRATEGIES: frozenset[str] = frozenset({"rolling_window", "watermark"}) -MICROBATCH_RUN_TYPES: frozenset[str] = frozenset({"normal", "backfill", "replay_on_change"}) -MICROBATCH_KINDS: frozenset[str] = frozenset({"ordinary", "recovery"}) -HOOK_PHASES: frozenset[str] = frozenset({"post_hooks", "pre_hooks"}) -HOOK_TYPES: frozenset[str] = frozenset({"python", "sql"}) -OPERATION_FIELDS: frozenset[str] = frozenset( - { - "operation_kind", - "operation_name", - METADATA_FIELD, - "hook_phase", - "hook_index", - "hook_type", - "hook_name", - "phase", - "strategy", - "adapter", - "target_kind", - "scope", - } -) -OPERATION_EVENT_PREFIX: str = "operation_" RENAME_OPERATION_STRATEGY: str = "rename" RETRY_SCHEDULED_EVENT: str = "retry_scheduled" -STATEMENT_FAILED_EVENT: str = "statement_failed" STATEMENT_HEARTBEAT_THRESHOLD_SECONDS: float = 30.0 -OPERATION_KINDS: frozenset[str] = frozenset( - { - "clone", - "freshness", - "janitor", - "loader", - "project", - "python_node", - "quality", - "scenario", - "subprocess", - "warehouse", - } -) -OPERATION_NAMES: frozenset[str] = frozenset( - { - "clone_finalization", - "clone_execution", - "clone_namespace_preparation", - "clone_relation_inspection", - "clone_relation_transfer", - "clone_retention_reconciliation", - "clone_state_connection", - "clone_state_inspection", - "clone_target_connection", - "dbt_command", - "discovery_declaration_parse", - "discovery_filesystem_walk", - "discovery_project_assembly", - "discovery_python_import", - "external_manifest_discovery", - "external_source_load", - "ingestr_command", - "janitor_candidate_planning", - "janitor_cleanup_action", - "janitor_execution", - "janitor_state_inspection", - "janitor_target_connection", - "janitor_warehouse_inspection", - "managed_source_load", - "project_compile", - "project_discovery", - "python_asset", - "python_check", - "python_hook", - "python_materialization", - "python_task", - "sql_hook", - "sql_test_planning", - "scenario_capture", - "scenario_cleanup", - "scenario_execution", - "scenario_relation_read", - "scenario_schema_inspection", - "scenario_snapshot_serialization", - "scenario_snapshot_write", - "scenario_target_connection", - "source_freshness_metadata_observation", - "source_freshness_query_observation", - "source_freshness_publication", - "audit_evaluation", - "sql_test_setup", - "sql_test_assertion", - "retention_inspection", - "retention_application", - "table_type_inspection", - "table_type_conversion", - "runtime_schema_inspection", - "schema_synchronization", - "staging_creation", - "relation_promotion", - } -) -OPERATION_METADATA_FIELDS: frozenset[str] = frozenset( - { - "attempt_number", - "byte_count", - "item_count", - "row_count", - "changed_count", - "added_count", - "removed_count", - "altered_count", - "unknown_count", - } -) -OPERATION_PHASES: frozenset[str] = frozenset( - { - "inspect", - "apply", - "assert", - "convert", - "create", - "evaluate", - "observe", - "promote", - "publish", - "reconcile", - "setup", - } -) -OPERATION_STRATEGIES: frozenset[str] = frozenset( - { - "atomic_replace", - "atomic_swap", - RENAME_OPERATION_STRATEGY, - "build_aside", - "create_new", - "append_new_columns", - "sync_all_columns", - "adapter", - "column", - "sql", - } -) -OPERATION_TARGET_KINDS: frozenset[str] = frozenset( - { - "audit", - "namespace", - "relation", - "source", - "sql_test", - "staging_relation", - "state_batch", - } -) -OPERATION_SCOPES: frozenset[str] = frozenset( - {"delta", "end", "final", "model", "seed", "source", "standalone"} -) -OPERATION_ADAPTERS: frozenset[str] = frozenset( - { - "bigquery", - "custom", - "databricks", - "duckdb", - "motherduck", - "postgres", - "snowflake", - "sqlserver", - } -) -STATEMENT_FIELDS: frozenset[str] = frozenset( - { - "adapter", - "affected_rows", - "batch_size", - "intent", - "job_id", - METADATA_FIELD, - "query_id", - "row_count", - "sql_digest", - "statement_kind", - } -) -STRING_PAYLOAD_FIELDS: frozenset[str] = frozenset( - { - "adapter", - "command", - "error_code", - "error_type", - "hook_name", - "hook_phase", - "hook_type", - "intent", - "job_id", - "operation_kind", - "operation_name", - "query_id", - "resource_kind", - "resource_name", - "run_kind", - "scope", - "sql_digest", - "statement_kind", - "audit_name", - "audit_definition_name", - "audit_description", - "evaluation_mode", - "outcome", - "severity", - "run_scope_phase", - "attachment_kind", - "binding_key", - "definition_fingerprint", - "execution_fingerprint", - "attached_target_kind", - "attached_target_name", - "attached_column_name", - "target_database", - "target_schema", - "target_name", - "sample_unit", - "evidence_error", - "measurement_sql", - "evidence_sql", - "executed_sql", - "execution_error", - "result_id", - "cursor_start", - "cursor_end_exclusive", - "planned_cursor_start", - "planned_cursor_end_exclusive", - "configured_batch_size", - "effective_batch_size", - "microbatch_strategy", - "microbatch_run_type", - "batch_kind", - } -) -NONNEGATIVE_INTEGER_PAYLOAD_FIELDS: frozenset[str] = frozenset( - { - "affected_rows", - "attempt_number", - "batch_size", - "delay_ms", - "failed_count", - "fail_count", - "failed_attempt_number", - "hook_index", - "next_attempt_number", - PROCESS_ID_FIELD, - "row_count", - "selected_count", - "configured_concurrency", - "pass_count", - SIGNAL_NUMBER_FIELD, - "skipped_count", - "succeeded_count", - "warn_count", - "worker_count", - "violation_count", - "sample_count", - "minimum_samples", - "evidence_count", - "batch_index", - "batch_count", - } -) -FINITE_NUMBER_PAYLOAD_FIELDS: frozenset[str] = frozenset({"measured_value"}) -BOOLEAN_PAYLOAD_FIELDS: frozenset[str] = frozenset({"evidence_truncated", "reused"}) -JSON_PAYLOAD_FIELDS: frozenset[str] = frozenset({"thresholds", "evidence"}) -FORBIDDEN_STATEMENT_PAYLOAD_FIELDS: frozenset[str] = frozenset( - { - "bindings", - "full_sql", - "parameter_values", - "parameters", - "params", - "query_sql", - "sql", - "statement_sql", - } -) LIFECYCLE_ENVELOPE_FIELDS_V1: frozenset[str] = frozenset( { "event_id", @@ -370,212 +55,3 @@ "statement_id", } ) -LIFECYCLE_EVENT_CATALOG_V1: Mapping[str, LifecycleEventDefinition] = MappingProxyType( - { - "invocation_started": LifecycleEventDefinition.create(allowed=frozenset({"command"})), - "invocation_completed": LifecycleEventDefinition.create( - allowed=frozenset({"command", EXIT_CODE_FIELD}) | DURATION_FIELDS, terminal=True - ), - "invocation_failed": LifecycleEventDefinition.create( - allowed=frozenset({"command", EXIT_CODE_FIELD}) | DURATION_FIELDS | ERROR_FIELDS, - terminal=True, - ), - "run_started": LifecycleEventDefinition.create( - required_correlations=frozenset({"run_id"}), - allowed=frozenset( - {"run_kind", "selected_count", "configured_concurrency", "worker_count"} - ), - ), - "run_completed": LifecycleEventDefinition.create( - required_correlations=frozenset({"run_id"}), - allowed=frozenset( - { - "run_kind", - "succeeded_count", - "failed_count", - "skipped_count", - "pass_count", - "warn_count", - "fail_count", - } - ) - | DURATION_FIELDS, - terminal=True, - ), - "run_failed": LifecycleEventDefinition.create( - required_correlations=frozenset({"run_id"}), - allowed=frozenset( - { - "run_kind", - "succeeded_count", - "failed_count", - "skipped_count", - "pass_count", - "warn_count", - "fail_count", - } - ) - | DURATION_FIELDS - | ERROR_FIELDS, - terminal=True, - ), - "resource_attempt_started": LifecycleEventDefinition.create( - required_correlations=frozenset({"run_id", "resource_id", "resource_attempt_id"}), - allowed=RESOURCE_FIELDS, - ), - "resource_attempt_completed": LifecycleEventDefinition.create( - required_correlations=frozenset({"run_id", "resource_id", "resource_attempt_id"}), - allowed=RESOURCE_FIELDS | DURATION_FIELDS, - terminal=True, - ), - "resource_attempt_failed": LifecycleEventDefinition.create( - required_correlations=frozenset({"run_id", "resource_id", "resource_attempt_id"}), - allowed=RESOURCE_FIELDS | DURATION_FIELDS | ERROR_FIELDS, - terminal=True, - ), - RESOURCE_ATTEMPT_SKIPPED_EVENT: LifecycleEventDefinition.create( - required_correlations=frozenset({"run_id", "resource_id", "resource_attempt_id"}), - required_payload=RESOURCE_FIELDS | frozenset({"skip_code"}), - allowed=RESOURCE_FIELDS | DURATION_FIELDS | frozenset({"skip_code", "skip_mode"}), - terminal=True, - ), - "operation_started": LifecycleEventDefinition.create( - required_correlations=frozenset({"operation_id"}), allowed=OPERATION_FIELDS - ), - "operation_completed": LifecycleEventDefinition.create( - required_correlations=frozenset({"operation_id"}), - allowed=OPERATION_FIELDS - | DURATION_FIELDS - | frozenset({EXIT_CODE_FIELD, PROCESS_ID_FIELD, SIGNAL_NUMBER_FIELD}), - terminal=True, - ), - "operation_failed": LifecycleEventDefinition.create( - required_correlations=frozenset({"operation_id"}), - allowed=OPERATION_FIELDS - | DURATION_FIELDS - | ERROR_FIELDS - | frozenset({EXIT_CODE_FIELD, PROCESS_ID_FIELD, SIGNAL_NUMBER_FIELD}), - terminal=True, - ), - RETRY_SCHEDULED_EVENT: LifecycleEventDefinition.create( - required_correlations=frozenset({"run_id", "resource_id", "resource_attempt_id"}), - required_payload=frozenset( - { - "failed_attempt_number", - "next_attempt_number", - "delay_ms", - "error_type", - } - ), - allowed=frozenset( - { - "failed_attempt_number", - "next_attempt_number", - "delay_ms", - "error_type", - "error_code", - } - ), - ), - "audit_completed": LifecycleEventDefinition.create( - required_correlations=frozenset({"run_id"}), - required_payload=frozenset( - { - "audit_name", - "audit_definition_name", - "evaluation_mode", - "outcome", - "severity", - "run_scope_phase", - "attachment_kind", - } - ), - allowed=frozenset( - { - "audit_name", - "audit_definition_name", - "audit_description", - "evaluation_mode", - "outcome", - "severity", - "run_scope_phase", - "attachment_kind", - "binding_key", - "definition_fingerprint", - "execution_fingerprint", - "attached_target_kind", - "attached_target_name", - "attached_column_name", - "target_database", - "target_schema", - "target_name", - "violation_count", - "measured_value", - "sample_count", - "sample_unit", - "minimum_samples", - "thresholds", - "evidence", - "evidence_count", - "evidence_truncated", - "evidence_error", - "measurement_sql", - "evidence_sql", - "executed_sql", - "sql_digest", - "execution_error", - "reused", - "result_id", - } - ), - terminal=True, - ), - "statement_started": LifecycleEventDefinition.create( - required_correlations=frozenset({"statement_id"}), allowed=STATEMENT_FIELDS - ), - "statement_submitted": LifecycleEventDefinition.create( - required_correlations=frozenset({"statement_id"}), - allowed=STATEMENT_FIELDS - frozenset({"row_count"}), - ), - "statement_heartbeat": LifecycleEventDefinition.create( - required_correlations=frozenset({"statement_id"}), - allowed=STATEMENT_FIELDS | DURATION_FIELDS, - ), - "statement_completed": LifecycleEventDefinition.create( - required_correlations=frozenset({"statement_id"}), - allowed=STATEMENT_FIELDS | DURATION_FIELDS, - terminal=True, - ), - "statement_failed": LifecycleEventDefinition.create( - required_correlations=frozenset({"statement_id"}), - allowed=STATEMENT_FIELDS | DURATION_FIELDS | ERROR_FIELDS, - terminal=True, - ), - } -) -LIFECYCLE_EVENT_CATALOG_V2: Mapping[str, LifecycleEventDefinition] = MappingProxyType( - { - **LIFECYCLE_EVENT_CATALOG_V1, - "microbatch_started": LifecycleEventDefinition.create( - required_correlations=frozenset({"run_id", "resource_id", "resource_attempt_id"}), - required_payload=MICROBATCH_FIELDS, - allowed=MICROBATCH_FIELDS, - ), - "microbatch_completed": LifecycleEventDefinition.create( - required_correlations=frozenset({"run_id", "resource_id", "resource_attempt_id"}), - required_payload=MICROBATCH_FIELDS, - allowed=MICROBATCH_FIELDS | DURATION_FIELDS | frozenset({"affected_rows"}), - terminal=True, - ), - "microbatch_failed": LifecycleEventDefinition.create( - required_correlations=frozenset({"run_id", "resource_id", "resource_attempt_id"}), - required_payload=MICROBATCH_FIELDS | frozenset({"error_type"}), - allowed=MICROBATCH_FIELDS | DURATION_FIELDS | ERROR_FIELDS, - terminal=True, - ), - } -) -LIFECYCLE_EVENT_CATALOGS: Mapping[int, Mapping[str, LifecycleEventDefinition]] = MappingProxyType( - {1: LIFECYCLE_EVENT_CATALOG_V1, 2: LIFECYCLE_EVENT_CATALOG_V2} -) -LIFECYCLE_EVENT_CATALOG: Mapping[str, LifecycleEventDefinition] = LIFECYCLE_EVENT_CATALOG_V2 diff --git a/src/sqlbuild/runtime/observability/main/canonicalize_operation_adapter.py b/src/sqlbuild/runtime/observability/main/canonicalize_operation_adapter.py index 411f9259ff..76636ad78f 100644 --- a/src/sqlbuild/runtime/observability/main/canonicalize_operation_adapter.py +++ b/src/sqlbuild/runtime/observability/main/canonicalize_operation_adapter.py @@ -1,6 +1,6 @@ """Canonical operation adapter identity.""" -from sqlbuild.runtime.observability.constants import OPERATION_ADAPTERS +from sqlbuild._native import OPERATION_ADAPTERS from sqlbuild.runtime.observability.exceptions import ObservabilityValidationError diff --git a/src/sqlbuild/runtime/observability/models.py b/src/sqlbuild/runtime/observability/models.py index 1efbb621d0..1272ee6d80 100644 --- a/src/sqlbuild/runtime/observability/models.py +++ b/src/sqlbuild/runtime/observability/models.py @@ -6,8 +6,18 @@ from collections.abc import Mapping from dataclasses import dataclass, field from datetime import datetime -from typing import Literal, Self - +from typing import Literal + +from sqlbuild._native import ( + CURRENT_LIFECYCLE_EVENT_SCHEMA_VERSION, + LIFECYCLE_EVENT_SCHEMA_VERSIONS, + freeze_lifecycle_json, + validate_known_lifecycle_event, + validate_lifecycle_optional_text, + validate_lifecycle_required_text, + validate_lifecycle_schema_version, + validate_lifecycle_timestamp, +) from sqlbuild.runtime.observability.exceptions import ObservabilityValidationError from sqlbuild.runtime.observability.types import JSONValue @@ -68,12 +78,8 @@ class ExecutionIdentity: log_stream_id: str | None = None def __post_init__(self) -> None: - from sqlbuild.runtime.observability._helpers.validation import ( - validate_optional_text, - validate_required_text, - ) - validate_required_text(value=self.invocation_id, field_name="invocation_id") + validate_lifecycle_required_text(value=self.invocation_id, field_name="invocation_id") for field_name in ( "run_id", "resource_id", @@ -82,30 +88,7 @@ def __post_init__(self) -> None: "statement_id", "log_stream_id", ): - validate_optional_text(value=getattr(self, field_name), field_name=field_name) - - -@dataclass(frozen=True) -class LifecycleEventDefinition: - """Validation contract for one known lifecycle event type.""" - - required_correlations: frozenset[str] - required_payload_fields: frozenset[str] - allowed_payload_fields: frozenset[str] - terminal: bool - - @classmethod - def create( - cls, - *, - required_correlations: frozenset[str] = frozenset(), - required_payload: frozenset[str] = frozenset(), - allowed: frozenset[str] = frozenset(), - terminal: bool = False, - ) -> Self: - """Create one immutable catalog definition.""" - - return cls(required_correlations, required_payload, allowed, terminal) + validate_lifecycle_optional_text(value=getattr(self, field_name), field_name=field_name) @dataclass(frozen=True) @@ -129,18 +112,6 @@ class LifecycleEvent: payload: Mapping[str, JSONValue] = field(default_factory=dict) def __post_init__(self) -> None: - from sqlbuild.runtime.observability._helpers.validation import ( - freeze_json, - validate_known_lifecycle_event, - validate_optional_text, - validate_required_text, - validate_schema_version, - validate_timestamp, - ) - from sqlbuild.runtime.observability.constants import ( - CURRENT_LIFECYCLE_EVENT_SCHEMA_VERSION, - LIFECYCLE_EVENT_CATALOGS, - ) for field_name in ( "event_id", @@ -149,9 +120,9 @@ def __post_init__(self) -> None: "producer_version", "invocation_id", ): - validate_required_text(value=getattr(self, field_name), field_name=field_name) - validate_schema_version(value=self.schema_version) - if self.schema_version not in LIFECYCLE_EVENT_CATALOGS: + validate_lifecycle_required_text(value=getattr(self, field_name), field_name=field_name) + validate_lifecycle_schema_version(value=self.schema_version) + if self.schema_version not in LIFECYCLE_EVENT_SCHEMA_VERSIONS: raise ObservabilityValidationError( "LifecycleEvent only represents known schema versions through " f"{CURRENT_LIFECYCLE_EVENT_SCHEMA_VERSION}; decode other versions as " @@ -170,7 +141,7 @@ def __post_init__(self) -> None: raise ObservabilityValidationError( "invocation_sequence must be a non-negative integer excluding bool" ) - validate_timestamp(value=self.occurred_at) + validate_lifecycle_timestamp(value=self.occurred_at) for field_name in ( "run_id", "resource_id", @@ -178,14 +149,14 @@ def __post_init__(self) -> None: "operation_id", "statement_id", ): - validate_optional_text(value=getattr(self, field_name), field_name=field_name) - frozen_context: JSONValue = freeze_json( + validate_lifecycle_optional_text(value=getattr(self, field_name), field_name=field_name) + frozen_context: JSONValue = freeze_lifecycle_json( value=self.external_context, path="external_context" ) if not isinstance(frozen_context, Mapping): raise ObservabilityValidationError("external_context must be a JSON object") object.__setattr__(self, "external_context", frozen_context) - frozen_payload: JSONValue = freeze_json(value=self.payload, path="payload") + frozen_payload: JSONValue = freeze_lifecycle_json(value=self.payload, path="payload") if not isinstance(frozen_payload, Mapping): raise ObservabilityValidationError("payload must be a JSON object") object.__setattr__(self, "payload", frozen_payload) @@ -210,9 +181,7 @@ class OpaqueLifecycleEvent: raw: Mapping[str, JSONValue] def __post_init__(self) -> None: - from sqlbuild.runtime.observability._helpers.validation import freeze_json - - frozen_raw: JSONValue = freeze_json(value=self.raw, path="event") + frozen_raw: JSONValue = freeze_lifecycle_json(value=self.raw, path="event") if not isinstance(frozen_raw, Mapping): raise ObservabilityValidationError("event must be a JSON object") object.__setattr__(self, "raw", frozen_raw) @@ -240,19 +209,12 @@ class DiagnosticLog: statement_id: str | None = None def __post_init__(self) -> None: - from sqlbuild.runtime.observability._helpers.validation import ( - freeze_json, - validate_optional_text, - validate_required_text, - validate_schema_version, - validate_timestamp, - ) from sqlbuild.runtime.observability.constants import ( CURRENT_DIAGNOSTIC_LOG_SCHEMA_VERSION, DIAGNOSTIC_SEVERITIES, ) - validate_schema_version(value=self.schema_version) + validate_lifecycle_schema_version(value=self.schema_version) if self.schema_version != CURRENT_DIAGNOSTIC_LOG_SCHEMA_VERSION: raise ObservabilityValidationError( f"diagnostic schema_version must be {CURRENT_DIAGNOSTIC_LOG_SCHEMA_VERSION}" @@ -265,12 +227,12 @@ def __post_init__(self) -> None: "source", "message", ): - validate_required_text(value=getattr(self, field_name), field_name=field_name) + validate_lifecycle_required_text(value=getattr(self, field_name), field_name=field_name) if self.severity not in DIAGNOSTIC_SEVERITIES: raise ObservabilityValidationError( "severity must be one of trace, debug, info, warning, error, critical" ) - validate_timestamp(value=self.occurred_at) + validate_lifecycle_timestamp(value=self.occurred_at) for field_name in ( "log_stream_id", "invocation_id", @@ -280,8 +242,8 @@ def __post_init__(self) -> None: "operation_id", "statement_id", ): - validate_optional_text(value=getattr(self, field_name), field_name=field_name) - frozen_fields: JSONValue = freeze_json(value=self.fields, path="fields") + validate_lifecycle_optional_text(value=getattr(self, field_name), field_name=field_name) + frozen_fields: JSONValue = freeze_lifecycle_json(value=self.fields, path="fields") if not isinstance(frozen_fields, Mapping): raise ObservabilityValidationError("fields must be a JSON object") object.__setattr__(self, "fields", frozen_fields) diff --git a/tests/unit/src/sqlbuild/runtime/event_exporting/helpers.py b/tests/unit/src/sqlbuild/runtime/event_exporting/helpers.py new file mode 100644 index 0000000000..6faca8e083 --- /dev/null +++ b/tests/unit/src/sqlbuild/runtime/event_exporting/helpers.py @@ -0,0 +1,29 @@ +"""Shared event export test data.""" + +from __future__ import annotations + +CATALOGUED_EVENT_TYPES: tuple[str, ...] = ( + "invocation_started", + "invocation_completed", + "invocation_failed", + "run_started", + "run_completed", + "run_failed", + "resource_attempt_started", + "resource_attempt_completed", + "resource_attempt_failed", + "resource_attempt_skipped", + "operation_started", + "operation_completed", + "operation_failed", + "retry_scheduled", + "audit_completed", + "statement_started", + "statement_submitted", + "statement_heartbeat", + "statement_completed", + "statement_failed", + "microbatch_started", + "microbatch_completed", + "microbatch_failed", +) diff --git a/tests/unit/src/sqlbuild/runtime/event_exporting/test_policy.py b/tests/unit/src/sqlbuild/runtime/event_exporting/test_policy.py index 9ddf36bb45..dab0c4b482 100644 --- a/tests/unit/src/sqlbuild/runtime/event_exporting/test_policy.py +++ b/tests/unit/src/sqlbuild/runtime/event_exporting/test_policy.py @@ -8,16 +8,16 @@ ) from sqlbuild.runtime.event_exporting.constants import EVENT_EXPORT_KINDS from sqlbuild.runtime.event_exporting.types import LifecycleEventKind -from sqlbuild.runtime.observability.constants import LIFECYCLE_EVENT_CATALOG from tests.unit.src.sqlbuild.runtime.event_exporting._test_types import ( EventExportPolicyTestCase, SeverityNarrowingTestCase, ) +from tests.unit.src.sqlbuild.runtime.event_exporting.helpers import CATALOGUED_EVENT_TYPES @pytest.mark.parametrize( "test_case", - (EventExportPolicyTestCase("complete canonical mapping", len(LIFECYCLE_EVENT_CATALOG)),), + (EventExportPolicyTestCase("complete canonical mapping", len(CATALOGUED_EVENT_TYPES)),), ids=lambda case: case.description, ) def test_given_canonical_catalog_when_building_export_policy_then_every_type_is_mapped_once( @@ -25,7 +25,7 @@ def test_given_canonical_catalog_when_building_export_policy_then_every_type_is_ ) -> None: policy: dict[str, tuple[str, EventExportSeverity, int]] = lifecycle_export_policy_catalog() assert len(policy) == test_case.expected_event_count - assert policy.keys() == LIFECYCLE_EVENT_CATALOG.keys() + assert tuple(policy) == CATALOGUED_EVENT_TYPES assert {kind for kind, _, _ in policy.values()} == EVENT_EXPORT_KINDS assert policy["invocation_failed"] == ("invocation", EventExportSeverity.ERROR, 3) assert policy["resource_attempt_skipped"] == ("resource", EventExportSeverity.INFO, 2) diff --git a/tests/unit/src/sqlbuild/runtime/observability/test_catalog.py b/tests/unit/src/sqlbuild/runtime/observability/test_catalog.py index 52e65f7816..e14d16718b 100644 --- a/tests/unit/src/sqlbuild/runtime/observability/test_catalog.py +++ b/tests/unit/src/sqlbuild/runtime/observability/test_catalog.py @@ -5,9 +5,13 @@ import pytest +from sqlbuild._native import ( + LIFECYCLE_EVENT_SCHEMA_VERSIONS, + lifecycle_event_catalogued, + lifecycle_event_terminal, +) from sqlbuild.observability import create_lifecycle_event, invocation_scope, run_scope from sqlbuild.runtime.observability.classes.run_lifecycle import RunLifecycle -from sqlbuild.runtime.observability.constants import LIFECYCLE_EVENT_CATALOGS from sqlbuild.runtime.observability.exceptions import ObservabilityValidationError from sqlbuild.runtime.observability.main.canonicalize_operation_adapter import ( canonicalize_operation_adapter, @@ -398,10 +402,12 @@ def test_given_catalog_when_looking_up_events_then_authority_is_version_dimensio test_case: CatalogVersionCase, ) -> None: assert ( - LIFECYCLE_EVENT_CATALOGS[test_case.schema_version][test_case.event_type].terminal + lifecycle_event_terminal( + schema_version=test_case.schema_version, event_type=test_case.event_type + ) is test_case.expected_terminal ) - assert test_case.expected_unsupported_version not in LIFECYCLE_EVENT_CATALOGS + assert test_case.expected_unsupported_version not in LIFECYCLE_EVENT_SCHEMA_VERSIONS @pytest.mark.parametrize( @@ -426,8 +432,11 @@ def test_given_microbatch_events_when_inspecting_versioned_catalog_then_v1_remai test_case: CatalogEventPresenceCase, ) -> None: assert ( - test_case.event_type in LIFECYCLE_EVENT_CATALOGS[test_case.schema_version] - ) is test_case.expected_present + lifecycle_event_catalogued( + schema_version=test_case.schema_version, event_type=test_case.event_type + ) + is test_case.expected_present + ) @pytest.mark.parametrize( diff --git a/tests/unit/src/sqlbuild/runtime/observability/test_models.py b/tests/unit/src/sqlbuild/runtime/observability/test_models.py index 25e67c29bf..a98a51c61c 100644 --- a/tests/unit/src/sqlbuild/runtime/observability/test_models.py +++ b/tests/unit/src/sqlbuild/runtime/observability/test_models.py @@ -4,7 +4,7 @@ import pytest -from sqlbuild.runtime.observability._helpers.validation import freeze_json +from sqlbuild._native import freeze_lifecycle_json from sqlbuild.runtime.observability.exceptions import ObservabilityValidationError from sqlbuild.runtime.observability.models import DiagnosticLog, LifecycleEvent from tests.unit.src.sqlbuild.runtime.observability._test_types import ( @@ -173,7 +173,7 @@ def test_given_invalid_nested_payload_when_freezing_then_error_names_full_path( test_case: FreezeJsonErrorCase, ) -> None: with pytest.raises(ObservabilityValidationError) as error_info: - freeze_json(value=test_case.payload, path="payload") + freeze_lifecycle_json(value=test_case.payload, path="payload") assert str(error_info.value) == test_case.expected_error From 4432cc520ca271428b5077d6f67a85ab140145ca Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Thu, 15 Oct 2026 21:55:45 +0100 Subject: [PATCH 31/48] refactor: name the lifecycle payload rule type --- .../sqlbuild-core/src/lifecycle_events/_helpers/payloads.rs | 3 ++- crates/sqlbuild-core/src/lifecycle_events/types.rs | 4 ++++ 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/crates/sqlbuild-core/src/lifecycle_events/_helpers/payloads.rs b/crates/sqlbuild-core/src/lifecycle_events/_helpers/payloads.rs index a653721d5a..f792e8f4a6 100644 --- a/crates/sqlbuild-core/src/lifecycle_events/_helpers/payloads.rs +++ b/crates/sqlbuild-core/src/lifecycle_events/_helpers/payloads.rs @@ -14,6 +14,7 @@ use crate::lifecycle_events::constants::{ RETRY_SCHEDULED_EVENT, RUN_STARTED_EVENT, RUN_TERMINALS, STATEMENT_EVENT_PREFIX, }; use crate::lifecycle_events::models::{KnownEventFacts, PayloadValue}; +use crate::lifecycle_events::types::PayloadCheck; type Payload = [(String, PayloadValue)]; @@ -222,7 +223,7 @@ forbidden field(s): {}", )); } } - let checks: [(bool, fn(&[(String, PayloadValue)]) -> Option); 4] = [ + let checks: [(bool, PayloadCheck); 4] = [ ( event_type.starts_with(OPERATION_EVENT_PREFIX), operation_problem, diff --git a/crates/sqlbuild-core/src/lifecycle_events/types.rs b/crates/sqlbuild-core/src/lifecycle_events/types.rs index 3fcd3dd26c..2af438e15f 100644 --- a/crates/sqlbuild-core/src/lifecycle_events/types.rs +++ b/crates/sqlbuild-core/src/lifecycle_events/types.rs @@ -8,3 +8,7 @@ pub(crate) type CatalogEntry = ( &'static [&'static str], bool, ); + +/// One event-family payload rule: the first violation in a payload, if any. +pub(crate) type PayloadCheck = + fn(&[(String, crate::lifecycle_events::models::PayloadValue)]) -> Option; From 53723fc60f63384312c85ff965e86c6c8b518012 Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Thu, 15 Oct 2026 23:00:28 +0100 Subject: [PATCH 32/48] fix: use the single source freshness table name constant after integration --- .../src/bindings/_helpers/state_rows/state_tables.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/crates/sqlbuild-python/src/bindings/_helpers/state_rows/state_tables.rs b/crates/sqlbuild-python/src/bindings/_helpers/state_rows/state_tables.rs index 6499dad472..7f8bc455a9 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/state_rows/state_tables.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/state_rows/state_tables.rs @@ -6,7 +6,7 @@ use pyo3::prelude::{ }; use pyo3::types::{PyDict, PyString, PyStringMethods}; use pyo3::{pyfunction, wrap_pyfunction}; -use sqlbuild_state::freshness::constants::SOURCE_FRESHNESS_TABLE_NAME; +use sqlbuild_core::constants::SOURCE_FRESHNESS_TABLE_NAME; use sqlbuild_state::state_tables::main::create_table_sql::create_table_sql; use sqlbuild_state::state_tables::main::values_insert_sql::values_insert_sql; use sqlbuild_state::state_tables::models::TableColumn; From 01fad76f5444080e322a8df77ccd966831299373 Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Thu, 15 Oct 2026 23:19:15 +0100 Subject: [PATCH 33/48] chore: require one evidenced rule for every Python boundary stays entry --- python-boundary.toml | 355 +++++++++--------- scripts/python_boundary/_helpers/policy.py | 45 ++- scripts/python_boundary/constants.py | 9 +- scripts/python_boundary/models.py | 5 +- .../python_boundary/main/test_check.py | 68 +++- 5 files changed, 285 insertions(+), 197 deletions(-) diff --git a/python-boundary.toml b/python-boundary.toml index 45d844c679..add74a5e5b 100644 --- a/python-boundary.toml +++ b/python-boundary.toml @@ -1,800 +1,803 @@ # Python that stays once Rust hosts SQLBuild. Every other Python file under src/sqlbuild is # migration work that moves to Rust. `remaining_lines` must equal the current count of that work: # a PR that moves code lowers it, and adding movable Python fails the check. +# Each `[[stays]]` entry names the one rule that keeps it Python and a note with the evidence: +# 1 = executor or adapter runtime and the shapes it consumes; 2 = thin shim that invokes user +# Python; 3 = public facade (facade only); 4 = glue around a Python-only third-party library. # Native failures must surface as errors; `handled_native_errors` lists the reviewed exceptions. remaining_lines = 0 [[stays]] path = "executor" -reason = "executor" +rule = 1 note = "Runs plans against warehouses; a separate later decision." [[stays]] path = "adapters" -reason = "adapter" +rule = 1 note = "Built-in warehouse adapters that project adapters subclass and override." [[stays]] path = "adapter/contract" -reason = "adapter" +rule = 1 note = "BaseAdapter and StrictAdapter, the documented classes project adapters subclass." [[stays]] path = "adapter/discovery" -reason = "user-code" +rule = 2 note = "Imports project adapter modules." [[stays]] path = "adapter/relations" -reason = "adapter" +rule = 1 note = "Relation inspection wrappers adapters call, with per-connection caches; executed-statement classification and invalidation keys are native." [[stays]] path = "adapter/type_system" -reason = "adapter" +rule = 1 note = "Type normalisation adapters import; already a thin native wrapper." [[stays]] path = "provider" -reason = "user-code" +rule = 2 note = "Provider base class users subclass, and injection into user functions." [[stays]] path = "python_nodes" -reason = "user-code" +rule = 3 note = "Decorators that run when user modules are imported." [[stays]] path = "microbatches" -reason = "executor" +rule = 1 note = "Rule 1: the executor's direct-mode event store (warehouse I/O and retries), the event row codec and the event, scope and projection shapes it consumes. Coverage and replay projection, event IDs and state SQL are native (sqlbuild-state microbatches)." [[stays]] path = "rules" -reason = "public-api" +rule = 3 note = "Public custom-rule API and test helper." [[stays]] path = "diagnostics/_helpers/logging.py" -reason = "public-api" +rule = 4 note = "Installs SQLBuild's handlers on Python's logging, which users and integrations configure." [[stays]] path = "diagnostics/_helpers/redaction.py" -reason = "public-api" +rule = 4 note = "Collects redacted extra fields from a Python LogRecord; the redaction itself is native." [[stays]] path = "diagnostics/classes/diagnostic_record_redactor.py" -reason = "public-api" +rule = 4 note = "Redaction entry points that Python logging handlers call per record." [[stays]] path = "diagnostics/classes/diagnostics_console_formatter.py" -reason = "public-api" +rule = 4 note = "Rule 4 glue: the logging.Formatter subclass for the console stream; the line text is native (format_diagnostics_console_record) and only stdlib exception formatting stays here." [[stays]] path = "diagnostics/classes/dynamic_stderr_handler.py" -reason = "public-api" +rule = 4 note = "A logging.Handler that follows the current sys.stderr." [[stays]] path = "diagnostics/classes/invocation_diagnostic_routing.py" -reason = "public-api" +rule = 4 note = "Attaches and removes per-invocation logging handlers." [[stays]] path = "diagnostics/classes/log_record_route_filter.py" -reason = "public-api" +rule = 4 note = "A logging.Filter that routes records to destinations." [[stays]] path = "diagnostics/main" -reason = "public-api" +rule = 1 note = "Diagnostics entry points built on Python logging: configuration, context, debug and SQL logging." [[stays]] path = "diagnostics/_helpers/process_resources.py" -reason = "public-api" +rule = 4 note = "Samples process resources for debug logging through Python's resource module." [[stays]] path = "rule_engine/_helpers/host" -reason = "user-code" +rule = 2 note = "Rule 2: the subprocess host that loads and runs user rule modules (custom_host, custom_evaluation, tracked_facts, module_state, custom_host_launch); host fan-out, partitioning and CPU capacity run natively." [[stays]] path = "rule_engine/_helpers/engine/custom_payloads.py" -reason = "user-code" +rule = 2 note = "Reads user rule functions (source lines, qualified names, import closures) to describe custom rules for native selection." [[stays]] path = "rule_engine/_helpers/engine/_custom_catalogue.py" -reason = "user-code" +rule = 2 note = "Imports the project's custom rule modules and collects their @rule functions for the native catalogue." [[stays]] path = "rule_engine/classes/rule_context.py" -reason = "user-code" +rule = 3 note = "The context object user rules receive." [[stays]] path = "rule_engine/_helpers/engine/definition.py" -reason = "user-code" +rule = 2 note = "The @rule decorator implementation; inspects user rule functions and their signatures." [[stays]] path = "rule_engine/_helpers/engine/hermeticity.py" -reason = "user-code" +rule = 2 note = "Statically checks the imports of user rule source files." [[stays]] path = "rule_engine/_helpers/engine/rule_harness.py" -reason = "user-code" +rule = 2 note = "Public custom-rule test harness that runs user rules over a real compiled project." [[stays]] path = "rule_engine/_helpers/engine/fact_replay.py" -reason = "user-code" +rule = 2 note = "Rule 2: the fact digest scheme shared with the rule host (tracked_facts.py, custom_host.py); recomputes values through the user-facing fact views and encodes live Python objects with orjson." [[stays]] path = "rule_engine/main/define.py" -reason = "user-code" +rule = 3 note = "Public @rule decorator entry point." [[stays]] path = "rule_engine/main/evaluate_rule.py" -reason = "user-code" +rule = 3 note = "Public custom-rule test harness entry point." [[stays]] path = "rule_engine/classes/audit_facts.py" -reason = "user-code" +rule = 3 note = "Fact view user rules read through RuleContext; the Python API over native data." [[stays]] path = "rule_engine/classes/column_facts.py" -reason = "user-code" +rule = 3 note = "Fact view user rules read through RuleContext; the Python API over native data." [[stays]] path = "rule_engine/classes/contract_facts.py" -reason = "user-code" +rule = 3 note = "Fact view user rules read through RuleContext; the Python API over native data." [[stays]] path = "rule_engine/classes/declaration_facts.py" -reason = "user-code" +rule = 3 note = "Fact view user rules read through RuleContext; the Python API over native data." [[stays]] path = "rule_engine/classes/graph_facts.py" -reason = "user-code" +rule = 3 note = "Fact view user rules read through RuleContext; the Python API over native data." [[stays]] path = "rule_engine/classes/model_test_facts.py" -reason = "user-code" +rule = 3 note = "Fact view user rules read through RuleContext; the Python API over native data." [[stays]] path = "rule_engine/classes/project_facts.py" -reason = "user-code" +rule = 3 note = "Fact view user rules read through RuleContext; the Python API over native data." [[stays]] path = "rule_engine/classes/sql_facts.py" -reason = "user-code" +rule = 3 note = "Fact view user rules read through RuleContext; the Python API over native data." [[stays]] path = "rule_engine/classes/project_tree.py" -reason = "user-code" +rule = 3 note = "Project-tree fact view user rules read through RuleContext." [[stays]] path = "rule_engine/classes/sql_document.py" -reason = "user-code" +rule = 4 note = "Lazy Polyglot SQL document user rules read through RuleContext." [[stays]] path = "rule_engine/classes/fact_reads.py" -reason = "user-code" +rule = 2 note = "Records the fact-view reads of one user rule evaluation." [[stays]] path = "rule_engine/classes/shared_fact_encoder.py" -reason = "user-code" +rule = 2 note = "Encodes the Python fact-view values user rules read, for their digests." [[stays]] path = "runtime/observability" -reason = "public-api" +rule = 1 note = "Documented observability API used throughout the executor: lifecycle emitters, dispatcher, scopes and envelope shapes. The event catalogue, payload validation and JSON freezing are native (sqlbuild-core lifecycle_events)." [[stays]] path = "runtime/execution_history/__init__.py" -reason = "public-api" +rule = 3 note = "Package marker of the public execution history shapes below; storage, projection and validation are native (sqlbuild-state execution_history)." [[stays]] path = "runtime/execution_history/models.py" -reason = "public-api" +rule = 3 note = "Rule 3 shapes: StoredEvent, filter, page and RunRecord dataclasses re-exported by sqlbuild.execution_history; __post_init__ delegates validation to native validate_history_* functions." [[stays]] path = "runtime/execution_history/types.py" -reason = "public-api" +rule = 3 note = "Rule 3 shapes: EventFamily, RunStatus, cursor aliases and the storage protocols re-exported by sqlbuild.execution_history." [[stays]] path = "runtime/execution_history/exceptions.py" -reason = "public-api" +rule = 3 note = "Rule 3 shapes: public storage errors re-exported by sqlbuild.execution_history and raised by the native backends." [[stays]] path = "runtime/execution_limits" -reason = "executor" +rule = 1 note = "Deadline the executor scheduler and warehouse cursors enforce." [[stays]] path = "runtime/compute_logs/classes" -reason = "user-code" +rule = 1 note = "Rule 1 capture shim: sys.stdout/sys.stderr tees and a logging handler that record Python output from user code, adapters and the executor (_dispatch_with_compute_logs.py, compile_command/reuse_attempt.rs); storage itself is native LocalFilesystemComputeLogStorage." [[stays]] path = "runtime/compute_logs/main" -reason = "user-code" +rule = 1 note = "Rule 1 capture shim entry: installs the stream tees around the CLI command (_dispatch_with_compute_logs.py)." [[stays]] path = "runtime/compute_logs/_helpers" -reason = "user-code" +rule = 1 note = "Rule 1 tee shim: how much of a write the wrapped Python stream accepted (text_tee.py, binary_tee.py)." [[stays]] path = "runtime/compute_logs/__init__.py" -reason = "public-api" +rule = 3 note = "Package marker of the public compute log shapes below." [[stays]] path = "runtime/compute_logs/models.py" -reason = "public-api" +rule = 3 note = "Rule 3 shapes: capture metadata and result dataclasses re-exported by sqlbuild.compute_logs." [[stays]] path = "runtime/compute_logs/types.py" -reason = "public-api" +rule = 3 note = "Rule 3 shapes: ComputeLogStream and the ComputeLogStorage protocol re-exported by sqlbuild.compute_logs." [[stays]] path = "runtime/compute_logs/exceptions.py" -reason = "public-api" +rule = 3 note = "Rule 3 shapes: public storage errors re-exported by sqlbuild.compute_logs and raised by the native storage." [[stays]] path = "runtime/compute_logs/constants.py" -reason = "public-api" +rule = 1 note = "Rule 1 capture shim constants: the SQL log record field and redaction message the Python diagnostic log handler uses; format version, retention and read limits are native (sqlbuild._native)." [[stays]] path = "runtime/event_exporting/__init__.py" -reason = "public-api" +rule = 3 note = "Package marker of the public sink declarations below; the dispatcher, queue, accounting and command scope are native EventExporterDispatcher/EventExporterCommandScope." [[stays]] path = "runtime/event_exporting/main/__init__.py" -reason = "public-api" +rule = 3 note = "Package marker of the public sink decorators and the user-sink invocation shims below." [[stays]] path = "runtime/event_exporting/main/lifecycle_event_sink.py" -reason = "public-api" +rule = 3 note = "Rule 3 facade: the @lifecycle_event_sink decorator re-exported by sqlbuild.sinks, run when user sink modules are imported." [[stays]] path = "runtime/event_exporting/main/command_output_sink.py" -reason = "public-api" +rule = 3 note = "Rule 3 facade: the @command_output_sink decorator re-exported by sqlbuild.sinks, run when user sink modules are imported." [[stays]] path = "runtime/event_exporting/main/get_lifecycle_event_sink_definition.py" -reason = "public-api" +rule = 3 note = "Rule 3 facade: reads the decorator's definition off a user function; re-exported by sqlbuild.sinks." [[stays]] path = "runtime/event_exporting/main/get_command_output_sink_definition.py" -reason = "public-api" +rule = 3 note = "Rule 3 facade: reads the decorator's definition off a user function; re-exported by sqlbuild.sinks." [[stays]] path = "runtime/event_exporting/main/_invoke_lifecycle_event_sink.py" -reason = "user-code" +rule = 2 note = "Rule 2 shim: the daemon delivery thread's target that calls one user lifecycle sink and reports how it ended to the native dispatcher, so user code never runs above native frames that interpreter finalization would abort (event_exporting/workers.rs)." [[stays]] path = "runtime/event_exporting/main/_run_command_output_sinks.py" -reason = "user-code" +rule = 2 note = "Rule 2 shim: target of each dispatcher's single command-output sink thread; waits on a Python queue and calls user sinks in order on that one thread (provider thread affinity), reporting each outcome through a Python queue, so no native frame is on that thread when finalization can end it (event_exporting/dispatcher.rs)." [[stays]] path = "runtime/event_exporting/models.py" -reason = "public-api" +rule = 3 note = "Rule 3 shape: LifecycleEventSinkDefinition re-exported by sqlbuild.sinks." [[stays]] path = "runtime/event_exporting/types.py" -reason = "public-api" +rule = 3 note = "Rule 3 shape: LifecycleEventKind re-exported by sqlbuild.sinks." [[stays]] path = "runtime/event_exporting/exceptions.py" -reason = "public-api" +rule = 3 note = "Rule 3 shapes: errors the public decorators raise and the native dispatcher raises." [[stays]] path = "runtime/event_exporting/constants.py" -reason = "public-api" +rule = 3 note = "Rule 3 shapes: decorator attribute names, sink kinds and severities the public decorators and discovery read." [[stays]] path = "runtime/output_capture/classes" -reason = "user-code" +rule = 1 note = "Rule 1 capture shim: sys.stdout/sys.stderr tees forwarding accepted writes from user code, adapters and the executor to the native OutputCaptureDispatcher." [[stays]] path = "runtime/output_capture/main" -reason = "user-code" +rule = 1 note = "Rule 1 capture shim entry: swaps sys.stdout/sys.stderr for the tees around one CLI command (Rust cli/observability.rs)." [[stays]] path = "runtime/output_capture/__init__.py" -reason = "public-api" +rule = 3 note = "Package marker of the public command-output shapes below." [[stays]] path = "runtime/output_capture/models.py" -reason = "public-api" +rule = 3 note = "Rule 3 shapes: CommandOutputRecord and sink definitions re-exported by sqlbuild.sinks; validation is native validate_command_output_record." [[stays]] path = "runtime/output_capture/types.py" -reason = "public-api" +rule = 3 note = "Rule 3 shapes: CommandOutputStream re-exported by sqlbuild.sinks, priorities and the exporter protocol." [[stays]] path = "runtime/output_capture/exceptions.py" -reason = "public-api" +rule = 3 note = "Rule 3 shapes: CommandOutputValidationError re-exported by sqlbuild.sinks and raised by the native records." [[stays]] path = "runtime/output_capture/constants.py" -reason = "public-api" +rule = 3 note = "Rule 3 shapes: the command-output sink parameter name and producer version; INVOCATION_CONTEXT_ENV is re-exported from sqlbuild._native for integrations." [[stays]] path = "integrations/dagster" -reason = "public-api" +rule = 4 note = "Documented orchestrator integration and rule 4 Dagster glue: translator invocation, context reads, subprocess streaming and Dagster event construction; CLI arguments, selector logs, payload and failure parsing, asset/check/selection projection of the DAG artifact are native." [[stays]] path = "integrations/rivers" -reason = "public-api" +rule = 4 note = "Documented orchestrator integration and rule 4 Rivers glue: translator invocation and AssetDef construction; the asset projection of the DAG artifact is native." [[stays]] path = "integrations/dlt" -reason = "user-code" +rule = 4 note = "Loader runtime that runs inside user loaders; synthetic loader record building and naming are native." [[stays]] path = "integrations/ingestr" -reason = "user-code" +rule = 1 note = "Loader runtime that runs the ingestr subprocess inside the executor load phase; synthetic loader record building, naming, destination URIs and command lines are native." [[stays]] path = "integrations/dbt/_helpers/profile/load.py" -reason = "user-code" +rule = 4 note = "Rule 4: PyYAML glue only; parses dbt_project.yml and profiles.yml with yaml.safe_load and reports YAMLError as DbtProfileError. Existence, mapping, field and target validation and output selection are native (dbt_interop/profile_load.rs)." [[stays]] path = "integrations/dbt/_helpers/profile/render.py" -reason = "user-code" +rule = 4 note = "Renders the selected dbt profile output with Jinja, env_var and var as dbt does." [[stays]] path = "cli/commands/main/entrypoint/_dispatch_with_compute_logs.py" -reason = "user-code" +rule = 1 note = "Captures the command's stdout, stderr and logging into the local compute log around user code." [[stays]] path = "cli/commands/_helpers/entry/compute_log_diagnostics.py" -reason = "user-code" +rule = 1 note = "Logs compute log capture failures through Python logging during capture." [[stays]] path = "cli/compile_reuse/_helpers/provider_settings.py" -reason = "user-code" +rule = 4 note = "Inspects user provider settings classes for the environment and files they read." [[stays]] path = "cli/compile_reuse/_helpers/startup_inputs.py" -reason = "user-code" +rule = 2 note = "Interpreter and site-path inputs user modules import from." [[stays]] path = "cli/entry/main/entry.py" -reason = "public-api" +rule = 3 note = "The installed sqlbuild-python console entry point." [[stays]] path = "cli/entry/main/launched.py" -reason = "public-api" +rule = 3 note = "The Python entry point the native sqb executable hands invocations to." [[stays]] path = "cli/entry/_helpers/streams.py" -reason = "public-api" +rule = 3 note = "Reconfigures the Python process's standard streams for the console entry points." [[stays]] path = "compiler/python_nodes/_helpers/identity.py" -reason = "user-code" +rule = 2 note = "Identity hashing over live user functions (source and globals)." [[stays]] path = "compiler/macro_bridge/_helpers/store_environment.py" -reason = "user-code" +rule = 2 note = "Fingerprints the installed packages, import path and live module files that project macros import from; the macro call store cache key. The interpreter stamp it shares with compile reuse is native (interpreter_identity)." [[stays]] path = "rule_engine/classes/runtime_guard.py" -reason = "user-code" +rule = 2 note = "Guards custom-rule execution against side effects in user rule code." [[stays]] path = "rule_engine/classes/module_state_monitor.py" -reason = "user-code" +rule = 2 note = "Watches user rule modules for state changes between custom-rule runs." [[stays]] path = "compiler/compile/_helpers/render/user_macros.py" -reason = "user-code" +rule = 2 note = "Loads and executes project macro modules, and builds the declaration and enum mappings user macros read." [[stays]] path = "compiler/macro_bridge/classes/macro_bridge.py" -reason = "user-code" +rule = 2 note = "Tracks the loaded user modules macro call results depend on, and runs a render stage through the bridge." [[stays]] path = "compiler/python_nodes/_helpers/user_inventory.py" -reason = "user-code" +rule = 2 note = "Reads loaded user node functions: identities over live functions and decorated dependency names; graph construction is native." [[stays]] path = "compiler/discovery/_helpers/filesystem/project_modules.py" -reason = "user-code" +rule = 2 note = "Rule 2 shim: imports project Python node, hook, provider, sink and materialization modules and reads live members, signatures, provider parameter annotations and source lines; record building, provider naming and validation, settings errors, factory results and duplicate checks are native." [[stays]] path = "compiler/discovery/_helpers/validation/discovery.py" -reason = "user-code" +rule = 2 note = "Reads decorator definitions from loaded project Python node functions." [[stays]] path = "compiler/discovery/main/runtime_extensions.py" -reason = "user-code" +rule = 2 note = "Binds imported project providers to event exporter and sink declarations." [[stays]] path = "compiler/compile/_helpers/diagnostics/python_resources.py" -reason = "user-code" +rule = 2 note = "Reads docstrings and source lines of loaded project Python functions and provider classes." [[stays]] path = "compiler/discovery/_helpers/filesystem/command_output_sinks.py" -reason = "user-code" +rule = 2 note = "Inspects loaded user sink functions." [[stays]] path = "compiler/pipeline/_helpers/materializations.py" -reason = "user-code" +rule = 2 note = "Imports user materialization modules." [[stays]] path = "compiler/macro_bridge/main" -reason = "user-code" +rule = 2 note = "Macro bridge entry points for the running render stage that executes user macros." [[stays]] path = "compiler/compile/_helpers/macro_bridge/call_store.py" -reason = "user-code" +rule = 2 note = "Hands the running macro bridge its user macro call store." [[stays]] path = "compiler/python_nodes/main/identity.py" -reason = "user-code" +rule = 2 note = "Identity entry point over live user node functions." [[stays]] path = "compiler/python_nodes/main/_hook_identities.py" -reason = "user-code" +rule = 2 note = "Builds identities of loaded user hook functions." [[stays]] path = "compiler/auditing/main/above.py" -reason = "public-api" +rule = 3 note = "Audit bound helper project audit factories call." [[stays]] path = "compiler/auditing/main/below.py" -reason = "public-api" +rule = 3 note = "Audit bound helper project audit factories call." [[stays]] path = "compiler/auditing/main/outside.py" -reason = "public-api" +rule = 3 note = "Audit bound helper project audit factories call." [[stays]] path = "compiler/auditing/main/audit_factory.py" -reason = "public-api" +rule = 3 note = "Decorator that declares project audit factories." [[stays]] path = "compiler/auditing/main/get_audit_factory_definition.py" -reason = "public-api" +rule = 3 note = "Reads the declaration attached to a project audit factory." [[stays]] path = "compiler/sql_analysis/main/import_polyglot.py" -reason = "public-api" +rule = 4 note = "Imports the Polyglot Python package for the Python API." [[stays]] path = "compiler/sql_analysis/main/import_polyglot_sql.py" -reason = "public-api" +rule = 4 note = "Imports the Polyglot SQL Python package for the Python API." [[stays]] path = "compiler/sql_analysis/_helpers/polyglot.py" -reason = "public-api" +rule = 4 note = "Wraps the Polyglot Python package with complexity guards." [[stays]] path = "assets.py" -reason = "public-api" -note = "Public facade." +rule = 3 +note = "Public facade: re-exports plus one-line delegating wrappers (asset, get_asset_definition)." [[stays]] path = "audits.py" -reason = "public-api" -note = "Public facade." +rule = 3 +note = "Public facade: re-exports only." [[stays]] path = "checks.py" -reason = "public-api" -note = "Public facade." +rule = 3 +note = "Public facade: re-exports plus one-line delegating wrappers (check, get_check_definition)." [[stays]] path = "compute_logs.py" -reason = "public-api" -note = "Public facade." +rule = 3 +note = "Public facade: re-exports plus one-line delegating wrappers (open_local_compute_log_storage)." [[stays]] path = "execution_history.py" -reason = "public-api" -note = "Public facade." +rule = 3 +note = "Public facade: re-exports plus one-line delegating wrappers (append_and_project, canonical_event_id, canonical_event_content, project_runs, validate_page_limit)." [[stays]] path = "factories.py" -reason = "public-api" -note = "Public facade." +rule = 3 +note = "Public facade: re-exports plus one-line delegating wrappers (factory, get_factory_definition)." [[stays]] path = "functions.py" -reason = "public-api" -note = "Public facade." +rule = 3 +note = "Public facade: re-exports only." [[stays]] path = "hooks.py" -reason = "public-api" -note = "Public facade." +rule = 3 +note = "Public facade: re-exports plus one-line delegating wrappers (hook, get_hook_definition)." [[stays]] path = "loaders.py" -reason = "public-api" -note = "Public facade." +rule = 3 +note = "Public facade: re-exports only." [[stays]] path = "observability.py" -reason = "public-api" -note = "Public facade." +rule = 3 +note = "Public facade: re-exports only." [[stays]] path = "postgres_history.py" -reason = "public-api" -note = "Public facade." +rule = 3 +note = "Public facade: re-exports only." [[stays]] path = "providers.py" -reason = "public-api" -note = "Public facade." +rule = 3 +note = "Public facade: re-exports only." [[stays]] path = "refs.py" -reason = "public-api" -note = "Public facade." +rule = 3 +note = "Public facade: re-exports plus one-line delegating wrappers (model, source, seed)." [[stays]] path = "retries.py" -reason = "public-api" -note = "Public facade." +rule = 3 +note = "Public facade: re-exports only." [[stays]] path = "sinks.py" -reason = "public-api" -note = "Public facade." +rule = 3 +note = "Public facade: re-exports only." [[stays]] path = "sqlite_history.py" -reason = "public-api" -note = "Public facade." +rule = 3 +note = "Public facade: re-exports only." [[stays]] path = "tasks.py" -reason = "public-api" -note = "Public facade." +rule = 3 +note = "Public facade: re-exports plus one-line delegating wrappers (task, get_task_definition)." [[stays]] path = "cost/models.py" -reason = "executor" +rule = 1 note = "Shape rule: data records only (StatementExecutionTelemetry, CostResourceContext, RunCostSummary), no logic; consumed by adapters/snowflake/classes/snowflake_adapter.py, adapters/snowflake/classes/snowflake_cursor.py, executor/build/classes/build_scheduler.py and 2 more." [[stays]] path = "cli/compile_reuse/models.py" -reason = "user-code" +rule = 4 note = "Shape rule: data records only (CompileReuseRequest, SettingsEnvironmentInputs, SettingsInputsResult), no logic; consumed by cli/compile_reuse/_helpers/provider_settings.py, and read natively by the compile reuse check." [[stays]] path = "compiler/source_freshness/constants.py" -reason = "adapter" +rule = 1 note = "Shape rule: source freshness column names and adapter column types, no logic; SOURCE_FRESHNESS_TABLE_NAME is re-exported from sqlbuild._native." [[stays]] path = "compiler/auditing/types.py" -reason = "executor" +rule = 1 note = "Shape rule: data enums and aliases only (AuditSeverity, AuditRunScope, AuditAttachmentKind, AuditEvaluationMode, ...), no logic; consumed by audits.py, compiler/auditing/main/above.py, compiler/auditing/main/below.py and 17 more." [[stays]] path = "compiler/python_nodes/types.py" -reason = "executor" +rule = 1 note = "Shape rule: data enums and aliases only (PythonNodeKind, PythonNodeStatus, SkipMode, PythonNodeFanInAction), no logic; consumed by assets.py, assets.pyi, executor/load/_helpers/dag_runtime.py and 16 more." [[stays]] path = "compiler/macro_bridge/constants.py" -reason = "user-code" +rule = 2 note = "Shape rule: data constants only (ACTIVE_MACRO_BRIDGE, COMPILES_STARTED, STORE_MODULE_PATHS, MACRO_CALL_STORE_FILE_NAME, ...), no logic; consumed by compiler/macro_bridge/_helpers/store_environment.py, compiler/macro_bridge/classes/macro_bridge.py, compiler/macro_bridge/main/_claim_first_compile.py and 2 more." [[stays]] path = "compiler/references/models.py" -reason = "executor" +rule = 1 note = "Shape rule: data records only (RelationName, ProjectRelation, ProjectRelationIndex), no logic; consumed by executor/python_nodes/classes/runtime_relation_guard.py." [[stays]] path = "runtime/contracts/types.py" -reason = "executor" +rule = 1 note = "Shape rule: data enums and aliases only (ExecutionResourceKind, ConnectionElapsedCallback, NodeStartCallback), no logic; consumed by executor/build/_helpers/scheduler.py, executor/build/_helpers/source_node.py, executor/build/classes/build_scheduler.py and 15 more." [[stays]] path = "errors/contracts/exceptions.py" -reason = "executor" +rule = 1 note = "Shape rule: data exceptions only (SharedInputError, ExecutorInputError), no logic; consumed by executor/audit_results/_helpers/sql.py, executor/build/_helpers/external_source_loads.py, executor/build/_helpers/retention.py and 53 more." [[stays]] path = "compiler/fingerprints/constants.py" -reason = "executor" +rule = 1 note = "Shape rule: Python-only node type names (NODE_TYPE_HOOK, NODE_TYPE_QUERY_DIFF_ARTIFACT), no logic; table and column names are native (sqlbuild._native)." [[stays]] path = "diagnostics/models.py" -reason = "public-api" +rule = 4 note = "Shape rule: data records only (ProcessResourceUsage, DiagnosticRoutingOptions), no logic; consumed by cli/commands/main/entrypoint/_dispatch_with_compute_logs.py, diagnostics/classes/invocation_diagnostic_routing.py, diagnostics/main/_log_process_resources.py and 3 more." [[stays]] path = "compiler/dag/types.py" -reason = "public-api" +rule = 4 note = "Shape rule: data enums and aliases only (NodeKind), no logic; consumed by integrations/dagster/constants.py, integrations/rivers/constants.py." [[stays]] path = "runtime/contracts/models.py" -reason = "executor" +rule = 1 note = "Shape rule: data records only (ConnectionHooks), no logic; consumed by executor/pipeline/_helpers/scenario.py." [[stays]] path = "cursor_algebra/types.py" -reason = "executor" +rule = 1 note = "Shape rule: data enums and aliases only (CursorScalar, BoundSentinel, Bound), no logic; consumed by executor/load/_helpers/cursors.py, executor/load/main/_execute.py, executor/run/_helpers/materializations/microbatch.py and 2 more." [[stays]] path = "cost/types.py" -reason = "adapter" +rule = 1 note = "Shape rule: data enums and aliases only (CostCapability, CostStatus), no logic; consumed by adapters/snowflake/classes/snowflake_adapter.py." [[stays]] path = "compiler/lineage/types.py" -reason = "adapter" +rule = 1 note = "Shape rule: data enums and aliases only (InferredNullability), no logic; consumed by adapter/contract/types.py, adapter/type_system/_helpers/inference_rules.py, adapter/type_system/main/conditional_result_nullability.py and 2 more." [[stays]] path = "compiler/macro_bridge/models.py" -reason = "user-code" +rule = 2 note = "Shape rule: data records only (ModuleSources), no logic; consumed by compiler/macro_bridge/_helpers/store_environment.py, compiler/macro_bridge/classes/macro_bridge.py." [[stays]] path = "integrations/dbt/exceptions.py" -reason = "user-code" +rule = 4 note = "Shape rule: data exceptions only (DbtProfileError), no logic; consumed by integrations/dbt/_helpers/profile/load.py, integrations/dbt/_helpers/profile/render.py." [[stays]] path = "diagnostics/constants.py" -reason = "public-api" +rule = 4 note = "Shape rule: data constants only (LOGGER_ROOT_NAME, LOGGER_PREFIX, SQL_DIGEST_FIELD, SQL_TEXT_FIELD, ...), no logic; consumed by diagnostics/_helpers/logging.py, diagnostics/_helpers/redaction.py, diagnostics/classes/invocation_diagnostic_routing.py and 2 more." [[stays]] path = "cursor_algebra/exceptions.py" -reason = "executor" +rule = 1 note = "Shape rule: data exceptions only (CursorAlgebraError), no logic; consumed by executor/run/models.py." [[stays]] path = "spec/contracts/constants.py" -reason = "adapter" +rule = 1 note = "Shape rule: data constants only (DEFAULT_SEED_CSV_SETTINGS), no logic; consumed by adapter contract classes." [[stays]] path = "cli/entry/constants.py" -reason = "public-api" +rule = 3 note = "Shape rule: data constants only (NATIVE_ALLOCATOR_PURGE_DELAY_VARIABLE, NATIVE_ALLOCATOR_PURGE_DELAY_MILLISECONDS, OUTPUT_STREAM_ENCODING, OUTPUT_STREAM_ERRORS, ...), no logic; consumed by cli/entry/_helpers/streams.py, cli/entry/main/entry.py, cli/entry/main/launched.py." [[stays]] path = "sql_values/exceptions.py" -reason = "adapter" +rule = 1 note = "Shape rule: data exceptions only (SqlValueRenderingError), no logic; consumed by adapter/contract/classes/base_adapter.py, adapter/contract/exceptions.py." [[stays]] path = "cli/output/constants.py" -reason = "public-api" +rule = 4 note = "Shape rule: re-export only (INTEGRATION_RESULT_PATH_ENV, from sqlbuild._native), no logic; consumed by integrations/dagster/_helpers/invocation_factory.py." [[stays]] path = "compiler/compile/types.py" -reason = "executor" +rule = 1 note = "Shape rule: typing protocols, enums and aliases only (CompiledProjectView, CompiledModelView, CompiledResourceType, AuditInput, ...), no logic; consumed by executor/pipeline/_helpers/testing.py, executor/scenario/_helpers/local/execution.py and project audit factories that annotate AuditInput." [[stays]] path = "rule_engine/types.py" -reason = "public-api" +rule = 3 note = "Shape rule: data protocol and aliases only (RuleContext, RuleCheck, RuleOptionValue, FactKey, CustomRulePlan), no logic; re-exported by rules/__init__.py and consumed by rule_engine/main/define.py, rule_engine/_helpers/host/custom_host.py, rule_engine/_helpers/engine/fact_replay.py and 4 more." [[stays]] path = "rule_engine/constants.py" -reason = "user-code" +rule = 2 note = "Shape rule: custom-rule host and fact-name constants (CUSTOM_HOST_MAX_TRACKED_READS, CUSTOM_RULE_IMPORT_ROOTS, FACT_PROJECT_MODELS, ...) used by the Python custom-rule host, no logic; versions, launch module and shared keys are native (sqlbuild._native)." [[stays]] path = "compiler/compile/exceptions.py" -reason = "user-code" +rule = 2 note = "Shape rule: data exceptions only (MacroDeclarationLookupError, a CompileInputError that is also a KeyError, which a native error class with one parent cannot express), no logic; consumed by compiler/compile/_helpers/render/user_macros.py." [[stays]] path = "compiler/sql_analysis/constants.py" -reason = "public-api" +rule = 4 note = "Rule 4: polyglot_sql glue only; the entry point, call-shape and option names (POLYGLOT_ANALYZE_QUERY_ENTRY_POINT, POLYGLOT_COMPLEXITY_GUARD_OPTION, POLYGLOT_MAX_FUNCTION_CALL_DEPTH, ...) that compiler/sql_analysis/_helpers/polyglot.py uses to wrap the polyglot_sql Python package." [[stays]] path = "compiler/compile/models.py" -reason = "public-api" +rule = 3 note = "Shape rule: re-export only (MacroContext, now hosted natively), no logic; project macro modules import it from this path to annotate their ctx parameter, as tests/e2e/fixtures/waffle_shop/models/marts/_macros/time.py does." diff --git a/scripts/python_boundary/_helpers/policy.py b/scripts/python_boundary/_helpers/policy.py index a3c2538b46..8bcd3ac7c0 100644 --- a/scripts/python_boundary/_helpers/policy.py +++ b/scripts/python_boundary/_helpers/policy.py @@ -2,11 +2,18 @@ from __future__ import annotations +import re import tomllib from pathlib import Path from typing import Any -from scripts.python_boundary.constants import ESCAPING_PATH_PARTS, SOURCE_ROOT, STAY_REASONS +from scripts.python_boundary.constants import ( + ESCAPING_PATH_PARTS, + NOTE_RULE_PATTERN, + SOURCE_ROOT, + STAY_KEYS, + STAY_RULES, +) from scripts.python_boundary.models import BoundaryPolicy, ReviewedNativeHandler, StayEntry @@ -19,8 +26,9 @@ def read_boundary_policy(path: Path) -> BoundaryPolicy: stays=tuple( StayEntry( path=str(entry.get("path", "")), - reason=str(entry.get("reason", "")), + rule=entry.get("rule"), note=str(entry.get("note", "")), + unknown_keys=tuple(sorted(set(entry) - STAY_KEYS)), ) for entry in document.get("stays", ()) ), @@ -49,14 +57,33 @@ def get_policy_errors(*, root: Path, policy: BoundaryPolicy) -> tuple[str, ...]: errors.append(f"stays entry {entry.path!r} must name a module under {SOURCE_ROOT}.") elif not (source / entry.path).exists(): errors.append(f"stays entry {entry.path!r} does not exist under {SOURCE_ROOT}.") - if entry.reason not in STAY_REASONS: - errors.append( - f"stays entry {entry.path!r} has reason {entry.reason!r}; " - f"use one of {', '.join(STAY_REASONS)}." - ) - if not entry.note.strip(): - errors.append(f"stays entry {entry.path!r} needs a note explaining why it stays.") + errors.extend(_stay_rule_errors(entry)) for handler in policy.handled_native_errors: if not handler.note.strip(): errors.append(f"handled_native_errors entry {handler.location!r} needs a note.") return tuple(errors) + + +def _stay_rule_errors(entry: StayEntry) -> tuple[str, ...]: + """Errors for a stays entry without exactly one valid rule backed by a note.""" + errors: list[str] = [] + choices: str = "; ".join(f"{number} = {meaning}" for number, meaning in STAY_RULES.items()) + if entry.unknown_keys: + errors.append( + f"stays entry {entry.path!r} has unknown key(s) {', '.join(entry.unknown_keys)}; " + "an entry has only path, rule and note." + ) + rule: object = entry.rule + if rule is None: + errors.append(f"stays entry {entry.path!r} needs a rule: one of {choices}.") + elif isinstance(rule, bool) or not isinstance(rule, int) or rule not in STAY_RULES: + errors.append(f"stays entry {entry.path!r} has rule {rule!r}; use one of {choices}.") + if not entry.note.strip(): + errors.append(f"stays entry {entry.path!r} needs a note with the evidence for its rule.") + cited: set[int] = {int(number) for number in re.findall(NOTE_RULE_PATTERN, entry.note)} + if isinstance(rule, int) and not isinstance(rule, bool) and cited - {rule}: + errors.append( + f"stays entry {entry.path!r} has rule {rule} but its note cites rule " + f"{', '.join(str(number) for number in sorted(cited - {rule}))}." + ) + return tuple(errors) diff --git a/scripts/python_boundary/constants.py b/scripts/python_boundary/constants.py index ce4aa5a72d..2c1eac8012 100644 --- a/scripts/python_boundary/constants.py +++ b/scripts/python_boundary/constants.py @@ -6,5 +6,12 @@ NATIVE_MARKER: str = "native" SUPPRESS_NAME: str = "suppress" MODULE_SCOPE: str = "" -STAY_REASONS: tuple[str, ...] = ("user-code", "adapter", "executor", "public-api") +STAY_RULES: dict[int, str] = { + 1: "executor or adapter runtime and the shapes it consumes", + 2: "thin shim that invokes user Python", + 3: "public facade", + 4: "glue around a Python-only third-party library", +} +STAY_KEYS: frozenset[str] = frozenset({"path", "rule", "note"}) +NOTE_RULE_PATTERN: str = r"\b[Rr]ule (\d+)\b" ESCAPING_PATH_PARTS: frozenset[str] = frozenset({".", ".."}) diff --git a/scripts/python_boundary/models.py b/scripts/python_boundary/models.py index 0abf0f0f5b..86a2ba6d93 100644 --- a/scripts/python_boundary/models.py +++ b/scripts/python_boundary/models.py @@ -7,11 +7,12 @@ @dataclass(frozen=True) class StayEntry: - """A module or package that stays Python, and why.""" + """A module or package that stays Python, the rule that keeps it, and the evidence.""" path: str - reason: str + rule: object note: str + unknown_keys: tuple[str, ...] = () @dataclass(frozen=True) diff --git a/tests/unit/scripts/python_boundary/main/test_check.py b/tests/unit/scripts/python_boundary/main/test_check.py index cd5071b1a3..2e41167a10 100644 --- a/tests/unit/scripts/python_boundary/main/test_check.py +++ b/tests/unit/scripts/python_boundary/main/test_check.py @@ -50,7 +50,7 @@ " return native_project_graph(project)\n" " return None\n" ) -STAYS: str = '[[stays]]\npath = "executor"\nreason = "executor"\nnote = "Runs plans."\n' +STAYS: str = '[[stays]]\npath = "executor"\nrule = 1\nnote = "Rule 1: runs plans."\n' REVIEWED: str = ( '[[handled_native_errors]]\nlocation = "compiler/keys.py::digest"\nnote = "No key."\n' ) @@ -149,9 +149,7 @@ def test_given_policy_matching_source_when_checking_then_passes( PythonBoundaryTestCase( description="a staying path escapes the source root", source_files={"compiler/plan.py": "a = 1\n"}, - policy=( - 'remaining_lines = 0\n[[stays]]\npath = ".."\nreason = "executor"\nnote = "No."\n' - ), + policy=('remaining_lines = 0\n[[stays]]\npath = ".."\nrule = 1\nnote = "No."\n'), expected_messages=("stays entry '..' must name a module under src/sqlbuild.",), ), PythonBoundaryTestCase( @@ -161,16 +159,68 @@ def test_given_policy_matching_source_when_checking_then_passes( expected_messages=("'compiler/keys.py::digest' no longer matches any handler",), ), PythonBoundaryTestCase( - description="a staying module is missing, unexplained and has an unknown reason", + description="a staying module is missing and has no rule or note", source_files={"compiler/plan.py": "a = 1\n"}, + policy='remaining_lines = 1\n[[stays]]\npath = "planner"\nnote = ""\n', + expected_messages=( + "stays entry 'planner' does not exist under src/sqlbuild.", + "stays entry 'planner' needs a rule: one of 1 = executor or adapter runtime", + "stays entry 'planner' needs a note with the evidence for its rule.", + ), + ), + PythonBoundaryTestCase( + description="a staying module has a rule outside 1 to 4", + source_files={"compiler/plan.py": "a = 1\n", "executor/run.py": "c = 3\n"}, + policy=( + 'remaining_lines = 1\n[[stays]]\npath = "executor"\nrule = 5\nnote = "Runs."\n' + ), + expected_messages=("stays entry 'executor' has rule 5; use one of 1 = ",), + ), + PythonBoundaryTestCase( + description="a staying module has a non-integer rule", + source_files={"compiler/plan.py": "a = 1\n", "executor/run.py": "c = 3\n"}, + policy=( + 'remaining_lines = 1\n[[stays]]\npath = "executor"\nrule = "1"\nnote = "Runs."\n' + ), + expected_messages=("stays entry 'executor' has rule '1'; use one of 1 = ",), + ), + PythonBoundaryTestCase( + description="a staying module has a boolean rule", + source_files={"compiler/plan.py": "a = 1\n", "executor/run.py": "c = 3\n"}, policy=( - 'remaining_lines = 1\n[[stays]]\npath = "planner"\nreason = "too hard"\nnote = ""\n' + 'remaining_lines = 1\n[[stays]]\npath = "executor"\nrule = true\nnote = "Runs."\n' ), + expected_messages=("stays entry 'executor' has rule True; use one of 1 = ",), + ), + PythonBoundaryTestCase( + description="a staying module has a whitespace-only note", + source_files={"compiler/plan.py": "a = 1\n", "executor/run.py": "c = 3\n"}, + policy=('remaining_lines = 1\n[[stays]]\npath = "executor"\nrule = 1\nnote = " "\n'), expected_messages=( - "stays entry 'planner' does not exist under src/sqlbuild.", - "stays entry 'planner' has reason 'too hard'", - "stays entry 'planner' needs a note explaining why it stays.", + "stays entry 'executor' needs a note with the evidence for its rule.", + ), + ), + PythonBoundaryTestCase( + description="a staying module still uses the replaced reason key", + source_files={"compiler/plan.py": "a = 1\n", "executor/run.py": "c = 3\n"}, + policy=( + 'remaining_lines = 1\n[[stays]]\npath = "executor"\nreason = "executor"\n' + 'note = "Runs."\n' + ), + expected_messages=( + "stays entry 'executor' has unknown key(s) reason; " + "an entry has only path, rule and note.", + "stays entry 'executor' needs a rule: one of ", + ), + ), + PythonBoundaryTestCase( + description="a staying module's note cites a different rule", + source_files={"compiler/plan.py": "a = 1\n", "executor/run.py": "c = 3\n"}, + policy=( + 'remaining_lines = 1\n[[stays]]\npath = "executor"\nrule = 1\n' + 'note = "Rule 4 glue around a library."\n' ), + expected_messages=("stays entry 'executor' has rule 1 but its note cites rule 4.",), ), ], ids=lambda case: case.description, From 315d0cc7d5328bb53baa2ad8f6d72b0236a950f1 Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Thu, 15 Oct 2026 23:28:52 +0100 Subject: [PATCH 34/48] refactor: drop constant re-export shims and host freshness columns natively - Point importers of INTEGRATION_RESULT_PATH_ENV, INVOCATION_CONTEXT_ENV and SOURCE_FRESHNESS_TABLE_NAME at sqlbuild._native and remove the re-exports under old module paths. - Define the source freshness columns and their state value types once in sqlbuild-state; the state table DDL, insert and latest-state SQL read them from Rust instead of a Python constants module. - Export the diagnostic logger prefix and SQL text log field from Rust, which the native console formatter already used. - Delete cli/output, compiler/source_freshness and runtime/compute_logs constants modules (and the then-empty packages) with their stays entries. --- .../src/compute_logs/constants.rs | 2 +- .../src/command_output/constants.rs | 2 +- .../src/diagnostic_text/constants.rs | 4 +- .../sqlbuild-core/src/diagnostic_text/mod.rs | 2 +- .../_helpers/contracts/state_literals.rs | 4 +- .../_helpers/diagnostics/console_records.rs | 3 +- .../_helpers/exported_constants/constants.rs | 3 ++ .../bindings/_helpers/freshness/state_io.rs | 18 +++++--- .../bindings/_helpers/freshness/support.rs | 1 - .../_helpers/state_rows/state_tables.rs | 26 ++++++----- .../sqlbuild-state/src/freshness/constants.rs | 19 +++++++- .../src/freshness/main/read_latest_sql.rs | 9 +++- python-boundary.toml | 4 ++ src/sqlbuild/_native.pyi | 2 + src/sqlbuild/cli/output/__init__.py | 0 src/sqlbuild/cli/output/constants.py | 3 -- .../compiler/source_freshness/__init__.py | 0 .../compiler/source_freshness/constants.py | 43 ------------------- src/sqlbuild/diagnostics/_helpers/logging.py | 2 +- .../diagnostics/_helpers/redaction.py | 4 +- .../classes/invocation_diagnostic_routing.py | 7 +-- .../classes/log_record_route_filter.py | 3 +- src/sqlbuild/diagnostics/constants.py | 2 - .../dagster/_helpers/invocation_factory.py | 8 ++-- .../classes/diagnostic_log_handler.py | 13 +++--- .../runtime/compute_logs/constants.py | 4 -- .../runtime/output_capture/constants.py | 2 - 27 files changed, 83 insertions(+), 107 deletions(-) delete mode 100644 src/sqlbuild/cli/output/__init__.py delete mode 100644 src/sqlbuild/cli/output/constants.py delete mode 100644 src/sqlbuild/compiler/source_freshness/__init__.py delete mode 100644 src/sqlbuild/compiler/source_freshness/constants.py delete mode 100644 src/sqlbuild/runtime/compute_logs/constants.py diff --git a/crates/sqlbuild-cli/src/compute_logs/constants.rs b/crates/sqlbuild-cli/src/compute_logs/constants.rs index 54650af368..f9ba294a61 100644 --- a/crates/sqlbuild-cli/src/compute_logs/constants.rs +++ b/crates/sqlbuild-cli/src/compute_logs/constants.rs @@ -1,4 +1,4 @@ -//! Python's `sqlbuild.runtime.compute_logs.constants` and storage file names. +//! Compute log format version, retention and read limits, and storage file names. pub const COMPUTE_LOG_FORMAT_VERSION: i64 = 1; pub const DEFAULT_RETENTION_COUNT: usize = 20; diff --git a/crates/sqlbuild-core/src/command_output/constants.rs b/crates/sqlbuild-core/src/command_output/constants.rs index 627d6f538e..57c7e05962 100644 --- a/crates/sqlbuild-core/src/command_output/constants.rs +++ b/crates/sqlbuild-core/src/command_output/constants.rs @@ -1,4 +1,4 @@ -//! Python's `sqlbuild.runtime.output_capture.constants` delivery limits and text bytes. +//! Command output delivery limits, record types and text bytes. pub const MIN_OUTPUT_RECORD_BYTES: i64 = 4; pub const FLUSH_INTERVAL_SECONDS: f64 = 0.25; diff --git a/crates/sqlbuild-core/src/diagnostic_text/constants.rs b/crates/sqlbuild-core/src/diagnostic_text/constants.rs index 59af01ed09..fa2933b9d4 100644 --- a/crates/sqlbuild-core/src/diagnostic_text/constants.rs +++ b/crates/sqlbuild-core/src/diagnostic_text/constants.rs @@ -1,7 +1,9 @@ //! Prefixes, statements and widths of `--debug` console lines. /// The logger prefix diagnostics console headers drop from SQLBuild logger names. -pub(crate) const DIAGNOSTIC_LOGGER_PREFIX: &str = "sqlbuild."; +pub const DIAGNOSTIC_LOGGER_PREFIX: &str = "sqlbuild."; +/// The log record attribute carrying a diagnostic's SQL text. +pub const SQL_TEXT_LOG_FIELD: &str = "sqlbuild_sql"; /// Statements a diagnostics console line shows inline instead of in a block. pub(crate) const INLINE_SQL_STATEMENTS: [&str; 3] = ["BEGIN", "COMMIT", "ROLLBACK"]; /// The rule above and below a block of logged SQL. diff --git a/crates/sqlbuild-core/src/diagnostic_text/mod.rs b/crates/sqlbuild-core/src/diagnostic_text/mod.rs index 1e64f3d116..f01181f853 100644 --- a/crates/sqlbuild-core/src/diagnostic_text/mod.rs +++ b/crates/sqlbuild-core/src/diagnostic_text/mod.rs @@ -1,5 +1,5 @@ pub(crate) mod _helpers; -pub(crate) mod constants; +pub mod constants; pub mod main; pub mod models; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/contracts/state_literals.rs b/crates/sqlbuild-python/src/bindings/_helpers/contracts/state_literals.rs index 3c65cc66aa..e8cc03cc1d 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/contracts/state_literals.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/contracts/state_literals.rs @@ -10,11 +10,11 @@ use pyo3::{pyfunction, wrap_pyfunction}; const EXCEPTIONS_MODULE: &str = "sqlbuild.sql_values.exceptions"; const DATETIME_MODULE: &str = "datetime"; const JSON_MODULE: &str = "json"; -const STRING_TYPE: &str = "string"; +use sqlbuild_state::freshness::constants::STRING_STATE_VALUE_TYPE as STRING_TYPE; const INTEGER_TYPE: &str = "integer"; const BOOLEAN_TYPE: &str = "boolean"; const TIMESTAMP_TYPE: &str = "timestamp"; -const TEXT_TIMESTAMP_TYPE: &str = "text_timestamp"; +use sqlbuild_state::freshness::constants::TEXT_TIMESTAMP_STATE_VALUE_TYPE as TEXT_TIMESTAMP_TYPE; const DATE_TYPE: &str = "date"; const JSON_TYPE: &str = "json"; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/diagnostics/console_records.rs b/crates/sqlbuild-python/src/bindings/_helpers/diagnostics/console_records.rs index 76a093f321..c1432c8c1f 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/diagnostics/console_records.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/diagnostics/console_records.rs @@ -4,6 +4,7 @@ use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyModule, PyModuleMethods, PyRes use pyo3::types::{PyInt, PyString}; use pyo3::{pyfunction, wrap_pyfunction}; use sqlbuild_core::cli_text::models::CliStyle; +use sqlbuild_core::diagnostic_text::constants::SQL_TEXT_LOG_FIELD; use sqlbuild_core::diagnostic_text::main::format_diagnostic_record::format_diagnostic_record; use sqlbuild_core::diagnostic_text::models::DiagnosticRecordText; @@ -61,7 +62,7 @@ fn format_diagnostics_console_record( event: text_field(record, "sqlbuild_event")?, tokens, duration_ms, - sql: text_field(record, "sqlbuild_sql")?, + sql: text_field(record, SQL_TEXT_LOG_FIELD)?, }; Ok(format_diagnostic_record(&text, CliStyle { use_color })) } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/exported_constants/constants.rs b/crates/sqlbuild-python/src/bindings/_helpers/exported_constants/constants.rs index 6cb8ba52a8..01ed8cecef 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/exported_constants/constants.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/exported_constants/constants.rs @@ -26,6 +26,7 @@ use sqlbuild_core::constants::{ SETTINGS_SECTION, SOURCE_FRESHNESS_TABLE_NAME, STAGED_PROMOTION_MODE, TABLE_PROMOTION_MODE_SETTING_KEY, ZERO_DAY_DURATION, }; +use sqlbuild_core::diagnostic_text::constants::{DIAGNOSTIC_LOGGER_PREFIX, SQL_TEXT_LOG_FIELD}; use sqlbuild_rules::constants::MIN_CUSTOM_RULE_TEST_CASES_KEY; use sqlbuild_rules::custom_host_pool::constants::CUSTOM_RULE_PROJECT_SUBJECT; use sqlbuild_state::execution_history::constants::{ @@ -102,6 +103,8 @@ pub(crate) const EXPORTED_TEXTS: &[(&str, &str)] = &[ ("LAUNCH_RECORD_OPTION", LAUNCH_RECORD_OPTION), ("ZERO_DAY_CURSOR_DURATION", ZERO_DAY_DURATION), ("AST_CTE_KIND", CTE_AST_KIND), + ("LOGGER_PREFIX", DIAGNOSTIC_LOGGER_PREFIX), + ("SQL_TEXT_FIELD", SQL_TEXT_LOG_FIELD), ("MIN_CUSTOM_RULE_TEST_CASES", MIN_CUSTOM_RULE_TEST_CASES_KEY), ]; /// Integer constants exported under their Python names. diff --git a/crates/sqlbuild-python/src/bindings/_helpers/freshness/state_io.rs b/crates/sqlbuild-python/src/bindings/_helpers/freshness/state_io.rs index 3e2bc9a4ff..308c987873 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/freshness/state_io.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/freshness/state_io.rs @@ -1,10 +1,10 @@ //! Source freshness state table DDL, latest-row reads and appends through adapter callbacks. use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyModule, PyModuleMethods, PyResult, Python}; -use pyo3::types::{PyDict, PyDictMethods, PyFrozenSet}; +use pyo3::types::{PyDict, PyDictMethods, PyFrozenSet, PyTuple}; use pyo3::{pyfunction, wrap_pyfunction}; use sqlbuild_core::constants::SOURCE_FRESHNESS_TABLE_NAME; -use sqlbuild_state::freshness::constants::REQUIRED_STATE_COLUMNS; +use sqlbuild_state::freshness::constants::{REQUIRED_STATE_COLUMNS, SOURCE_FRESHNESS_COLUMN_TYPES}; use sqlbuild_state::freshness::main::read_latest_sql::read_latest_sql; use sqlbuild_state::freshness::main::relevant_sources_sql::relevant_sources_sql; @@ -13,8 +13,8 @@ use crate::bindings::_helpers::compile_driver::python_calls::{ }; use crate::bindings::_helpers::cursor::values::DATETIME_MODULE; use crate::bindings::_helpers::freshness::support::{ - FRESHNESS_CONSTANTS, FRESHNESS_MODELS, INPUT_ERROR, exception_text, freshness_error, - optional_str, presumed_utc_datetime, text, + FRESHNESS_MODELS, INPUT_ERROR, exception_text, freshness_error, optional_str, + presumed_utc_datetime, text, }; use crate::bindings::_helpers::state_rows::state_tables::state_table_create_sql; use crate::bindings::_helpers::state_rows::timestamps::date_time_typed; @@ -83,9 +83,13 @@ fn create_table_sql<'py>( ) -> PyResult> { let py: Python<'py> = database.py(); let qualified_name: String = qualified_table_name(render_qualified_name, (database, schema))?; - let constants: Bound<'py, PyModule> = py.import(FRESHNESS_CONSTANTS)?; - let columns: Bound<'py, PyAny> = constants.getattr("SOURCE_FRESHNESS_COLUMNS")?; - let column_types: Bound<'py, PyAny> = constants.getattr("SOURCE_FRESHNESS_COLUMN_TYPES")?; + let columns: Bound<'py, PyAny> = + PyTuple::new(py, SOURCE_FRESHNESS_COLUMN_TYPES.map(|(column, _)| column))?.into_any(); + let column_types: Bound<'py, PyDict> = PyDict::new(py); + for (column, declared) in SOURCE_FRESHNESS_COLUMN_TYPES { + column_types.set_item(column, declared)?; + } + let column_types: Bound<'py, PyAny> = column_types.into_any(); let required: Bound<'py, PyAny> = PyFrozenSet::new(py, REQUIRED_STATE_COLUMNS)?.into_any(); let sql: String = state_table_create_sql( &qualified_name, diff --git a/crates/sqlbuild-python/src/bindings/_helpers/freshness/support.rs b/crates/sqlbuild-python/src/bindings/_helpers/freshness/support.rs index 575818f625..e448a148da 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/freshness/support.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/freshness/support.rs @@ -10,7 +10,6 @@ use crate::bindings::_helpers::state_rows::timestamps::{date_time_typed, in_utc, pub(crate) const EXCEPTIONS_MODULE: &str = "sqlbuild._native"; pub(crate) const FRESHNESS_MODELS: &str = "sqlbuild._native"; -pub(crate) const FRESHNESS_CONSTANTS: &str = "sqlbuild.compiler.source_freshness.constants"; pub(crate) const CONTRACT_MODELS: &str = "sqlbuild._native"; pub(crate) const CONTRACT_TYPES: &str = "sqlbuild._native"; pub(crate) const ADAPTER_MODELS: &str = "sqlbuild.adapter.contract.models"; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/state_rows/state_tables.rs b/crates/sqlbuild-python/src/bindings/_helpers/state_rows/state_tables.rs index 7f8bc455a9..3cc4994e3a 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/state_rows/state_tables.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/state_rows/state_tables.rs @@ -7,6 +7,9 @@ use pyo3::prelude::{ use pyo3::types::{PyDict, PyString, PyStringMethods}; use pyo3::{pyfunction, wrap_pyfunction}; use sqlbuild_core::constants::SOURCE_FRESHNESS_TABLE_NAME; +use sqlbuild_state::freshness::constants::{ + SOURCE_FRESHNESS_COLUMN_TYPES, STRING_STATE_VALUE_TYPE, TEXT_TIMESTAMP_STATE_VALUE_TYPE, +}; use sqlbuild_state::state_tables::main::create_table_sql::create_table_sql; use sqlbuild_state::state_tables::main::values_insert_sql::values_insert_sql; use sqlbuild_state::state_tables::models::TableColumn; @@ -14,15 +17,15 @@ use sqlbuild_state::state_tables::models::TableColumn; use crate::bindings::_helpers::boundary::keyword_request::Request; use crate::bindings::_helpers::compile_driver::python_calls::{call_with, imported}; use crate::bindings::_helpers::contracts::state_literals::render_state_sql_literal; -use crate::bindings::_helpers::freshness::support::{ADAPTER_EXCEPTIONS, FRESHNESS_CONSTANTS}; +use crate::bindings::_helpers::freshness::support::ADAPTER_EXCEPTIONS; const ADAPTER_TYPES: &str = "sqlbuild.adapter.contract.types"; const MISSING_SCHEMA_MESSAGE: &str = "source freshness table requires a target schema"; /// Each state value type a state table may declare and the framework type that renders it. const DECLARED_TYPES: [(&str, &str); 3] = [ - ("string", "STRING"), + (STRING_STATE_VALUE_TYPE, "STRING"), ("integer", "INTEGER"), - ("text_timestamp", "TIMESTAMP"), + (TEXT_TIMESTAMP_STATE_VALUE_TYPE, "TIMESTAMP"), ]; /// Create-if-missing DDL whose column types the adapter renders. @@ -114,26 +117,21 @@ fn render_insert_source_freshness_records_sql<'py>( if table.is_none() { return Err(adapter_user_error(py, MISSING_SCHEMA_MESSAGE)); } - let constants: Bound<'py, PyModule> = py.import(FRESHNESS_CONSTANTS)?; - let column_types: Bound<'py, PyAny> = constants.getattr("SOURCE_FRESHNESS_COLUMN_TYPES")?; - let columns: Vec = constants.getattr("SOURCE_FRESHNESS_COLUMNS")?.extract()?; - let declared: Vec = columns - .iter() - .map(|column| column_types.get_item(column)?.str()?.extract()) - .collect::>()?; let mut rows: Vec> = Vec::new(); for record in records.try_iter()? { let record: Bound<'py, PyAny> = record?; - let row: Vec = columns + let row: Vec = SOURCE_FRESHNESS_COLUMN_TYPES .iter() - .zip(&declared) .map(|(column, declared)| { - render_state_sql_literal(py, &record.getattr(column.as_str())?, declared) + render_state_sql_literal(py, &record.getattr(*column)?, declared) }) .collect::>()?; rows.push(row); } - let names: Vec<&str> = columns.iter().map(String::as_str).collect(); + let names: Vec<&str> = SOURCE_FRESHNESS_COLUMN_TYPES + .iter() + .map(|(column, _)| *column) + .collect(); Ok(values_insert_sql( &table.extract::()?, &names, diff --git a/crates/sqlbuild-state/src/freshness/constants.rs b/crates/sqlbuild-state/src/freshness/constants.rs index 6cc649bec7..8083997c83 100644 --- a/crates/sqlbuild-state/src/freshness/constants.rs +++ b/crates/sqlbuild-state/src/freshness/constants.rs @@ -37,7 +37,22 @@ pub const REQUIRED_STATE_COLUMNS: [&str; 6] = [ "data_version_hash", "observed_at", ]; -pub(crate) const STATE_SELECT_COLUMNS: &str = "source_name, target_database, target_schema, \ -target_name, run_id, strategy, value_kind, data_version, data_version_hash, observed_at"; +/// State value type of a text column. +pub const STRING_STATE_VALUE_TYPE: &str = "string"; +/// State value type of a timestamp stored as ISO text. +pub const TEXT_TIMESTAMP_STATE_VALUE_TYPE: &str = "text_timestamp"; +/// Source freshness state table columns in table order, with the state value type each declares. +pub const SOURCE_FRESHNESS_COLUMN_TYPES: [(&str, &str); 10] = [ + ("source_name", STRING_STATE_VALUE_TYPE), + ("target_database", STRING_STATE_VALUE_TYPE), + ("target_schema", STRING_STATE_VALUE_TYPE), + ("target_name", STRING_STATE_VALUE_TYPE), + ("run_id", STRING_STATE_VALUE_TYPE), + ("strategy", STRING_STATE_VALUE_TYPE), + ("value_kind", STRING_STATE_VALUE_TYPE), + ("data_version", STRING_STATE_VALUE_TYPE), + ("data_version_hash", STRING_STATE_VALUE_TYPE), + ("observed_at", TEXT_TIMESTAMP_STATE_VALUE_TYPE), +]; pub(crate) const PHYSICAL_TABLE_SOURCE_FRAGMENT: &str = "requires a physical table source"; pub(crate) const INCOMPLETE_CONFIGURATION_FRAGMENT: &str = "incomplete"; diff --git a/crates/sqlbuild-state/src/freshness/main/read_latest_sql.rs b/crates/sqlbuild-state/src/freshness/main/read_latest_sql.rs index 9dc677915d..443585bfdc 100644 --- a/crates/sqlbuild-state/src/freshness/main/read_latest_sql.rs +++ b/crates/sqlbuild-state/src/freshness/main/read_latest_sql.rs @@ -1,11 +1,16 @@ //! `read_latest_sql`: the windowed SELECT of the latest state row per source identity. -use crate::freshness::constants::STATE_SELECT_COLUMNS; +use crate::freshness::constants::SOURCE_FRESHNESS_COLUMN_TYPES; /// Latest row per source, target database, schema and name, newest observation and run first. pub fn read_latest_sql(qualified_name: &str) -> String { + let columns: String = SOURCE_FRESHNESS_COLUMN_TYPES + .iter() + .map(|(column, _)| *column) + .collect::>() + .join(", "); format!( - "SELECT {STATE_SELECT_COLUMNS} FROM (SELECT {STATE_SELECT_COLUMNS}, ROW_NUMBER() OVER (\ + "SELECT {columns} FROM (SELECT {columns}, ROW_NUMBER() OVER (\ PARTITION BY source_name, target_database, target_schema, target_name \ ORDER BY observed_at DESC, run_id DESC) AS __sqlbuild_latest_rank FROM {qualified_name}) \ AS __sqlbuild_latest_source_freshness WHERE __sqlbuild_latest_rank = 1" diff --git a/python-boundary.toml b/python-boundary.toml index add74a5e5b..1a276f5154 100644 --- a/python-boundary.toml +++ b/python-boundary.toml @@ -376,6 +376,8 @@ note = "Rule 3 shapes: CommandOutputValidationError re-exported by sqlbuild.sink path = "runtime/output_capture/constants.py" rule = 3 note = "Rule 3 shapes: the command-output sink parameter name and producer version; INVOCATION_CONTEXT_ENV is re-exported from sqlbuild._native for integrations." +reason = "public-api" +note = "Rule 3 shapes: the command-output sink parameter name and the installed producer version, no logic." [[stays]] path = "integrations/dagster" @@ -746,6 +748,8 @@ note = "Shape rule: data exceptions only (DbtProfileError), no logic; consumed b path = "diagnostics/constants.py" rule = 4 note = "Shape rule: data constants only (LOGGER_ROOT_NAME, LOGGER_PREFIX, SQL_DIGEST_FIELD, SQL_TEXT_FIELD, ...), no logic; consumed by diagnostics/_helpers/logging.py, diagnostics/_helpers/redaction.py, diagnostics/classes/invocation_diagnostic_routing.py and 2 more." +reason = "public-api" +note = "Shape rule: Python logging policy constants only (LOGGER_ROOT_NAME, SQL_DIGEST_FIELD, SQL_PRIVATE_RECORD_FIELD, REDACTED_SQL_LOG_MESSAGE, ...), no logic; the logger prefix and SQL text field are native (sqlbuild._native)." [[stays]] path = "cursor_algebra/exceptions.py" diff --git a/src/sqlbuild/_native.pyi b/src/sqlbuild/_native.pyi index ea273cb89b..4572912a07 100644 --- a/src/sqlbuild/_native.pyi +++ b/src/sqlbuild/_native.pyi @@ -216,6 +216,8 @@ CONTRACT_ENFORCED: str LAUNCH_RECORD_OPTION: str ZERO_DAY_CURSOR_DURATION: str AST_CTE_KIND: str +LOGGER_PREFIX: str +SQL_TEXT_FIELD: str MIN_CUSTOM_RULE_TEST_CASES: str DEFAULT_PAGE_LIMIT: int MAX_PAGE_LIMIT: int diff --git a/src/sqlbuild/cli/output/__init__.py b/src/sqlbuild/cli/output/__init__.py deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/src/sqlbuild/cli/output/constants.py b/src/sqlbuild/cli/output/constants.py deleted file mode 100644 index 3e27072c58..0000000000 --- a/src/sqlbuild/cli/output/constants.py +++ /dev/null @@ -1,3 +0,0 @@ -"""CLI execution output constants.""" - -from sqlbuild._native import INTEGRATION_RESULT_PATH_ENV as INTEGRATION_RESULT_PATH_ENV diff --git a/src/sqlbuild/compiler/source_freshness/__init__.py b/src/sqlbuild/compiler/source_freshness/__init__.py deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/src/sqlbuild/compiler/source_freshness/constants.py b/src/sqlbuild/compiler/source_freshness/constants.py deleted file mode 100644 index dc199c72ff..0000000000 --- a/src/sqlbuild/compiler/source_freshness/constants.py +++ /dev/null @@ -1,43 +0,0 @@ -"""Stable constants for direct source freshness storage.""" - -from __future__ import annotations - -from sqlbuild._native import SOURCE_FRESHNESS_TABLE_NAME as SOURCE_FRESHNESS_TABLE_NAME -from sqlbuild._native import StateSqlValueType - -COLUMN_SOURCE_NAME: str = "source_name" -COLUMN_TARGET_DATABASE: str = "target_database" -COLUMN_TARGET_SCHEMA: str = "target_schema" -COLUMN_TARGET_NAME: str = "target_name" -COLUMN_RUN_ID: str = "run_id" -COLUMN_STRATEGY: str = "strategy" -COLUMN_VALUE_KIND: str = "value_kind" -COLUMN_DATA_VERSION: str = "data_version" -COLUMN_DATA_VERSION_HASH: str = "data_version_hash" -COLUMN_OBSERVED_AT: str = "observed_at" - -SOURCE_FRESHNESS_COLUMNS: tuple[str, ...] = ( - COLUMN_SOURCE_NAME, - COLUMN_TARGET_DATABASE, - COLUMN_TARGET_SCHEMA, - COLUMN_TARGET_NAME, - COLUMN_RUN_ID, - COLUMN_STRATEGY, - COLUMN_VALUE_KIND, - COLUMN_DATA_VERSION, - COLUMN_DATA_VERSION_HASH, - COLUMN_OBSERVED_AT, -) - -SOURCE_FRESHNESS_COLUMN_TYPES: dict[str, StateSqlValueType] = { - COLUMN_SOURCE_NAME: StateSqlValueType.STRING, - COLUMN_TARGET_DATABASE: StateSqlValueType.STRING, - COLUMN_TARGET_SCHEMA: StateSqlValueType.STRING, - COLUMN_TARGET_NAME: StateSqlValueType.STRING, - COLUMN_RUN_ID: StateSqlValueType.STRING, - COLUMN_STRATEGY: StateSqlValueType.STRING, - COLUMN_VALUE_KIND: StateSqlValueType.STRING, - COLUMN_DATA_VERSION: StateSqlValueType.STRING, - COLUMN_DATA_VERSION_HASH: StateSqlValueType.STRING, - COLUMN_OBSERVED_AT: StateSqlValueType.TEXT_TIMESTAMP, -} diff --git a/src/sqlbuild/diagnostics/_helpers/logging.py b/src/sqlbuild/diagnostics/_helpers/logging.py index a6ae2cab03..30d3ff00d3 100644 --- a/src/sqlbuild/diagnostics/_helpers/logging.py +++ b/src/sqlbuild/diagnostics/_helpers/logging.py @@ -8,7 +8,7 @@ from contextlib import contextmanager from contextvars import ContextVar, Token -from sqlbuild.diagnostics.constants import SQL_TEXT_FIELD +from sqlbuild._native import SQL_TEXT_FIELD from sqlbuild.runtime.observability.main.current_execution_identity import ( current_execution_identity, ) diff --git a/src/sqlbuild/diagnostics/_helpers/redaction.py b/src/sqlbuild/diagnostics/_helpers/redaction.py index 169d967780..60e197fd1c 100644 --- a/src/sqlbuild/diagnostics/_helpers/redaction.py +++ b/src/sqlbuild/diagnostics/_helpers/redaction.py @@ -5,8 +5,8 @@ import logging from typing import Final -from sqlbuild._native import redact_diagnostic_value -from sqlbuild.diagnostics.constants import FORMATTER_RECORD_FIELDS, SQL_TEXT_FIELD +from sqlbuild._native import SQL_TEXT_FIELD, redact_diagnostic_value +from sqlbuild.diagnostics.constants import FORMATTER_RECORD_FIELDS _STANDARD_RECORD_FIELDS: Final[frozenset[str]] = frozenset(logging.makeLogRecord({}).__dict__) diff --git a/src/sqlbuild/diagnostics/classes/invocation_diagnostic_routing.py b/src/sqlbuild/diagnostics/classes/invocation_diagnostic_routing.py index ba42f73323..cf684cac9f 100644 --- a/src/sqlbuild/diagnostics/classes/invocation_diagnostic_routing.py +++ b/src/sqlbuild/diagnostics/classes/invocation_diagnostic_routing.py @@ -7,15 +7,12 @@ from contextvars import Token from types import TracebackType +from sqlbuild._native import SQL_TEXT_FIELD from sqlbuild.diagnostics._helpers.logging import set_sql_diagnostic_sink from sqlbuild.diagnostics.classes.diagnostics_console_formatter import DiagnosticsConsoleFormatter from sqlbuild.diagnostics.classes.dynamic_stderr_handler import DynamicStderrHandler from sqlbuild.diagnostics.classes.log_record_route_filter import LogRecordRouteFilter -from sqlbuild.diagnostics.constants import ( - SQL_PRIVATE_RECORD_FIELD, - SQL_PRIVATE_RECORD_MARKER, - SQL_TEXT_FIELD, -) +from sqlbuild.diagnostics.constants import SQL_PRIVATE_RECORD_FIELD, SQL_PRIVATE_RECORD_MARKER from sqlbuild.diagnostics.models import DiagnosticRoutingOptions from sqlbuild.runtime.compute_logs.classes.diagnostic_log_handler import ( ComputeDiagnosticLogHandler, diff --git a/src/sqlbuild/diagnostics/classes/log_record_route_filter.py b/src/sqlbuild/diagnostics/classes/log_record_route_filter.py index 08c08fd697..5f37e19531 100644 --- a/src/sqlbuild/diagnostics/classes/log_record_route_filter.py +++ b/src/sqlbuild/diagnostics/classes/log_record_route_filter.py @@ -4,13 +4,12 @@ import logging +from sqlbuild._native import LOGGER_PREFIX, SQL_TEXT_FIELD from sqlbuild.diagnostics.classes.diagnostic_record_redactor import DiagnosticRecordRedactor from sqlbuild.diagnostics.constants import ( - LOGGER_PREFIX, LOGGER_ROOT_NAME, REDACTED_SQL_LOG_MESSAGE, SQL_DIGEST_FIELD, - SQL_TEXT_FIELD, ) diff --git a/src/sqlbuild/diagnostics/constants.py b/src/sqlbuild/diagnostics/constants.py index a0c5ab610c..72c33e8eda 100644 --- a/src/sqlbuild/diagnostics/constants.py +++ b/src/sqlbuild/diagnostics/constants.py @@ -1,9 +1,7 @@ """Diagnostic logging policy constants.""" LOGGER_ROOT_NAME: str = "sqlbuild" -LOGGER_PREFIX: str = "sqlbuild." SQL_DIGEST_FIELD: str = "sqlbuild_sql_digest" -SQL_TEXT_FIELD: str = "sqlbuild_sql" SQL_PRIVATE_RECORD_FIELD: str = "sqlbuild_private_sql_record" SQL_PRIVATE_RECORD_MARKER: object = object() REDACTED_SQL_LOG_MESSAGE: str = "SQL diagnostic omitted" diff --git a/src/sqlbuild/integrations/dagster/_helpers/invocation_factory.py b/src/sqlbuild/integrations/dagster/_helpers/invocation_factory.py index 7206d01408..8121443831 100644 --- a/src/sqlbuild/integrations/dagster/_helpers/invocation_factory.py +++ b/src/sqlbuild/integrations/dagster/_helpers/invocation_factory.py @@ -9,8 +9,11 @@ from pathlib import Path from typing import Any, cast -from sqlbuild._native import dagster_cli_arguments -from sqlbuild.cli.output.constants import INTEGRATION_RESULT_PATH_ENV +from sqlbuild._native import ( + INTEGRATION_RESULT_PATH_ENV, + INVOCATION_CONTEXT_ENV, + dagster_cli_arguments, +) from sqlbuild.integrations.dagster._helpers.invocation import ( _selected_asset_check_keys, _selected_asset_paths, @@ -19,7 +22,6 @@ dagster_invocation_context, ) from sqlbuild.integrations.dagster.classes.sqlbuild_cli_invocation import SqlBuildCliInvocation -from sqlbuild.runtime.output_capture.constants import INVOCATION_CONTEXT_ENV def start_sqlbuild_cli_invocation( diff --git a/src/sqlbuild/runtime/compute_logs/classes/diagnostic_log_handler.py b/src/sqlbuild/runtime/compute_logs/classes/diagnostic_log_handler.py index 45e6f11387..03d3322de7 100644 --- a/src/sqlbuild/runtime/compute_logs/classes/diagnostic_log_handler.py +++ b/src/sqlbuild/runtime/compute_logs/classes/diagnostic_log_handler.py @@ -9,16 +9,15 @@ from importlib.metadata import PackageNotFoundError, version from typing import cast +from sqlbuild._native import LOGGER_PREFIX, SQL_TEXT_FIELD from sqlbuild.diagnostics.classes.diagnostic_record_redactor import DiagnosticRecordRedactor from sqlbuild.diagnostics.constants import ( - LOGGER_PREFIX, LOGGER_ROOT_NAME, REDACTED_SQL_LOG_MESSAGE, SQL_DIGEST_FIELD, SQL_PRIVATE_RECORD_FIELD, SQL_PRIVATE_RECORD_MARKER, ) -from sqlbuild.runtime.compute_logs.constants import SQL_LOG_RECORD_FIELD from sqlbuild.runtime.compute_logs.types import ComputeLogStorage from sqlbuild.runtime.observability.constants import CURRENT_DIAGNOSTIC_LOG_SCHEMA_VERSION from sqlbuild.runtime.observability.models import DiagnosticLog @@ -65,17 +64,17 @@ def emit(self, record: logging.LogRecord) -> None: if not internal and record.levelno < logging.INFO: return contains_sql: bool = ( - SQL_DIGEST_FIELD in record.__dict__ or SQL_LOG_RECORD_FIELD in record.__dict__ + SQL_DIGEST_FIELD in record.__dict__ or SQL_TEXT_FIELD in record.__dict__ ) trusted_private_sql: bool = ( record.__dict__.get(SQL_PRIVATE_RECORD_FIELD) is SQL_PRIVATE_RECORD_MARKER ) - if self._include_sql_text and contains_sql and SQL_LOG_RECORD_FIELD not in record.__dict__: + if self._include_sql_text and contains_sql and SQL_TEXT_FIELD not in record.__dict__: return structured: dict[str, object] = { key: value for key, value in record.__dict__.items() - if key.startswith("sqlbuild_") and key != SQL_LOG_RECORD_FIELD + if key.startswith("sqlbuild_") and key != SQL_TEXT_FIELD } identities: dict[str, str | None] = {} for field_name in _IDENTITY_FIELDS: @@ -109,8 +108,8 @@ def emit(self, record: logging.LogRecord) -> None: if not trusted_private_sql: identities = {field_name: None for field_name in _IDENTITY_FIELDS} identities["invocation_id"] = self._invocation_id - if self._include_sql_text and SQL_LOG_RECORD_FIELD in record.__dict__: - fields["sql_text"] = str(record.__dict__[SQL_LOG_RECORD_FIELD]) + if self._include_sql_text and SQL_TEXT_FIELD in record.__dict__: + fields["sql_text"] = str(record.__dict__[SQL_TEXT_FIELD]) elif not internal: fields = cast(dict[str, JSONValue], DiagnosticRecordRedactor.extras(record)) diagnostic: DiagnosticLog = DiagnosticLog( diff --git a/src/sqlbuild/runtime/compute_logs/constants.py b/src/sqlbuild/runtime/compute_logs/constants.py deleted file mode 100644 index b25b998008..0000000000 --- a/src/sqlbuild/runtime/compute_logs/constants.py +++ /dev/null @@ -1,4 +0,0 @@ -"""Compute log format and policy constants.""" - -SQL_LOG_RECORD_FIELD: str = "sqlbuild_sql" -REDACTED_SQL_LOG_MESSAGE: str = "SQL diagnostic omitted" diff --git a/src/sqlbuild/runtime/output_capture/constants.py b/src/sqlbuild/runtime/output_capture/constants.py index 9f12858e26..efbe81b295 100644 --- a/src/sqlbuild/runtime/output_capture/constants.py +++ b/src/sqlbuild/runtime/output_capture/constants.py @@ -2,7 +2,5 @@ from importlib.metadata import version -from sqlbuild._native import INVOCATION_CONTEXT_ENV as INVOCATION_CONTEXT_ENV - COMMAND_OUTPUT_SINK_RECORD_PARAMETER_NAME: str = "record" SQLBUILD_COMMAND_OUTPUT_PRODUCER_VERSION: str = version("sqlbuild") From fb461da821d08e478e461ad7b7c5ace3a866d8eb Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Thu, 15 Oct 2026 23:31:41 +0100 Subject: [PATCH 35/48] chore: reconcile stays entries after removing re-export shims --- python-boundary.toml | 19 ------------------- 1 file changed, 19 deletions(-) diff --git a/python-boundary.toml b/python-boundary.toml index 1a276f5154..3315d9e84a 100644 --- a/python-boundary.toml +++ b/python-boundary.toml @@ -277,11 +277,6 @@ path = "runtime/compute_logs/exceptions.py" rule = 3 note = "Rule 3 shapes: public storage errors re-exported by sqlbuild.compute_logs and raised by the native storage." -[[stays]] -path = "runtime/compute_logs/constants.py" -rule = 1 -note = "Rule 1 capture shim constants: the SQL log record field and redaction message the Python diagnostic log handler uses; format version, retention and read limits are native (sqlbuild._native)." - [[stays]] path = "runtime/event_exporting/__init__.py" rule = 3 @@ -375,8 +370,6 @@ note = "Rule 3 shapes: CommandOutputValidationError re-exported by sqlbuild.sink [[stays]] path = "runtime/output_capture/constants.py" rule = 3 -note = "Rule 3 shapes: the command-output sink parameter name and producer version; INVOCATION_CONTEXT_ENV is re-exported from sqlbuild._native for integrations." -reason = "public-api" note = "Rule 3 shapes: the command-output sink parameter name and the installed producer version, no logic." [[stays]] @@ -664,11 +657,6 @@ path = "cli/compile_reuse/models.py" rule = 4 note = "Shape rule: data records only (CompileReuseRequest, SettingsEnvironmentInputs, SettingsInputsResult), no logic; consumed by cli/compile_reuse/_helpers/provider_settings.py, and read natively by the compile reuse check." -[[stays]] -path = "compiler/source_freshness/constants.py" -rule = 1 -note = "Shape rule: source freshness column names and adapter column types, no logic; SOURCE_FRESHNESS_TABLE_NAME is re-exported from sqlbuild._native." - [[stays]] path = "compiler/auditing/types.py" rule = 1 @@ -747,8 +735,6 @@ note = "Shape rule: data exceptions only (DbtProfileError), no logic; consumed b [[stays]] path = "diagnostics/constants.py" rule = 4 -note = "Shape rule: data constants only (LOGGER_ROOT_NAME, LOGGER_PREFIX, SQL_DIGEST_FIELD, SQL_TEXT_FIELD, ...), no logic; consumed by diagnostics/_helpers/logging.py, diagnostics/_helpers/redaction.py, diagnostics/classes/invocation_diagnostic_routing.py and 2 more." -reason = "public-api" note = "Shape rule: Python logging policy constants only (LOGGER_ROOT_NAME, SQL_DIGEST_FIELD, SQL_PRIVATE_RECORD_FIELD, REDACTED_SQL_LOG_MESSAGE, ...), no logic; the logger prefix and SQL text field are native (sqlbuild._native)." [[stays]] @@ -771,11 +757,6 @@ path = "sql_values/exceptions.py" rule = 1 note = "Shape rule: data exceptions only (SqlValueRenderingError), no logic; consumed by adapter/contract/classes/base_adapter.py, adapter/contract/exceptions.py." -[[stays]] -path = "cli/output/constants.py" -rule = 4 -note = "Shape rule: re-export only (INTEGRATION_RESULT_PATH_ENV, from sqlbuild._native), no logic; consumed by integrations/dagster/_helpers/invocation_factory.py." - [[stays]] path = "compiler/compile/types.py" rule = 1 From 4a36397291f3f2e6efff361e6c9d7e0ecf571c78 Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Fri, 16 Oct 2026 00:21:30 +0100 Subject: [PATCH 36/48] refactor: route, redact and retain diagnostic log records and validate lifecycle envelopes natively SQL log digests and record fields, LogRecordRouteFilter routing and redaction, compute log retention, identity and severity, and lifecycle event, diagnostic log, run bound, operation metadata, hook and producer validation now run in Rust. The logging.Filter/Handler subclasses and public lifecycle classes keep their signatures and call native for every decision. --- Cargo.lock | 1 + .../src/compute_logs/constants.rs | 2 +- crates/sqlbuild-core/Cargo.toml | 1 + .../src/diagnostic_routing/constants.rs | 64 +++ .../main/compute_log_admits.rs | 14 + .../main/diagnostic_severity.rs | 21 + .../main/internal_logger.rs | 9 + .../src/diagnostic_routing/main/mod.rs | 5 + .../diagnostic_routing/main/route_admits.rs | 19 + .../src/diagnostic_routing/main/sql_digest.rs | 11 + .../src/diagnostic_routing/mod.rs | 6 + .../src/diagnostic_routing/models.rs | 24 ++ .../src/diagnostic_routing/tests.rs | 4 + .../diagnostic_routing/tests/test_routing.rs | 200 +++++++++ .../diagnostic_routing/tests/test_types.rs | 33 ++ crates/sqlbuild-core/src/lib.rs | 1 + .../src/lifecycle_events/_helpers/payloads.rs | 2 +- .../src/lifecycle_events/constants.rs | 12 + .../main/diagnostic_severity_problem.rs | 10 + .../main/diagnostic_version_problem.rs | 11 + .../src/lifecycle_events/main/hook_problem.rs | 19 + .../main/lifecycle_version_problem.rs | 32 ++ .../src/lifecycle_events/main/mod.rs | 8 + .../main/operation_metadata_field.rs | 9 + .../main/producer_version_problem.rs | 10 + .../main/publishable_hook_name.rs | 9 + .../main/run_bounds_problem.rs | 18 + .../src/lifecycle_events/models.rs | 20 + .../src/lifecycle_events/tests.rs | 2 + .../tests/test_envelope_rules.rs | 160 ++++++++ .../src/lifecycle_events/tests/test_types.rs | 22 +- .../_helpers/diagnostics/log_records.rs | 381 ++++++++++++++++++ .../src/bindings/_helpers/diagnostics/mod.rs | 1 + .../_helpers/diagnostics/redaction.rs | 5 +- .../_helpers/diagnostics/registration.rs | 3 +- .../_helpers/lifecycle_events/catalog.rs | 8 +- .../_helpers/lifecycle_events/envelopes.rs | 306 ++++++++++++++ .../bindings/_helpers/lifecycle_events/mod.rs | 1 + .../_helpers/lifecycle_events/registration.rs | 3 +- .../_helpers/lifecycle_events/validation.rs | 15 +- python-boundary.toml | 17 +- src/sqlbuild/_native.pyi | 42 ++ src/sqlbuild/diagnostics/_helpers/logging.py | 48 +-- .../diagnostics/_helpers/redaction.py | 23 -- .../classes/diagnostic_record_redactor.py | 9 +- .../classes/log_record_route_filter.py | 52 +-- src/sqlbuild/diagnostics/constants.py | 5 - .../classes/diagnostic_log_handler.py | 103 +---- .../runtime/observability/_helpers/factory.py | 7 +- .../observability/_helpers/observability.py | 2 +- .../classes/operation_lifecycle.py | 56 +-- .../observability/classes/run_lifecycle.py | 26 +- .../runtime/observability/constants.py | 4 - src/sqlbuild/runtime/observability/models.py | 102 +---- 54 files changed, 1573 insertions(+), 405 deletions(-) create mode 100644 crates/sqlbuild-core/src/diagnostic_routing/constants.rs create mode 100644 crates/sqlbuild-core/src/diagnostic_routing/main/compute_log_admits.rs create mode 100644 crates/sqlbuild-core/src/diagnostic_routing/main/diagnostic_severity.rs create mode 100644 crates/sqlbuild-core/src/diagnostic_routing/main/internal_logger.rs create mode 100644 crates/sqlbuild-core/src/diagnostic_routing/main/mod.rs create mode 100644 crates/sqlbuild-core/src/diagnostic_routing/main/route_admits.rs create mode 100644 crates/sqlbuild-core/src/diagnostic_routing/main/sql_digest.rs create mode 100644 crates/sqlbuild-core/src/diagnostic_routing/mod.rs create mode 100644 crates/sqlbuild-core/src/diagnostic_routing/models.rs create mode 100644 crates/sqlbuild-core/src/diagnostic_routing/tests.rs create mode 100644 crates/sqlbuild-core/src/diagnostic_routing/tests/test_routing.rs create mode 100644 crates/sqlbuild-core/src/diagnostic_routing/tests/test_types.rs create mode 100644 crates/sqlbuild-core/src/lifecycle_events/main/diagnostic_severity_problem.rs create mode 100644 crates/sqlbuild-core/src/lifecycle_events/main/diagnostic_version_problem.rs create mode 100644 crates/sqlbuild-core/src/lifecycle_events/main/hook_problem.rs create mode 100644 crates/sqlbuild-core/src/lifecycle_events/main/lifecycle_version_problem.rs create mode 100644 crates/sqlbuild-core/src/lifecycle_events/main/operation_metadata_field.rs create mode 100644 crates/sqlbuild-core/src/lifecycle_events/main/producer_version_problem.rs create mode 100644 crates/sqlbuild-core/src/lifecycle_events/main/publishable_hook_name.rs create mode 100644 crates/sqlbuild-core/src/lifecycle_events/main/run_bounds_problem.rs create mode 100644 crates/sqlbuild-core/src/lifecycle_events/tests/test_envelope_rules.rs create mode 100644 crates/sqlbuild-python/src/bindings/_helpers/diagnostics/log_records.rs create mode 100644 crates/sqlbuild-python/src/bindings/_helpers/lifecycle_events/envelopes.rs delete mode 100644 src/sqlbuild/diagnostics/_helpers/redaction.py diff --git a/Cargo.lock b/Cargo.lock index dbfac92b5c..0882c97e86 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1213,6 +1213,7 @@ name = "sqlbuild-core" version = "0.156.1" dependencies = [ "regex", + "sha2", ] [[package]] diff --git a/crates/sqlbuild-cli/src/compute_logs/constants.rs b/crates/sqlbuild-cli/src/compute_logs/constants.rs index f9ba294a61..ba6664c9b8 100644 --- a/crates/sqlbuild-cli/src/compute_logs/constants.rs +++ b/crates/sqlbuild-cli/src/compute_logs/constants.rs @@ -1,4 +1,4 @@ -//! Compute log format version, retention and read limits, and storage file names. +//! Compute log format version, retention defaults and storage file names. pub const COMPUTE_LOG_FORMAT_VERSION: i64 = 1; pub const DEFAULT_RETENTION_COUNT: usize = 20; diff --git a/crates/sqlbuild-core/Cargo.toml b/crates/sqlbuild-core/Cargo.toml index 2d235db809..e70eb050a7 100644 --- a/crates/sqlbuild-core/Cargo.toml +++ b/crates/sqlbuild-core/Cargo.toml @@ -9,6 +9,7 @@ rust-version.workspace = true [dependencies] regex.workspace = true +sha2.workspace = true [lints] workspace = true diff --git a/crates/sqlbuild-core/src/diagnostic_routing/constants.rs b/crates/sqlbuild-core/src/diagnostic_routing/constants.rs new file mode 100644 index 0000000000..a1b74ec18e --- /dev/null +++ b/crates/sqlbuild-core/src/diagnostic_routing/constants.rs @@ -0,0 +1,64 @@ +//! Diagnostic logging record names, fields and messages. + +/// The root SQLBuild logger name. +pub const LOGGER_ROOT_NAME: &str = "sqlbuild"; +/// The prefix of every SQLBuild child logger name. +pub const LOGGER_PREFIX: &str = "sqlbuild."; +/// The prefix of every SQLBuild structured record field. +pub const FIELD_PREFIX: &str = "sqlbuild_"; +/// The record field holding a SQL statement's digest. +pub const SQL_DIGEST_FIELD: &str = "sqlbuild_sql_digest"; +/// The record field holding SQL text. +pub const SQL_TEXT_FIELD: &str = "sqlbuild_sql"; +/// The record field marking a trusted private SQL record. +pub const SQL_PRIVATE_RECORD_FIELD: &str = "sqlbuild_private_sql_record"; +/// The message replacing every SQL record's text. +pub const REDACTED_SQL_LOG_MESSAGE: &str = "SQL diagnostic omitted"; +/// The message of a record whose text cannot be formatted. +pub const UNFORMATTABLE_MESSAGE: &str = ""; +/// The logger name of every retained SQL diagnostic. +pub const SQL_LOGGER_NAME: &str = "sqlbuild.sql"; +/// The producer of retained diagnostics. +pub const DIAGNOSTIC_PRODUCER: &str = "sqlbuild"; +/// The source of diagnostics retained from Python logging. +pub const DIAGNOSTIC_SOURCE: &str = "python_logging"; +/// The retained field holding private SQL text. +pub const SQL_TEXT_DIAGNOSTIC_FIELD: &str = "sql_text"; +/// Formatter-populated record fields never treated as structured extras. +pub const FORMATTER_RECORD_FIELDS: [&str; 2] = ["message", "asctime"]; +/// The SQL digest field value prefix. +pub const SQL_DIGEST_PREFIX: &str = "sha256:"; +/// The SQL action, intent and query ID record fields, in record order. +pub const SQL_ACTION_FIELD: &str = "sqlbuild_sql_action"; +pub const SQL_INTENT_FIELD: &str = "sqlbuild_sql_intent"; +pub const SQL_QUERY_ID_FIELD: &str = "sqlbuild_sql_query_id"; +/// Execution identity fields, in diagnostic order. +pub const IDENTITY_FIELDS: [&str; 7] = [ + "invocation_id", + "run_id", + "resource_id", + "resource_attempt_id", + "operation_id", + "statement_id", + "log_stream_id", +]; +/// The identity field defaulted to the handler's invocation. +pub const INVOCATION_ID_FIELD: &str = "invocation_id"; +/// Structured fields a retained diagnostic may carry, in retention order. +pub const SAFE_FIELDS: [&str; 7] = [ + "sql_action", + "sql_digest", + "sql_intent", + "sql_query_id", + "channel", + "subscriber", + "error_type", +]; +/// The safe fields a retained SQL diagnostic keeps. +pub const SQL_SAFE_FIELDS: [&str; 4] = ["sql_action", "sql_digest", "sql_intent", "sql_query_id"]; +/// Python logging level numbers. +pub const LEVEL_DEBUG: i64 = 10; +pub const LEVEL_INFO: i64 = 20; +pub const LEVEL_WARNING: i64 = 30; +pub const LEVEL_ERROR: i64 = 40; +pub const LEVEL_CRITICAL: i64 = 50; diff --git a/crates/sqlbuild-core/src/diagnostic_routing/main/compute_log_admits.rs b/crates/sqlbuild-core/src/diagnostic_routing/main/compute_log_admits.rs new file mode 100644 index 0000000000..152e70bd7e --- /dev/null +++ b/crates/sqlbuild-core/src/diagnostic_routing/main/compute_log_admits.rs @@ -0,0 +1,14 @@ +//! `compute_log_admits`: whether the compute log retains one record. + +use crate::diagnostic_routing::constants::LEVEL_INFO; +use crate::diagnostic_routing::models::RecordFacts; + +/// Whether compute logs retain the record; SQL text retention skips SQL records without text. +#[must_use] +pub fn compute_log_admits(record: RecordFacts, include_sql_text: bool) -> bool { + if !record.internal && record.level < LEVEL_INFO { + return false; + } + let contains_sql: bool = record.has_sql_digest || record.has_sql_text; + !(include_sql_text && contains_sql && !record.has_sql_text) +} diff --git a/crates/sqlbuild-core/src/diagnostic_routing/main/diagnostic_severity.rs b/crates/sqlbuild-core/src/diagnostic_routing/main/diagnostic_severity.rs new file mode 100644 index 0000000000..75a891a81e --- /dev/null +++ b/crates/sqlbuild-core/src/diagnostic_routing/main/diagnostic_severity.rs @@ -0,0 +1,21 @@ +//! `diagnostic_severity`: the diagnostic severity of a Python logging level. + +use crate::diagnostic_routing::constants::{ + LEVEL_CRITICAL, LEVEL_ERROR, LEVEL_INFO, LEVEL_WARNING, +}; + +/// The severity name of logging level `level`. +#[must_use] +pub fn diagnostic_severity(level: i64) -> &'static str { + if level >= LEVEL_CRITICAL { + "critical" + } else if level >= LEVEL_ERROR { + "error" + } else if level >= LEVEL_WARNING { + "warning" + } else if level >= LEVEL_INFO { + "info" + } else { + "debug" + } +} diff --git a/crates/sqlbuild-core/src/diagnostic_routing/main/internal_logger.rs b/crates/sqlbuild-core/src/diagnostic_routing/main/internal_logger.rs new file mode 100644 index 0000000000..dbd7e9a6a5 --- /dev/null +++ b/crates/sqlbuild-core/src/diagnostic_routing/main/internal_logger.rs @@ -0,0 +1,9 @@ +//! `internal_logger`: whether a logger name belongs to SQLBuild. + +use crate::diagnostic_routing::constants::{LOGGER_PREFIX, LOGGER_ROOT_NAME}; + +/// Whether `name` is the SQLBuild root logger or one of its children. +#[must_use] +pub fn internal_logger(name: &str) -> bool { + name == LOGGER_ROOT_NAME || name.starts_with(LOGGER_PREFIX) +} diff --git a/crates/sqlbuild-core/src/diagnostic_routing/main/mod.rs b/crates/sqlbuild-core/src/diagnostic_routing/main/mod.rs new file mode 100644 index 0000000000..cb6240ba1c --- /dev/null +++ b/crates/sqlbuild-core/src/diagnostic_routing/main/mod.rs @@ -0,0 +1,5 @@ +pub mod compute_log_admits; +pub mod diagnostic_severity; +pub mod internal_logger; +pub mod route_admits; +pub mod sql_digest; diff --git a/crates/sqlbuild-core/src/diagnostic_routing/main/route_admits.rs b/crates/sqlbuild-core/src/diagnostic_routing/main/route_admits.rs new file mode 100644 index 0000000000..01c8701128 --- /dev/null +++ b/crates/sqlbuild-core/src/diagnostic_routing/main/route_admits.rs @@ -0,0 +1,19 @@ +//! `route_admits`: whether a diagnostic destination receives one record. + +use crate::diagnostic_routing::constants::LEVEL_INFO; +use crate::diagnostic_routing::models::{RecordFacts, RouteOptions}; + +/// Whether a destination receives the record; SQL text routes skip digest-only SQL records. +#[must_use] +pub fn route_admits(options: RouteOptions, record: RecordFacts) -> bool { + if options.internal_only && !record.internal { + return false; + } + if record.internal && !options.allow_internal { + return false; + } + if !record.internal && record.level < LEVEL_INFO { + return false; + } + !(options.include_sql_text && record.has_sql_digest && !record.has_sql_text) +} diff --git a/crates/sqlbuild-core/src/diagnostic_routing/main/sql_digest.rs b/crates/sqlbuild-core/src/diagnostic_routing/main/sql_digest.rs new file mode 100644 index 0000000000..faac04ead9 --- /dev/null +++ b/crates/sqlbuild-core/src/diagnostic_routing/main/sql_digest.rs @@ -0,0 +1,11 @@ +//! `sql_digest`: the logged digest of one SQL statement. + +use sha2::{Digest, Sha256}; + +use crate::diagnostic_routing::constants::SQL_DIGEST_PREFIX; + +/// `sha256:` followed by the hex SHA-256 of the statement's UTF-8 bytes. +#[must_use] +pub fn sql_digest(sql: &str) -> String { + format!("{SQL_DIGEST_PREFIX}{:x}", Sha256::digest(sql.as_bytes())) +} diff --git a/crates/sqlbuild-core/src/diagnostic_routing/mod.rs b/crates/sqlbuild-core/src/diagnostic_routing/mod.rs new file mode 100644 index 0000000000..042268f15e --- /dev/null +++ b/crates/sqlbuild-core/src/diagnostic_routing/mod.rs @@ -0,0 +1,6 @@ +pub mod constants; +pub mod main; +pub mod models; + +#[cfg(test)] +mod tests; diff --git a/crates/sqlbuild-core/src/diagnostic_routing/models.rs b/crates/sqlbuild-core/src/diagnostic_routing/models.rs new file mode 100644 index 0000000000..fcd0a626ce --- /dev/null +++ b/crates/sqlbuild-core/src/diagnostic_routing/models.rs @@ -0,0 +1,24 @@ +//! Facts about one logging record that routing decisions read. + +/// One record's logger family, level and SQL fields. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub struct RecordFacts { + /// The record comes from the SQLBuild logger or one of its children. + pub internal: bool, + pub level: i64, + /// The record carries a SQL digest field. + pub has_sql_digest: bool, + /// The record carries a SQL text field. + pub has_sql_text: bool, +} + +/// Which records one diagnostic destination receives. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub struct RouteOptions { + /// Only SQLBuild records reach this destination. + pub internal_only: bool, + /// SQLBuild records reach this destination. + pub allow_internal: bool, + /// This destination receives SQL text, so it only accepts records that carry it. + pub include_sql_text: bool, +} diff --git a/crates/sqlbuild-core/src/diagnostic_routing/tests.rs b/crates/sqlbuild-core/src/diagnostic_routing/tests.rs new file mode 100644 index 0000000000..0f89c014b0 --- /dev/null +++ b/crates/sqlbuild-core/src/diagnostic_routing/tests.rs @@ -0,0 +1,4 @@ +#[path = "tests/test_routing.rs"] +mod routing; +#[path = "tests/test_types.rs"] +mod test_types; diff --git a/crates/sqlbuild-core/src/diagnostic_routing/tests/test_routing.rs b/crates/sqlbuild-core/src/diagnostic_routing/tests/test_routing.rs new file mode 100644 index 0000000000..59b840ef14 --- /dev/null +++ b/crates/sqlbuild-core/src/diagnostic_routing/tests/test_routing.rs @@ -0,0 +1,200 @@ +//! Diagnostic routing decisions match the Python logging filter and compute log handler. + +use crate::diagnostic_routing::main::compute_log_admits::compute_log_admits; +use crate::diagnostic_routing::main::diagnostic_severity::diagnostic_severity; +use crate::diagnostic_routing::main::internal_logger::internal_logger; +use crate::diagnostic_routing::main::route_admits::route_admits; +use crate::diagnostic_routing::main::sql_digest::sql_digest; +use crate::diagnostic_routing::models::{RecordFacts, RouteOptions}; +use crate::diagnostic_routing::tests::test_types::{ + ComputeLogTestCase, DigestTestCase, InternalLoggerTestCase, RouteTestCase, SeverityTestCase, +}; + +const EXTERNAL_DEBUG: RecordFacts = RecordFacts { + internal: false, + level: 10, + has_sql_digest: false, + has_sql_text: false, +}; +const DIGEST_ONLY: RecordFacts = RecordFacts { + internal: true, + level: 10, + has_sql_digest: true, + has_sql_text: false, +}; +const CONSOLE: RouteOptions = RouteOptions { + internal_only: false, + allow_internal: true, + include_sql_text: false, +}; + +#[test] +fn given_logger_names_when_classifying_then_only_sqlbuild_family_is_internal() { + let test_cases = [ + InternalLoggerTestCase { + description: "root", + name: "sqlbuild", + expected_internal: true, + }, + InternalLoggerTestCase { + description: "child", + name: "sqlbuild.sql", + expected_internal: true, + }, + InternalLoggerTestCase { + description: "shared prefix without separator", + name: "sqlbuildx", + expected_internal: false, + }, + InternalLoggerTestCase { + description: "third party", + name: "other", + expected_internal: false, + }, + ]; + + for test_case in test_cases { + assert_eq!( + internal_logger(test_case.name), + test_case.expected_internal, + "{}", + test_case.description + ); + } +} + +#[test] +fn given_route_options_when_routing_then_family_level_and_sql_rules_apply() { + let test_cases = [ + RouteTestCase { + description: "external debug is below INFO", + options: CONSOLE, + record: EXTERNAL_DEBUG, + expected_admitted: false, + }, + RouteTestCase { + description: "internal debug reaches a destination allowing internal records", + options: CONSOLE, + record: DIGEST_ONLY, + expected_admitted: true, + }, + RouteTestCase { + description: "private SQL route skips digest-only records", + options: RouteOptions { + include_sql_text: true, + ..CONSOLE + }, + record: DIGEST_ONLY, + expected_admitted: false, + }, + RouteTestCase { + description: "internal records need allow_internal", + options: RouteOptions { + allow_internal: false, + ..CONSOLE + }, + record: DIGEST_ONLY, + expected_admitted: false, + }, + ]; + + for test_case in test_cases { + assert_eq!( + route_admits(test_case.options, test_case.record), + test_case.expected_admitted, + "{}", + test_case.description + ); + } +} + +#[test] +fn given_compute_log_records_when_admitting_then_sql_without_text_is_dropped_with_text_retention() { + let test_cases = [ + ComputeLogTestCase { + description: "external debug is below INFO", + record: EXTERNAL_DEBUG, + include_sql_text: false, + expected_admitted: false, + }, + ComputeLogTestCase { + description: "digest-only SQL is kept without text retention", + record: DIGEST_ONLY, + include_sql_text: false, + expected_admitted: true, + }, + ComputeLogTestCase { + description: "digest-only SQL is skipped with text retention", + record: DIGEST_ONLY, + include_sql_text: true, + expected_admitted: false, + }, + ]; + + for test_case in test_cases { + assert_eq!( + compute_log_admits(test_case.record, test_case.include_sql_text), + test_case.expected_admitted, + "{}", + test_case.description + ); + } +} + +#[test] +fn given_logging_levels_when_mapping_then_severity_names_follow_thresholds() { + let test_cases = [ + SeverityTestCase { + description: "below DEBUG", + level: 5, + expected_severity: "debug", + }, + SeverityTestCase { + description: "between INFO and WARNING", + level: 25, + expected_severity: "info", + }, + SeverityTestCase { + description: "WARNING", + level: 30, + expected_severity: "warning", + }, + SeverityTestCase { + description: "ERROR", + level: 40, + expected_severity: "error", + }, + SeverityTestCase { + description: "above CRITICAL", + level: 99, + expected_severity: "critical", + }, + ]; + + for test_case in test_cases { + assert_eq!( + diagnostic_severity(test_case.level), + test_case.expected_severity, + "{}", + test_case.description + ); + } +} + +#[test] +fn given_sql_when_digesting_then_prefixed_sha256_hex() { + let test_cases = [DigestTestCase { + description: "empty statement", + sql: "", + expected_digest: "sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", + }]; + + for test_case in test_cases { + assert_eq!( + sql_digest(test_case.sql), + test_case.expected_digest, + "{}", + test_case.description + ); + } +} diff --git a/crates/sqlbuild-core/src/diagnostic_routing/tests/test_types.rs b/crates/sqlbuild-core/src/diagnostic_routing/tests/test_types.rs new file mode 100644 index 0000000000..275aa0b285 --- /dev/null +++ b/crates/sqlbuild-core/src/diagnostic_routing/tests/test_types.rs @@ -0,0 +1,33 @@ +use crate::diagnostic_routing::models::{RecordFacts, RouteOptions}; + +pub(super) struct InternalLoggerTestCase { + pub(super) description: &'static str, + pub(super) name: &'static str, + pub(super) expected_internal: bool, +} + +pub(super) struct RouteTestCase { + pub(super) description: &'static str, + pub(super) options: RouteOptions, + pub(super) record: RecordFacts, + pub(super) expected_admitted: bool, +} + +pub(super) struct ComputeLogTestCase { + pub(super) description: &'static str, + pub(super) record: RecordFacts, + pub(super) include_sql_text: bool, + pub(super) expected_admitted: bool, +} + +pub(super) struct SeverityTestCase { + pub(super) description: &'static str, + pub(super) level: i64, + pub(super) expected_severity: &'static str, +} + +pub(super) struct DigestTestCase { + pub(super) description: &'static str, + pub(super) sql: &'static str, + pub(super) expected_digest: &'static str, +} diff --git a/crates/sqlbuild-core/src/lib.rs b/crates/sqlbuild-core/src/lib.rs index 320d1e8996..bfb6bfed61 100644 --- a/crates/sqlbuild-core/src/lib.rs +++ b/crates/sqlbuild-core/src/lib.rs @@ -24,6 +24,7 @@ pub mod dbt_graph; pub mod dbt_plan; pub mod dbt_process; pub mod debug_report; +pub mod diagnostic_routing; pub mod diagnostic_text; pub mod event_export; pub mod format_command; diff --git a/crates/sqlbuild-core/src/lifecycle_events/_helpers/payloads.rs b/crates/sqlbuild-core/src/lifecycle_events/_helpers/payloads.rs index f792e8f4a6..e57df01e02 100644 --- a/crates/sqlbuild-core/src/lifecycle_events/_helpers/payloads.rs +++ b/crates/sqlbuild-core/src/lifecycle_events/_helpers/payloads.rs @@ -40,7 +40,7 @@ pub(crate) fn forbidden_statement_fields(value: &PayloadValue) -> BTreeSet bool { +pub(crate) fn safe_hook_name(name: &str) -> bool { let mut characters = name.chars(); let Some(first) = characters.next() else { return false; diff --git a/crates/sqlbuild-core/src/lifecycle_events/constants.rs b/crates/sqlbuild-core/src/lifecycle_events/constants.rs index f7ad1aa621..8743401fb4 100644 --- a/crates/sqlbuild-core/src/lifecycle_events/constants.rs +++ b/crates/sqlbuild-core/src/lifecycle_events/constants.rs @@ -6,6 +6,18 @@ use crate::lifecycle_events::types::CatalogEntry; pub const CURRENT_LIFECYCLE_EVENT_SCHEMA_VERSION: i64 = 2; /// Every lifecycle event schema version with a catalogue, oldest first. pub const LIFECYCLE_EVENT_SCHEMA_VERSIONS: [i64; 2] = [1, 2]; +/// The only lifecycle event schema version without invocation metadata. +pub(crate) const UNSEQUENCED_SCHEMA_VERSION: i128 = 1; +/// The diagnostic log schema version SQLBuild produces and reads. +pub const CURRENT_DIAGNOSTIC_LOG_SCHEMA_VERSION: i64 = 1; +/// Every diagnostic log severity. +pub const DIAGNOSTIC_SEVERITIES: [&str; 6] = + ["critical", "debug", "error", "info", "trace", "warning"]; +/// The producer whose version SQLBuild supplies itself. +pub const DEFAULT_PRODUCER: &str = "sqlbuild"; +/// The message for an invalid invocation sequence. +pub const INVOCATION_SEQUENCE_PROBLEM: &str = + "invocation_sequence must be a non-negative integer excluding bool"; pub(crate) const MAX_METADATA_BYTES: usize = 4096; /// The smallest duration a payload may carry. pub(crate) const MINIMUM_DURATION_MS: f64 = 0.0; diff --git a/crates/sqlbuild-core/src/lifecycle_events/main/diagnostic_severity_problem.rs b/crates/sqlbuild-core/src/lifecycle_events/main/diagnostic_severity_problem.rs new file mode 100644 index 0000000000..d19e85f0ad --- /dev/null +++ b/crates/sqlbuild-core/src/lifecycle_events/main/diagnostic_severity_problem.rs @@ -0,0 +1,10 @@ +//! `diagnostic_severity_problem`: the severity rule of a diagnostic log. + +use crate::lifecycle_events::constants::DIAGNOSTIC_SEVERITIES; + +/// The rule a diagnostic severity breaks. +#[must_use] +pub fn diagnostic_severity_problem(severity: &str) -> Option<&'static str> { + (!DIAGNOSTIC_SEVERITIES.contains(&severity)) + .then_some("severity must be one of trace, debug, info, warning, error, critical") +} diff --git a/crates/sqlbuild-core/src/lifecycle_events/main/diagnostic_version_problem.rs b/crates/sqlbuild-core/src/lifecycle_events/main/diagnostic_version_problem.rs new file mode 100644 index 0000000000..8f50767a3a --- /dev/null +++ b/crates/sqlbuild-core/src/lifecycle_events/main/diagnostic_version_problem.rs @@ -0,0 +1,11 @@ +//! `diagnostic_version_problem`: the schema version rule of a diagnostic log. + +use crate::lifecycle_events::constants::CURRENT_DIAGNOSTIC_LOG_SCHEMA_VERSION; + +/// The rule a diagnostic log's schema version breaks. +#[must_use] +pub fn diagnostic_version_problem(schema_version: i128) -> Option { + (schema_version != i128::from(CURRENT_DIAGNOSTIC_LOG_SCHEMA_VERSION)).then(|| { + format!("diagnostic schema_version must be {CURRENT_DIAGNOSTIC_LOG_SCHEMA_VERSION}") + }) +} diff --git a/crates/sqlbuild-core/src/lifecycle_events/main/hook_problem.rs b/crates/sqlbuild-core/src/lifecycle_events/main/hook_problem.rs new file mode 100644 index 0000000000..e3b76f7602 --- /dev/null +++ b/crates/sqlbuild-core/src/lifecycle_events/main/hook_problem.rs @@ -0,0 +1,19 @@ +//! `hook_problem`: the catalogue rules of an operation's hook fields. + +use crate::lifecycle_events::constants::{HOOK_PHASES, HOOK_TYPES}; +use crate::lifecycle_events::models::HookFacts; + +/// The first rule supplied hook fields break. +#[must_use] +pub fn hook_problem(hook: HookFacts<'_>) -> Option<&'static str> { + if !hook.phase.is_some_and(|phase| HOOK_PHASES.contains(&phase)) { + return Some("hook_phase must be a catalogued value"); + } + if !hook + .hook_type + .is_some_and(|hook_type| HOOK_TYPES.contains(&hook_type)) + { + return Some("hook_type must be a catalogued value"); + } + (!hook.valid_index).then_some("hook_index must be a nonnegative integer") +} diff --git a/crates/sqlbuild-core/src/lifecycle_events/main/lifecycle_version_problem.rs b/crates/sqlbuild-core/src/lifecycle_events/main/lifecycle_version_problem.rs new file mode 100644 index 0000000000..cf90f4ef27 --- /dev/null +++ b/crates/sqlbuild-core/src/lifecycle_events/main/lifecycle_version_problem.rs @@ -0,0 +1,32 @@ +//! `lifecycle_version_problem`: the schema version and invocation metadata rules of an event. + +use crate::lifecycle_events::constants::{ + CURRENT_LIFECYCLE_EVENT_SCHEMA_VERSION, INVOCATION_SEQUENCE_PROBLEM, + LIFECYCLE_EVENT_SCHEMA_VERSIONS, UNSEQUENCED_SCHEMA_VERSION, +}; +use crate::lifecycle_events::models::SequenceFact; + +/// The first rule a lifecycle event's schema version and invocation metadata break. +#[must_use] +pub fn lifecycle_version_problem( + schema_version: i128, + sequence: SequenceFact, + has_external_context: bool, +) -> Option { + let known: bool = LIFECYCLE_EVENT_SCHEMA_VERSIONS + .iter() + .any(|version| i128::from(*version) == schema_version); + if !known { + return Some(format!( + "LifecycleEvent only represents known schema versions through \ + {CURRENT_LIFECYCLE_EVENT_SCHEMA_VERSION}; decode other versions as \ + OpaqueLifecycleEvent" + )); + } + if schema_version == UNSEQUENCED_SCHEMA_VERSION { + return (sequence != SequenceFact::Absent || has_external_context).then(|| { + "schema version 1 lifecycle events cannot contain invocation metadata".to_owned() + }); + } + (sequence != SequenceFact::Valid).then(|| INVOCATION_SEQUENCE_PROBLEM.to_owned()) +} diff --git a/crates/sqlbuild-core/src/lifecycle_events/main/mod.rs b/crates/sqlbuild-core/src/lifecycle_events/main/mod.rs index e199b26b01..421bd96e7c 100644 --- a/crates/sqlbuild-core/src/lifecycle_events/main/mod.rs +++ b/crates/sqlbuild-core/src/lifecycle_events/main/mod.rs @@ -1,2 +1,10 @@ pub mod catalog; +pub mod diagnostic_severity_problem; +pub mod diagnostic_version_problem; +pub mod hook_problem; +pub mod lifecycle_version_problem; +pub mod operation_metadata_field; +pub mod producer_version_problem; +pub mod publishable_hook_name; +pub mod run_bounds_problem; pub mod validate_known_event; diff --git a/crates/sqlbuild-core/src/lifecycle_events/main/operation_metadata_field.rs b/crates/sqlbuild-core/src/lifecycle_events/main/operation_metadata_field.rs new file mode 100644 index 0000000000..a2ce662f67 --- /dev/null +++ b/crates/sqlbuild-core/src/lifecycle_events/main/operation_metadata_field.rs @@ -0,0 +1,9 @@ +//! `operation_metadata_field`: whether a key is an allow-listed operation metadata field. + +use crate::lifecycle_events::constants::OPERATION_METADATA_FIELDS; + +/// Whether a metadata key, when a string, is an allow-listed operation metadata field. +#[must_use] +pub fn operation_metadata_field(key: Option<&str>) -> bool { + key.is_some_and(|key| OPERATION_METADATA_FIELDS.contains(&key)) +} diff --git a/crates/sqlbuild-core/src/lifecycle_events/main/producer_version_problem.rs b/crates/sqlbuild-core/src/lifecycle_events/main/producer_version_problem.rs new file mode 100644 index 0000000000..de76192a3a --- /dev/null +++ b/crates/sqlbuild-core/src/lifecycle_events/main/producer_version_problem.rs @@ -0,0 +1,10 @@ +//! `producer_version_problem`: the producer version rule of a lifecycle event. + +use crate::lifecycle_events::constants::DEFAULT_PRODUCER; + +/// The rule a lifecycle event's producer breaks: other producers must name their version. +#[must_use] +pub fn producer_version_problem(producer: Option<&str>, has_version: bool) -> Option<&'static str> { + (producer != Some(DEFAULT_PRODUCER) && !has_version) + .then_some("producer_version is required when producer is not 'sqlbuild'") +} diff --git a/crates/sqlbuild-core/src/lifecycle_events/main/publishable_hook_name.rs b/crates/sqlbuild-core/src/lifecycle_events/main/publishable_hook_name.rs new file mode 100644 index 0000000000..77fe9c7417 --- /dev/null +++ b/crates/sqlbuild-core/src/lifecycle_events/main/publishable_hook_name.rs @@ -0,0 +1,9 @@ +//! `publishable_hook_name`: whether an operation's hook name is safe to publish. + +use crate::lifecycle_events::_helpers::payloads::safe_hook_name; + +/// Whether a hook name is an identifier of at most 128 characters. +#[must_use] +pub fn publishable_hook_name(name: &str) -> bool { + safe_hook_name(name) +} diff --git a/crates/sqlbuild-core/src/lifecycle_events/main/run_bounds_problem.rs b/crates/sqlbuild-core/src/lifecycle_events/main/run_bounds_problem.rs new file mode 100644 index 0000000000..ed15cb02b3 --- /dev/null +++ b/crates/sqlbuild-core/src/lifecycle_events/main/run_bounds_problem.rs @@ -0,0 +1,18 @@ +//! `run_bounds_problem`: the selection, concurrency and worker rules of a run. + +/// The rule a run's selection, concurrency and worker counts break. +#[must_use] +pub fn run_bounds_problem( + selected_count: i128, + configured_concurrency: i128, + worker_count: i128, +) -> Option<&'static str> { + if selected_count < 0 { + return Some("selected_count must be nonnegative"); + } + if configured_concurrency < 1 { + return Some("configured_concurrency must be positive"); + } + (worker_count != configured_concurrency.min(selected_count)) + .then_some("worker_count must equal min(configured_concurrency, selected_count)") +} diff --git a/crates/sqlbuild-core/src/lifecycle_events/models.rs b/crates/sqlbuild-core/src/lifecycle_events/models.rs index c39367f126..4fc84dd22e 100644 --- a/crates/sqlbuild-core/src/lifecycle_events/models.rs +++ b/crates/sqlbuild-core/src/lifecycle_events/models.rs @@ -35,3 +35,23 @@ pub struct KnownEventFacts { pub correlations: Vec<(&'static str, bool)>, pub payload: Vec<(String, PayloadValue)>, } + +/// The invocation sequence an event carries. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum SequenceFact { + Absent, + /// A non-negative integer other than a bool. + Valid, + Invalid, +} + +/// The supplied hook phase, type and index of an operation. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub struct HookFacts<'a> { + /// The phase when it is a string. + pub phase: Option<&'a str>, + /// The hook type when it is a string. + pub hook_type: Option<&'a str>, + /// The index is a non-negative integer other than a bool. + pub valid_index: bool, +} diff --git a/crates/sqlbuild-core/src/lifecycle_events/tests.rs b/crates/sqlbuild-core/src/lifecycle_events/tests.rs index e1b877d89c..b110b2a88e 100644 --- a/crates/sqlbuild-core/src/lifecycle_events/tests.rs +++ b/crates/sqlbuild-core/src/lifecycle_events/tests.rs @@ -1,3 +1,5 @@ +#[path = "tests/test_envelope_rules.rs"] +mod envelope_rules; #[path = "tests/helpers.rs"] mod helpers; #[path = "tests/test_types.rs"] diff --git a/crates/sqlbuild-core/src/lifecycle_events/tests/test_envelope_rules.rs b/crates/sqlbuild-core/src/lifecycle_events/tests/test_envelope_rules.rs new file mode 100644 index 0000000000..b37fd38f69 --- /dev/null +++ b/crates/sqlbuild-core/src/lifecycle_events/tests/test_envelope_rules.rs @@ -0,0 +1,160 @@ +use crate::lifecycle_events::main::hook_problem::hook_problem; +use crate::lifecycle_events::main::lifecycle_version_problem::lifecycle_version_problem; +use crate::lifecycle_events::main::run_bounds_problem::run_bounds_problem; +use crate::lifecycle_events::models::{HookFacts, SequenceFact}; +use crate::lifecycle_events::tests::test_types::{ + HookTestCase, LifecycleVersionTestCase, RunBoundsTestCase, +}; + +const SEQUENCE_PROBLEM: &str = "invocation_sequence must be a non-negative integer excluding bool"; +const VALID_HOOK: HookFacts<'static> = HookFacts { + phase: Some("pre_hooks"), + hook_type: Some("sql"), + valid_index: true, +}; + +#[test] +fn given_schema_versions_and_metadata_when_validating_then_version_rules_apply() { + let test_cases = [ + LifecycleVersionTestCase { + description: "unknown version", + schema_version: 3, + sequence: SequenceFact::Valid, + has_external_context: false, + expected_problem: Some( + "LifecycleEvent only represents known schema versions through 2; decode other versions as OpaqueLifecycleEvent", + ), + }, + LifecycleVersionTestCase { + description: "version 1 without metadata", + schema_version: 1, + sequence: SequenceFact::Absent, + has_external_context: false, + expected_problem: None, + }, + LifecycleVersionTestCase { + description: "version 1 with external context", + schema_version: 1, + sequence: SequenceFact::Absent, + has_external_context: true, + expected_problem: Some( + "schema version 1 lifecycle events cannot contain invocation metadata", + ), + }, + LifecycleVersionTestCase { + description: "version 2 requires a sequence", + schema_version: 2, + sequence: SequenceFact::Absent, + has_external_context: false, + expected_problem: Some(SEQUENCE_PROBLEM), + }, + LifecycleVersionTestCase { + description: "version 2 with sequence and context", + schema_version: 2, + sequence: SequenceFact::Valid, + has_external_context: true, + expected_problem: None, + }, + ]; + + for test_case in test_cases { + assert_eq!( + lifecycle_version_problem( + test_case.schema_version, + test_case.sequence, + test_case.has_external_context + ) + .as_deref(), + test_case.expected_problem, + "{}", + test_case.description + ); + } +} + +#[test] +fn given_run_bounds_when_validating_then_worker_count_matches_capped_concurrency() { + let test_cases = [ + RunBoundsTestCase { + description: "empty selection", + counts: (0, 1, 0), + expected_problem: None, + }, + RunBoundsTestCase { + description: "concurrency caps workers", + counts: (5, 2, 2), + expected_problem: None, + }, + RunBoundsTestCase { + description: "negative selection", + counts: (-1, 1, 0), + expected_problem: Some("selected_count must be nonnegative"), + }, + RunBoundsTestCase { + description: "zero concurrency", + counts: (1, 0, 0), + expected_problem: Some("configured_concurrency must be positive"), + }, + RunBoundsTestCase { + description: "too many workers", + counts: (5, 2, 5), + expected_problem: Some( + "worker_count must equal min(configured_concurrency, selected_count)", + ), + }, + ]; + + for test_case in test_cases { + let (selected, configured, workers) = test_case.counts; + assert_eq!( + run_bounds_problem(selected, configured, workers), + test_case.expected_problem, + "{}", + test_case.description + ); + } +} + +#[test] +fn given_hook_fields_when_validating_then_catalogue_rules_apply_in_order() { + let test_cases = [ + HookTestCase { + description: "catalogued hook", + hook: VALID_HOOK, + expected_problem: None, + }, + HookTestCase { + description: "missing phase", + hook: HookFacts { + phase: None, + ..VALID_HOOK + }, + expected_problem: Some("hook_phase must be a catalogued value"), + }, + HookTestCase { + description: "uncatalogued type", + hook: HookFacts { + hook_type: Some("shell"), + ..VALID_HOOK + }, + expected_problem: Some("hook_type must be a catalogued value"), + }, + HookTestCase { + description: "invalid index", + hook: HookFacts { + valid_index: false, + ..VALID_HOOK + }, + expected_problem: Some("hook_index must be a nonnegative integer"), + }, + ]; + + for test_case in test_cases { + assert_eq!( + hook_problem(test_case.hook), + test_case.expected_problem, + "{}", + test_case.description + ); + } +} diff --git a/crates/sqlbuild-core/src/lifecycle_events/tests/test_types.rs b/crates/sqlbuild-core/src/lifecycle_events/tests/test_types.rs index 34bc5cecc4..e843112479 100644 --- a/crates/sqlbuild-core/src/lifecycle_events/tests/test_types.rs +++ b/crates/sqlbuild-core/src/lifecycle_events/tests/test_types.rs @@ -1,7 +1,27 @@ -use crate::lifecycle_events::models::KnownEventFacts; +use crate::lifecycle_events::models::{HookFacts, KnownEventFacts, SequenceFact}; pub(crate) struct ValidateTestCase { pub(crate) description: &'static str, pub(crate) event: KnownEventFacts, pub(crate) expected_result: Result<(), &'static str>, } + +pub(crate) struct LifecycleVersionTestCase { + pub(crate) description: &'static str, + pub(crate) schema_version: i128, + pub(crate) sequence: SequenceFact, + pub(crate) has_external_context: bool, + pub(crate) expected_problem: Option<&'static str>, +} + +pub(crate) struct RunBoundsTestCase { + pub(crate) description: &'static str, + pub(crate) counts: (i128, i128, i128), + pub(crate) expected_problem: Option<&'static str>, +} + +pub(crate) struct HookTestCase { + pub(crate) description: &'static str, + pub(crate) hook: HookFacts<'static>, + pub(crate) expected_problem: Option<&'static str>, +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/diagnostics/log_records.rs b/crates/sqlbuild-python/src/bindings/_helpers/diagnostics/log_records.rs new file mode 100644 index 0000000000..efe696e343 --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/diagnostics/log_records.rs @@ -0,0 +1,381 @@ +//! Structured diagnostic record fields, destination routing and compute log retention. + +use pyo3::exceptions::PyException; +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyModule, PyModuleMethods, PyResult, Python}; +use pyo3::types::{ + PyBool, PyDict, PyDictMethods, PyFloat, PyInt, PyString, PyStringMethods, PyTuple, +}; +use pyo3::{pyfunction, wrap_pyfunction}; +use sqlbuild_core::diagnostic_routing::constants::{ + DIAGNOSTIC_PRODUCER, DIAGNOSTIC_SOURCE, FIELD_PREFIX, FORMATTER_RECORD_FIELDS, IDENTITY_FIELDS, + INVOCATION_ID_FIELD, LOGGER_PREFIX, LOGGER_ROOT_NAME, REDACTED_SQL_LOG_MESSAGE, SAFE_FIELDS, + SQL_ACTION_FIELD, SQL_DIGEST_FIELD, SQL_INTENT_FIELD, SQL_LOGGER_NAME, + SQL_PRIVATE_RECORD_FIELD, SQL_QUERY_ID_FIELD, SQL_SAFE_FIELDS, SQL_TEXT_DIAGNOSTIC_FIELD, + SQL_TEXT_FIELD, UNFORMATTABLE_MESSAGE, +}; +use sqlbuild_core::diagnostic_routing::main::compute_log_admits::compute_log_admits; +use sqlbuild_core::diagnostic_routing::main::diagnostic_severity::diagnostic_severity; +use sqlbuild_core::diagnostic_routing::main::internal_logger::internal_logger; +use sqlbuild_core::diagnostic_routing::main::route_admits::route_admits; +use sqlbuild_core::diagnostic_routing::main::sql_digest::sql_digest; +use sqlbuild_core::diagnostic_routing::models::{RecordFacts, RouteOptions}; +use sqlbuild_core::redaction::main::redact_text::redact_text; + +use crate::bindings::_helpers::boundary::keyword_request::Request; +use crate::bindings::_helpers::compile_driver::python_calls::imported; +use crate::bindings::_helpers::diagnostics::redaction::redacted_value; + +const PRIVATE_SQL_MARKER: (&str, &str) = ( + "sqlbuild.diagnostics.constants", + "SQL_PRIVATE_RECORD_MARKER", +); + +/// `context | fields | identity`, with later mappings overriding earlier keys in place. +fn merged<'py>( + py: Python<'py>, + parts: &[Option<&Bound<'py, PyAny>>], + skip: Option<&str>, +) -> PyResult> { + let merged: Bound<'py, PyDict> = PyDict::new(py); + for part in parts.iter().flatten() { + if part.is_none() { + continue; + } + for item in part.call_method0("items")?.try_iter()? { + let (key, value): (Bound<'py, PyAny>, Bound<'py, PyAny>) = item?.extract()?; + if skip.is_some_and(|skipped| key.eq(skipped).unwrap_or(false)) { + continue; + } + merged.set_item(key, value)?; + } + } + Ok(merged) +} + +/// The record fields of the current execution identity, each named `sqlbuild_`. +fn identity_fields<'py>( + py: Python<'py>, + identity: Option<&Bound<'py, PyAny>>, +) -> PyResult>> { + let Some(identity) = identity.filter(|identity| !identity.is_none()) else { + return Ok(None); + }; + let fields: Bound<'py, PyDict> = PyDict::new(py); + for item in identity.call_method0("items")?.try_iter()? { + let (key, value): (Bound<'py, PyAny>, Bound<'py, PyAny>) = item?.extract()?; + fields.set_item(format!("{FIELD_PREFIX}{}", key.str()?.to_str()?), value)?; + } + Ok(Some(fields.into_any())) +} + +/// The message and record fields of one logged SQL statement, which never carry its text. +#[pyfunction] +#[pyo3(signature = (**request))] +fn sql_log_record<'py>( + request: Option<&Bound<'py, PyDict>>, +) -> PyResult<(String, Bound<'py, PyDict>)> { + let request: Request<'_, 'py> = Request::new("sql_log_record", request)?; + let py: Python<'py> = request.py(); + let sql: String = request.get("sql")?.extract()?; + let action: Bound<'py, PyAny> = request.get("action")?; + let sql_fields: Bound<'py, PyDict> = PyDict::new(py); + sql_fields.set_item(SQL_ACTION_FIELD, &action)?; + sql_fields.set_item(SQL_DIGEST_FIELD, sql_digest(&sql))?; + for (field, name) in [ + (SQL_INTENT_FIELD, "intent"), + (SQL_QUERY_ID_FIELD, "query_id"), + ] { + let value: Bound<'py, PyAny> = request.get(name)?; + if !value.is_none() { + sql_fields.set_item(field, value)?; + } + } + let identity: Option> = identity_fields(py, Some(&request.get("identity")?))?; + let fields: Bound<'py, PyDict> = merged( + py, + &[ + Some(&request.get("context")?), + Some(sql_fields.as_any()), + identity.as_ref(), + ], + Some(SQL_TEXT_FIELD), + )?; + let message: String = format!("{} SQL", action.str()?.to_str()?); + Ok((message, fields)) +} + +/// The record fields of one structured debug event. +#[pyfunction] +#[pyo3(signature = (*, context, event_fields, identity))] +fn debug_event_record_fields<'py>( + py: Python<'py>, + context: &Bound<'py, PyAny>, + event_fields: &Bound<'py, PyAny>, + identity: &Bound<'py, PyAny>, +) -> PyResult> { + let identity: Option> = identity_fields(py, Some(identity))?; + merged( + py, + &[Some(context), Some(event_fields), identity.as_ref()], + None, + ) +} + +/// Whether a logger name belongs to SQLBuild, with Python semantics for non-string names. +fn internal_name(name: &Bound<'_, PyAny>) -> PyResult { + if let Ok(text) = name.downcast::() { + return Ok(internal_logger(text.to_str()?)); + } + Ok(name.eq(LOGGER_ROOT_NAME)? + || name + .call_method1("startswith", (LOGGER_PREFIX,))? + .is_truthy()?) +} + +/// `record.__dict__`. +fn record_values<'py>(record: &Bound<'py, PyAny>) -> PyResult> { + Ok(record.getattr("__dict__")?.downcast_into::()?) +} + +/// `Ok(None)` for an `Exception`, which the caller swallows; other errors propagate. +fn swallowed<'py, T>(py: Python<'py>, result: PyResult) -> PyResult> { + match result { + Ok(value) => Ok(Some(value)), + Err(error) if error.is_instance_of::(py) => Ok(None), + Err(error) => Err(error), + } +} + +/// The redacted formatted message of a record. +fn redacted_message(record: &Bound<'_, PyAny>) -> PyResult { + Ok(redact_text( + record.call_method0("getMessage")?.extract::<&str>()?, + )) +} + +/// A redacted copy of `record.__dict__` for one destination, or `None` when it does not belong. +#[pyfunction] +#[pyo3(signature = (*, record, internal_only, allow_internal, include_sql_text))] +fn route_diagnostic_log_record<'py>( + record: &Bound<'py, PyAny>, + internal_only: bool, + allow_internal: bool, + include_sql_text: bool, +) -> PyResult>> { + let py: Python<'py> = record.py(); + let values: Bound<'py, PyDict> = record_values(record)?; + let facts = RecordFacts { + internal: internal_name(&record.getattr("name")?)?, + level: record.getattr("levelno")?.extract()?, + has_sql_digest: values.contains(SQL_DIGEST_FIELD)?, + has_sql_text: values.contains(SQL_TEXT_FIELD)?, + }; + let options = RouteOptions { + internal_only, + allow_internal, + include_sql_text, + }; + if !route_admits(options, facts) { + return Ok(None); + } + let copied: Bound<'py, PyDict> = values.copy()?; + let mut message: String = swallowed(py, redacted_message(record))? + .unwrap_or_else(|| UNFORMATTABLE_MESSAGE.to_owned()); + copied.set_item("args", PyTuple::empty(py))?; + if facts.has_sql_digest { + REDACTED_SQL_LOG_MESSAGE.clone_into(&mut message); + } + let exc_info: Bound<'py, PyAny> = record.getattr("exc_info")?; + if !exc_info.is_none() { + let formatted: PyResult = imported(py, "logging", "Formatter") + .and_then(|formatter| formatter.call0()) + .and_then(|formatter| formatter.call_method1("formatException", (exc_info,))) + .and_then(|text| text.extract::()) + .map(|text| redact_text(&text)); + if let Some(exception_text) = swallowed(py, formatted)? { + message = format!("{message}\n{exception_text}"); + } + copied.set_item("exc_info", py.None())?; + copied.set_item("exc_text", py.None())?; + } + copied.set_item("msg", message)?; + for (key, value) in copied.copy()?.iter() { + let name: String = key.extract()?; + if name.starts_with(FIELD_PREFIX) && name != SQL_TEXT_FIELD { + copied.set_item(&name, redacted_value(&name, &value)?)?; + } + } + if !include_sql_text && copied.contains(SQL_TEXT_FIELD)? { + copied.del_item(SQL_TEXT_FIELD)?; + } + Ok(Some(copied)) +} + +/// Redacted structured extras of a record, excluding standard, formatter and SQL text fields. +fn extra_fields<'py>(record: &Bound<'py, PyAny>) -> PyResult> { + let py: Python<'py> = record.py(); + let standard: Bound<'py, PyDict> = + record_values(&imported(py, "logging", "makeLogRecord")?.call1((PyDict::new(py),))?)?; + let extras: Bound<'py, PyDict> = PyDict::new(py); + for (key, value) in record_values(record)?.iter() { + let name: String = key.extract()?; + let excluded: bool = standard.contains(&name)? + || FORMATTER_RECORD_FIELDS.contains(&name.as_str()) + || name == SQL_TEXT_FIELD; + if !excluded { + extras.set_item(&name, redacted_value(&name, &value)?)?; + } + } + Ok(extras) +} + +/// Redacted non-standard fields of a record without evaluating arbitrary formatters. +#[pyfunction] +#[pyo3(signature = (record))] +fn safe_diagnostic_extra_fields<'py>(record: &Bound<'py, PyAny>) -> PyResult> { + extra_fields(record) +} + +/// Whether a redacted structured value is a JSON scalar. +fn scalar(value: &Bound<'_, PyAny>) -> bool { + value.is_instance_of::() + || value.is_instance_of::() + || value.is_instance_of::() + || value.is_instance_of::() +} + +/// One record compute logs retain, with its safe structured fields. +struct RetainedRecord<'a, 'py> { + record: &'a Bound<'py, PyAny>, + values: &'a Bound<'py, PyDict>, + facts: RecordFacts, + include_sql_text: bool, +} + +/// The message, logger and fields of a retained record, or an error the handler swallows. +fn retained<'py>( + retained: &RetainedRecord<'_, 'py>, + fields: Bound<'py, PyDict>, +) -> PyResult<(String, Bound<'py, PyAny>, Bound<'py, PyDict>)> { + let RetainedRecord { + record, + values, + facts, + include_sql_text, + } = *retained; + let py: Python<'py> = record.py(); + let contains_sql: bool = facts.has_sql_digest || facts.has_sql_text; + if !contains_sql { + let message: String = redacted_message(record)?; + let fields: Bound<'py, PyDict> = if facts.internal { + fields + } else { + extra_fields(record)? + }; + return Ok((message, record.getattr("name")?, fields)); + } + let sql_fields: Bound<'py, PyDict> = PyDict::new(py); + for (key, value) in fields.iter() { + if SQL_SAFE_FIELDS.contains(&key.extract::<&str>()?) { + sql_fields.set_item(key, value)?; + } + } + if include_sql_text && let Some(text) = values.get_item(SQL_TEXT_FIELD)? { + sql_fields.set_item(SQL_TEXT_DIAGNOSTIC_FIELD, text.str()?)?; + } + Ok(( + REDACTED_SQL_LOG_MESSAGE.to_owned(), + PyString::new(py, SQL_LOGGER_NAME).into_any(), + sql_fields, + )) +} + +/// The `DiagnosticLog` fields compute logs retain for a record, or `None` when it is not kept. +#[pyfunction] +#[pyo3(signature = (*, record, invocation_id, include_sql_text))] +fn compute_diagnostic_log_fields<'py>( + py: Python<'py>, + record: &Bound<'py, PyAny>, + invocation_id: &str, + include_sql_text: bool, +) -> PyResult>> { + let name: Bound<'py, PyAny> = record.getattr("name")?; + let Ok(name) = name.downcast::() else { + return Ok(None); + }; + let values: Bound<'py, PyDict> = record_values(record)?; + let level: i64 = record.getattr("levelno")?.extract()?; + let facts = RecordFacts { + internal: internal_logger(name.to_str()?), + level, + has_sql_digest: values.contains(SQL_DIGEST_FIELD)?, + has_sql_text: values.contains(SQL_TEXT_FIELD)?, + }; + if !compute_log_admits(facts, include_sql_text) { + return Ok(None); + } + let trusted_private_sql: bool = match values.get_item(SQL_PRIVATE_RECORD_FIELD)? { + Some(marker) => marker.is(&imported(py, PRIVATE_SQL_MARKER.0, PRIVATE_SQL_MARKER.1)?), + None => false, + }; + let mut identities: Vec<(&str, Option)> = Vec::new(); + for field in IDENTITY_FIELDS { + let value: Option = match values.get_item(format!("{FIELD_PREFIX}{field}"))? { + Some(value) if !value.is_none() => Some(value.str()?.to_str()?.to_owned()), + _ => None, + }; + identities.push((field, value)); + } + let fields: Bound<'py, PyDict> = PyDict::new(py); + for safe_name in SAFE_FIELDS { + let raw: Bound<'py, PyAny> = values + .get_item(format!("{FIELD_PREFIX}{safe_name}"))? + .unwrap_or_else(|| py.None().into_bound(py)); + let safe_value: Bound<'py, PyAny> = redacted_value(safe_name, &raw)?; + if scalar(&safe_value) { + fields.set_item(safe_name, safe_value)?; + } + } + let record_to_retain = RetainedRecord { + record, + values: &values, + facts, + include_sql_text, + }; + let Some((message, logger, fields)) = swallowed(py, retained(&record_to_retain, fields))? + else { + return Ok(None); + }; + let contains_sql: bool = facts.has_sql_digest || facts.has_sql_text; + let diagnostic: Bound<'py, PyDict> = PyDict::new(py); + diagnostic.set_item("producer", DIAGNOSTIC_PRODUCER)?; + diagnostic.set_item("severity", diagnostic_severity(level))?; + diagnostic.set_item("logger", logger)?; + diagnostic.set_item("source", DIAGNOSTIC_SOURCE)?; + diagnostic.set_item("message", message)?; + diagnostic.set_item("fields", fields)?; + for (field, value) in identities { + let value: Option = if contains_sql && !trusted_private_sql { + None + } else { + value + }; + let value: Option = if field == INVOCATION_ID_FIELD + && (contains_sql || value.as_deref().is_none_or(str::is_empty)) + { + Some(invocation_id.to_owned()) + } else { + value + }; + diagnostic.set_item(field, value)?; + } + Ok(Some(diagnostic)) +} + +pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { + module.add_function(wrap_pyfunction!(sql_log_record, module)?)?; + module.add_function(wrap_pyfunction!(debug_event_record_fields, module)?)?; + module.add_function(wrap_pyfunction!(route_diagnostic_log_record, module)?)?; + module.add_function(wrap_pyfunction!(safe_diagnostic_extra_fields, module)?)?; + module.add_function(wrap_pyfunction!(compute_diagnostic_log_fields, module)?) +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/diagnostics/mod.rs b/crates/sqlbuild-python/src/bindings/_helpers/diagnostics/mod.rs index 6570df35c8..52003770f1 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/diagnostics/mod.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/diagnostics/mod.rs @@ -1,4 +1,5 @@ pub(crate) mod console_records; +pub(crate) mod log_records; pub(crate) mod redaction; pub(crate) mod registration; pub(crate) mod trackers; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/diagnostics/redaction.rs b/crates/sqlbuild-python/src/bindings/_helpers/diagnostics/redaction.rs index 603adb3049..591f179773 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/diagnostics/redaction.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/diagnostics/redaction.rs @@ -32,7 +32,10 @@ fn redacted_mapping<'py>(value: &Bound<'py, PyAny>) -> PyResult(name: &str, value: &Bound<'py, PyAny>) -> PyResult> { +pub(crate) fn redacted_value<'py>( + name: &str, + value: &Bound<'py, PyAny>, +) -> PyResult> { let py: Python<'py> = value.py(); if secret_field_name(name) { return Ok(PyString::new(py, REDACTED).into_any()); diff --git a/crates/sqlbuild-python/src/bindings/_helpers/diagnostics/registration.rs b/crates/sqlbuild-python/src/bindings/_helpers/diagnostics/registration.rs index a0e1da2c15..f3a44ab4cd 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/diagnostics/registration.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/diagnostics/registration.rs @@ -2,10 +2,11 @@ use pyo3::prelude::{Bound, PyModule, PyResult}; -use crate::bindings::_helpers::diagnostics::{console_records, redaction, trackers}; +use crate::bindings::_helpers::diagnostics::{console_records, log_records, redaction, trackers}; pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { console_records::register(module)?; + log_records::register(module)?; redaction::register(module)?; trackers::register(module) } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/lifecycle_events/catalog.rs b/crates/sqlbuild-python/src/bindings/_helpers/lifecycle_events/catalog.rs index b31699fda1..411f03bc91 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/lifecycle_events/catalog.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/lifecycle_events/catalog.rs @@ -5,8 +5,8 @@ use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyModule, PyModuleMethods, PyRes use pyo3::types::{PyFrozenSet, PyString}; use pyo3::{pyfunction, wrap_pyfunction}; use sqlbuild_core::lifecycle_events::constants::{ - CURRENT_LIFECYCLE_EVENT_SCHEMA_VERSION, HOOK_PHASES, HOOK_TYPES, - LIFECYCLE_EVENT_SCHEMA_VERSIONS, OPERATION_ADAPTERS, OPERATION_KINDS, + CURRENT_DIAGNOSTIC_LOG_SCHEMA_VERSION, CURRENT_LIFECYCLE_EVENT_SCHEMA_VERSION, HOOK_PHASES, + HOOK_TYPES, LIFECYCLE_EVENT_SCHEMA_VERSIONS, OPERATION_ADAPTERS, OPERATION_KINDS, OPERATION_METADATA_FIELDS, OPERATION_NAMES, OPERATION_PHASES, OPERATION_SCOPES, OPERATION_STRATEGIES, OPERATION_TARGET_KINDS, RESOURCE_SKIP_CODES, RESOURCE_SKIP_MODES, }; @@ -57,6 +57,10 @@ pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { "CURRENT_LIFECYCLE_EVENT_SCHEMA_VERSION", CURRENT_LIFECYCLE_EVENT_SCHEMA_VERSION, )?; + module.add( + "CURRENT_DIAGNOSTIC_LOG_SCHEMA_VERSION", + CURRENT_DIAGNOSTIC_LOG_SCHEMA_VERSION, + )?; module.add( "LIFECYCLE_EVENT_SCHEMA_VERSIONS", PyFrozenSet::new(py, LIFECYCLE_EVENT_SCHEMA_VERSIONS)?, diff --git a/crates/sqlbuild-python/src/bindings/_helpers/lifecycle_events/envelopes.rs b/crates/sqlbuild-python/src/bindings/_helpers/lifecycle_events/envelopes.rs new file mode 100644 index 0000000000..5864156618 --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/lifecycle_events/envelopes.rs @@ -0,0 +1,306 @@ +//! Lifecycle event and diagnostic log envelopes, run bounds and operation field rules. + +use pyo3::exceptions::PyTypeError; +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyModule, PyModuleMethods, PyResult, Python}; +use pyo3::types::{ + PyBool, PyByteArray, PyBytes, PyDict, PyDictMethods, PyInt, PyString, PyTypeMethods, +}; +use pyo3::{pyfunction, wrap_pyfunction}; +use sqlbuild_core::lifecycle_events::constants::INVOCATION_SEQUENCE_PROBLEM; +use sqlbuild_core::lifecycle_events::main::diagnostic_severity_problem::diagnostic_severity_problem; +use sqlbuild_core::lifecycle_events::main::diagnostic_version_problem::diagnostic_version_problem; +use sqlbuild_core::lifecycle_events::main::hook_problem::hook_problem; +use sqlbuild_core::lifecycle_events::main::lifecycle_version_problem::lifecycle_version_problem; +use sqlbuild_core::lifecycle_events::main::operation_metadata_field::operation_metadata_field; +use sqlbuild_core::lifecycle_events::main::producer_version_problem::producer_version_problem; +use sqlbuild_core::lifecycle_events::main::publishable_hook_name::publishable_hook_name; +use sqlbuild_core::lifecycle_events::main::run_bounds_problem::run_bounds_problem; +use sqlbuild_core::lifecycle_events::models::{HookFacts, SequenceFact}; + +use crate::bindings::_helpers::compile_driver::python_calls::imported; +use crate::bindings::_helpers::lifecycle_events::payloads::validation_error; +use crate::bindings::_helpers::lifecycle_events::validation::{ + frozen_json, known_event_check, optional_text_check, required_text_check, schema_version_check, + timestamp_check, +}; + +const EVENT_REQUIRED_TEXT: [&str; 5] = [ + "event_id", + "event_type", + "producer", + "producer_version", + "invocation_id", +]; +const EVENT_OPTIONAL_TEXT: [&str; 5] = [ + "run_id", + "resource_id", + "resource_attempt_id", + "operation_id", + "statement_id", +]; +const DIAGNOSTIC_REQUIRED_TEXT: [&str; 6] = [ + "producer", + "producer_version", + "severity", + "logger", + "source", + "message", +]; +const DIAGNOSTIC_OPTIONAL_TEXT: [&str; 7] = [ + "log_stream_id", + "invocation_id", + "run_id", + "resource_id", + "resource_attempt_id", + "operation_id", + "statement_id", +]; + +/// Whether a value is an `int` other than a `bool`. +fn plain_int(value: &Bound<'_, PyAny>) -> bool { + !value.is_instance_of::() && value.is_instance_of::() +} + +/// Whether a value is a non-negative `int` other than a `bool`. +fn nonnegative_int(value: &Bound<'_, PyAny>) -> PyResult { + Ok(plain_int(value) && !value.lt(0)?) +} + +/// An integer, saturated into `i128`. +fn saturated(value: &Bound<'_, PyAny>) -> PyResult { + match value.extract::() { + Ok(number) => Ok(number), + Err(_) if value.lt(0)? => Ok(i128::MIN), + Err(_) => Ok(i128::MAX), + } +} + +/// The text a value holds when it is a string. +fn text(value: &Bound<'_, PyAny>) -> PyResult> { + match value.downcast::() { + Ok(text) => Ok(Some(text.extract()?)), + Err(_) => Ok(None), + } +} + +/// Freeze a JSON object field of a frozen dataclass in place. +fn freeze_object_field(owner: &Bound<'_, PyAny>, field: &str, path: &str) -> PyResult<()> { + let py: Python<'_> = owner.py(); + let frozen: Bound<'_, PyAny> = frozen_json(&owner.getattr(field)?, path)?; + if !frozen.is_instance(&imported(py, "collections.abc", "Mapping")?)? { + return Err(validation_error( + py, + &format!("{path} must be a JSON object"), + )); + } + imported(py, "builtins", "object")? + .getattr("__setattr__")? + .call1((owner, field, frozen))?; + Ok(()) +} + +/// Validate and freeze one lifecycle event envelope in place. +#[pyfunction(name = "validate_lifecycle_event")] +#[pyo3(signature = (*, event))] +fn lifecycle_event_check(event: &Bound<'_, PyAny>) -> PyResult<()> { + let py: Python<'_> = event.py(); + for field in EVENT_REQUIRED_TEXT { + required_text_check(&event.getattr(field)?, field)?; + } + let schema_version: Bound<'_, PyAny> = event.getattr("schema_version")?; + schema_version_check(&schema_version)?; + let sequence: Bound<'_, PyAny> = event.getattr("invocation_sequence")?; + let sequence: SequenceFact = if sequence.is_none() { + SequenceFact::Absent + } else if nonnegative_int(&sequence)? { + SequenceFact::Valid + } else { + SequenceFact::Invalid + }; + let has_external_context: bool = event.getattr("external_context")?.is_truthy()?; + if let Some(problem) = + lifecycle_version_problem(saturated(&schema_version)?, sequence, has_external_context) + { + return Err(validation_error(py, &problem)); + } + timestamp_check(&event.getattr("occurred_at")?)?; + for field in EVENT_OPTIONAL_TEXT { + optional_text_check(&event.getattr(field)?, field)?; + } + freeze_object_field(event, "external_context", "external_context")?; + freeze_object_field(event, "payload", "payload")?; + known_event_check(event) +} + +/// Validate a dispatcher-assigned invocation sequence. +#[pyfunction(name = "validate_lifecycle_invocation_sequence")] +#[pyo3(signature = (*, value))] +fn invocation_sequence_check(value: &Bound<'_, PyAny>) -> PyResult<()> { + if !nonnegative_int(value)? { + return Err(validation_error(value.py(), INVOCATION_SEQUENCE_PROBLEM)); + } + Ok(()) +} + +/// Validate and freeze one diagnostic log record in place. +#[pyfunction(name = "validate_diagnostic_log")] +#[pyo3(signature = (*, record))] +fn diagnostic_log_check(record: &Bound<'_, PyAny>) -> PyResult<()> { + let py: Python<'_> = record.py(); + let schema_version: Bound<'_, PyAny> = record.getattr("schema_version")?; + schema_version_check(&schema_version)?; + if let Some(problem) = diagnostic_version_problem(saturated(&schema_version)?) { + return Err(validation_error(py, &problem)); + } + for field in DIAGNOSTIC_REQUIRED_TEXT { + required_text_check(&record.getattr(field)?, field)?; + } + let severity: String = record.getattr("severity")?.extract()?; + if let Some(problem) = diagnostic_severity_problem(&severity) { + return Err(validation_error(py, problem)); + } + timestamp_check(&record.getattr("occurred_at")?)?; + for field in DIAGNOSTIC_OPTIONAL_TEXT { + optional_text_check(&record.getattr(field)?, field)?; + } + freeze_object_field(record, "fields", "fields") +} + +/// Validate a run's selection, concurrency and worker counts. +#[pyfunction(name = "validate_run_bounds")] +#[pyo3(signature = (*, selected_count, configured_concurrency, worker_count))] +fn run_bounds_check( + selected_count: &Bound<'_, PyAny>, + configured_concurrency: &Bound<'_, PyAny>, + worker_count: &Bound<'_, PyAny>, +) -> PyResult<()> { + let py: Python<'_> = selected_count.py(); + let values: [(&str, &Bound<'_, PyAny>); 3] = [ + ("selected_count", selected_count), + ("configured_concurrency", configured_concurrency), + ("worker_count", worker_count), + ]; + for (field, value) in values { + if !plain_int(value) { + return Err(validation_error( + py, + &format!("{field} must be an integer excluding bool"), + )); + } + } + match run_bounds_problem( + saturated(selected_count)?, + saturated(configured_concurrency)?, + saturated(worker_count)?, + ) { + Some(problem) => Err(validation_error(py, problem)), + None => Ok(()), + } +} + +/// The allow-listed, non-negative operation metadata counts. +#[pyfunction] +#[pyo3(signature = (*, metadata))] +fn validated_operation_metadata<'py>( + py: Python<'py>, + metadata: &Bound<'py, PyAny>, +) -> PyResult> { + let result: Bound<'py, PyDict> = PyDict::new(py); + if metadata.is_none() { + return Ok(result); + } + for key in metadata.try_iter()? { + if !operation_metadata_field(text(&key?)?.as_deref()) { + return Err(validation_error( + py, + "operation metadata contains a non-allowlisted field", + )); + } + } + for item in metadata.call_method0("items")?.try_iter()? { + let (key, value): (Bound<'py, PyAny>, Bound<'py, PyAny>) = item?.extract()?; + if !nonnegative_int(&value)? { + return Err(validation_error( + py, + "operation metadata values must be nonnegative integers excluding bool", + )); + } + result.set_item(key, value)?; + } + Ok(result) +} + +/// Whether a hook name is publishable, raising like `re.fullmatch` for non-string names. +fn hook_name_publishable(name: &Bound<'_, PyAny>) -> PyResult { + if let Some(name) = text(name)? { + return Ok(publishable_hook_name(&name)); + } + if name.is_instance_of::() || name.is_instance_of::() { + return Err(PyTypeError::new_err( + "cannot use a string pattern on a bytes-like object", + )); + } + let type_name: String = name.get_type().name()?.extract()?; + Err(PyTypeError::new_err(format!( + "expected string or bytes-like object, got '{type_name}'" + ))) +} + +/// The validated hook fields of an operation, empty when none are supplied. +#[pyfunction] +#[pyo3(signature = (*, hook_phase, hook_index, hook_type, hook_name))] +fn validated_operation_hook_fields<'py>( + hook_phase: &Bound<'py, PyAny>, + hook_index: &Bound<'py, PyAny>, + hook_type: &Bound<'py, PyAny>, + hook_name: &Bound<'py, PyAny>, +) -> PyResult> { + let py: Python<'py> = hook_phase.py(); + let result: Bound<'py, PyDict> = PyDict::new(py); + let supplied: bool = [hook_phase, hook_index, hook_type, hook_name] + .iter() + .any(|value| !value.is_none()); + if !supplied { + return Ok(result); + } + let phase: Option = text(hook_phase)?; + let hook_type_text: Option = text(hook_type)?; + let facts = HookFacts { + phase: phase.as_deref(), + hook_type: hook_type_text.as_deref(), + valid_index: nonnegative_int(hook_index)?, + }; + if let Some(problem) = hook_problem(facts) { + return Err(validation_error(py, problem)); + } + result.set_item("hook_phase", hook_phase)?; + result.set_item("hook_index", hook_index)?; + result.set_item("hook_type", hook_type)?; + if !hook_name.is_none() && hook_name_publishable(hook_name)? { + result.set_item("hook_name", hook_name)?; + } + Ok(result) +} + +/// Validate that a producer other than SQLBuild names its version. +#[pyfunction(name = "validate_lifecycle_producer")] +#[pyo3(signature = (*, producer, producer_version))] +fn producer_check( + producer: &Bound<'_, PyAny>, + producer_version: &Bound<'_, PyAny>, +) -> PyResult<()> { + match producer_version_problem(text(producer)?.as_deref(), !producer_version.is_none()) { + Some(problem) => Err(validation_error(producer.py(), problem)), + None => Ok(()), + } +} + +pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { + module.add_function(wrap_pyfunction!(lifecycle_event_check, module)?)?; + module.add_function(wrap_pyfunction!(invocation_sequence_check, module)?)?; + module.add_function(wrap_pyfunction!(diagnostic_log_check, module)?)?; + module.add_function(wrap_pyfunction!(run_bounds_check, module)?)?; + module.add_function(wrap_pyfunction!(validated_operation_metadata, module)?)?; + module.add_function(wrap_pyfunction!(validated_operation_hook_fields, module)?)?; + module.add_function(wrap_pyfunction!(producer_check, module)?) +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/lifecycle_events/mod.rs b/crates/sqlbuild-python/src/bindings/_helpers/lifecycle_events/mod.rs index 00d4e7017d..ccd5f51ee1 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/lifecycle_events/mod.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/lifecycle_events/mod.rs @@ -1,4 +1,5 @@ pub(crate) mod catalog; +pub(crate) mod envelopes; pub(crate) mod payloads; pub(crate) mod registration; pub(crate) mod validation; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/lifecycle_events/registration.rs b/crates/sqlbuild-python/src/bindings/_helpers/lifecycle_events/registration.rs index 11c2a473a2..1bb8d62067 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/lifecycle_events/registration.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/lifecycle_events/registration.rs @@ -2,9 +2,10 @@ use pyo3::prelude::{Bound, PyModule, PyResult}; -use crate::bindings::_helpers::lifecycle_events::{catalog, validation}; +use crate::bindings::_helpers::lifecycle_events::{catalog, envelopes, validation}; pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { catalog::register(module)?; + envelopes::register(module)?; validation::register(module) } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/lifecycle_events/validation.rs b/crates/sqlbuild-python/src/bindings/_helpers/lifecycle_events/validation.rs index 2b4a55b45c..9c609ba121 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/lifecycle_events/validation.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/lifecycle_events/validation.rs @@ -22,7 +22,7 @@ const CORRELATION_FIELDS: [&str; 5] = [ /// Validate the shared positive integer schema-version contract. #[pyfunction(name = "validate_lifecycle_schema_version")] #[pyo3(signature = (*, value))] -fn schema_version_check(value: &Bound<'_, PyAny>) -> PyResult<()> { +pub(crate) fn schema_version_check(value: &Bound<'_, PyAny>) -> PyResult<()> { if value.is_instance_of::() || !value.is_instance_of::() || value.le(0)? { return Err(validation_error( value.py(), @@ -35,14 +35,17 @@ fn schema_version_check(value: &Bound<'_, PyAny>) -> PyResult<()> { /// Validate and recursively freeze a JSON-compatible value. #[pyfunction(name = "freeze_lifecycle_json")] #[pyo3(signature = (*, value, path))] -fn frozen_json<'py>(value: &Bound<'py, PyAny>, path: &str) -> PyResult> { +pub(crate) fn frozen_json<'py>( + value: &Bound<'py, PyAny>, + path: &str, +) -> PyResult> { frozen(value, &[PathSegment::Key(path.to_owned())]) } /// Validate one required non-empty string field. #[pyfunction(name = "validate_lifecycle_required_text")] #[pyo3(signature = (*, value, field_name))] -fn required_text_check(value: &Bound<'_, PyAny>, field_name: &str) -> PyResult<()> { +pub(crate) fn required_text_check(value: &Bound<'_, PyAny>, field_name: &str) -> PyResult<()> { let valid: bool = value .downcast::() .is_ok_and(|text| text.len().is_ok_and(|length| length > 0)); @@ -58,7 +61,7 @@ fn required_text_check(value: &Bound<'_, PyAny>, field_name: &str) -> PyResult<( /// Validate one optional non-empty string field. #[pyfunction(name = "validate_lifecycle_optional_text")] #[pyo3(signature = (*, value, field_name))] -fn optional_text_check(value: &Bound<'_, PyAny>, field_name: &str) -> PyResult<()> { +pub(crate) fn optional_text_check(value: &Bound<'_, PyAny>, field_name: &str) -> PyResult<()> { if value.is_none() { return Ok(()); } @@ -68,7 +71,7 @@ fn optional_text_check(value: &Bound<'_, PyAny>, field_name: &str) -> PyResult<( /// Validate a timezone-aware UTC timestamp. #[pyfunction(name = "validate_lifecycle_timestamp")] #[pyo3(signature = (*, value))] -fn timestamp_check(value: &Bound<'_, PyAny>) -> PyResult<()> { +pub(crate) fn timestamp_check(value: &Bound<'_, PyAny>) -> PyResult<()> { let py = value.py(); let datetime: Bound<'_, PyAny> = imported(py, "datetime", "datetime")?; if !value.is_instance(&datetime)? || value.getattr("tzinfo")?.is_none() { @@ -88,7 +91,7 @@ fn timestamp_check(value: &Bound<'_, PyAny>) -> PyResult<()> { /// Validate correlations and safe payload fields for a catalogued event. #[pyfunction(name = "validate_known_lifecycle_event")] #[pyo3(signature = (*, event))] -fn known_event_check(event: &Bound<'_, PyAny>) -> PyResult<()> { +pub(crate) fn known_event_check(event: &Bound<'_, PyAny>) -> PyResult<()> { let mut correlations: Vec<(&'static str, bool)> = Vec::new(); for field in CORRELATION_FIELDS { correlations.push((field, !event.getattr(field)?.is_none())); diff --git a/python-boundary.toml b/python-boundary.toml index 3315d9e84a..7ce5069489 100644 --- a/python-boundary.toml +++ b/python-boundary.toml @@ -60,17 +60,12 @@ note = "Public custom-rule API and test helper." [[stays]] path = "diagnostics/_helpers/logging.py" rule = 4 -note = "Installs SQLBuild's handlers on Python's logging, which users and integrations configure." - -[[stays]] -path = "diagnostics/_helpers/redaction.py" -rule = 4 -note = "Collects redacted extra fields from a Python LogRecord; the redaction itself is native." +note = "Installs SQLBuild's handlers on Python's logging and owns the diagnostics context variables; SQL digests and record field sets are native (sql_log_record, debug_event_record_fields)." [[stays]] path = "diagnostics/classes/diagnostic_record_redactor.py" rule = 4 -note = "Redaction entry points that Python logging handlers call per record." +note = "Redaction entry points that Python logging handlers call per record; text, value and extra-field redaction are native." [[stays]] path = "diagnostics/classes/diagnostics_console_formatter.py" @@ -90,7 +85,7 @@ note = "Attaches and removes per-invocation logging handlers." [[stays]] path = "diagnostics/classes/log_record_route_filter.py" rule = 4 -note = "A logging.Filter that routes records to destinations." +note = "A logging.Filter subclass; routing and SQL redaction of the record copy are native (route_diagnostic_log_record)." [[stays]] path = "diagnostics/main" @@ -215,7 +210,7 @@ note = "Encodes the Python fact-view values user rules read, for their digests." [[stays]] path = "runtime/observability" rule = 1 -note = "Documented observability API used throughout the executor: lifecycle emitters, dispatcher, scopes and envelope shapes. The event catalogue, payload validation and JSON freezing are native (sqlbuild-core lifecycle_events)." +note = "Documented observability API used throughout the executor: lifecycle emitters, dispatcher, scopes and envelope shapes. The event catalogue, envelope, diagnostic log, run bound, operation metadata, hook and producer validation and JSON freezing are native (sqlbuild-core lifecycle_events)." [[stays]] path = "runtime/execution_history/__init__.py" @@ -245,7 +240,7 @@ note = "Deadline the executor scheduler and warehouse cursors enforce." [[stays]] path = "runtime/compute_logs/classes" rule = 1 -note = "Rule 1 capture shim: sys.stdout/sys.stderr tees and a logging handler that record Python output from user code, adapters and the executor (_dispatch_with_compute_logs.py, compile_command/reuse_attempt.rs); storage itself is native LocalFilesystemComputeLogStorage." +note = "Rule 1 capture shim: sys.stdout/sys.stderr tees and a logging.Handler that record Python output from user code, adapters and the executor (_dispatch_with_compute_logs.py, compile_command/reuse_attempt.rs); record retention, SQL redaction, identity and severity are native (compute_diagnostic_log_fields) and storage is native LocalFilesystemComputeLogStorage." [[stays]] path = "runtime/compute_logs/main" @@ -735,7 +730,7 @@ note = "Shape rule: data exceptions only (DbtProfileError), no logic; consumed b [[stays]] path = "diagnostics/constants.py" rule = 4 -note = "Shape rule: Python logging policy constants only (LOGGER_ROOT_NAME, SQL_DIGEST_FIELD, SQL_PRIVATE_RECORD_FIELD, REDACTED_SQL_LOG_MESSAGE, ...), no logic; the logger prefix and SQL text field are native (sqlbuild._native)." +note = "Shape rule: data constants only (SQL_DIGEST_FIELD, SQL_TEXT_FIELD, SQL_PRIVATE_RECORD_FIELD, SQL_PRIVATE_RECORD_MARKER), no logic; the marker object identifies trusted private SQL records for the native compute log handler; consumed by diagnostics/classes/invocation_diagnostic_routing.py." [[stays]] path = "cursor_algebra/exceptions.py" diff --git a/src/sqlbuild/_native.pyi b/src/sqlbuild/_native.pyi index 4572912a07..9485ccd8c9 100644 --- a/src/sqlbuild/_native.pyi +++ b/src/sqlbuild/_native.pyi @@ -5094,6 +5094,7 @@ OPERATION_METADATA_FIELDS: frozenset[str] OPERATION_NAMES: frozenset[str] OPERATION_PHASES: frozenset[str] OPERATION_SCOPES: frozenset[str] +CURRENT_DIAGNOSTIC_LOG_SCHEMA_VERSION: int OPERATION_STRATEGIES: frozenset[str] OPERATION_TARGET_KINDS: frozenset[str] RESOURCE_SKIP_CODES: frozenset[str] @@ -5116,6 +5117,21 @@ def render_state_table_create_sql( render_framework_type: Callable[[FrameworkType], str], ) -> str: ... def render_insert_source_freshness_records_sql( +def validate_lifecycle_event(*, event: LifecycleEvent) -> None: ... +def validate_lifecycle_invocation_sequence(*, value: object) -> None: ... +def validate_diagnostic_log(*, record: DiagnosticLog) -> None: ... +def validate_run_bounds( + *, selected_count: int, configured_concurrency: int, worker_count: int +) -> None: ... +def validated_operation_metadata(*, metadata: Mapping[str, int] | None) -> dict[str, int]: ... +def validated_operation_hook_fields( + *, + hook_phase: str | None, + hook_index: int | None, + hook_type: str | None, + hook_name: str | None, +) -> dict[str, JSONValue]: ... +def validate_lifecycle_producer(*, producer: str, producer_version: str | None) -> None: ... *, database: str | None, schema: str, @@ -6637,6 +6653,32 @@ def sql_file_rule_key( ) -> str | None: ... def load_sql_rule_cache(*, path: Path) -> dict[str, tuple[str, str]]: ... def lookup_sql_rule_cache( +def sql_log_record( + *, + context: Mapping[str, object] | None, + identity: Mapping[str, str | None] | None, + sql: str, + action: str, + intent: str | None, + query_id: str | None, +) -> tuple[str, dict[str, object]]: ... +def debug_event_record_fields( + *, + context: Mapping[str, object] | None, + event_fields: Mapping[str, object], + identity: Mapping[str, str | None] | None, +) -> dict[str, object]: ... +def route_diagnostic_log_record( + *, + record: logging.LogRecord, + internal_only: bool, + allow_internal: bool, + include_sql_text: bool, +) -> dict[str, object] | None: ... +def safe_diagnostic_extra_fields(record: logging.LogRecord) -> dict[str, object]: ... +def compute_diagnostic_log_fields( + *, record: logging.LogRecord, invocation_id: str, include_sql_text: bool +) -> dict[str, object] | None: ... *, entries: dict[str, tuple[str, str]], keys: list[tuple[str, str]] ) -> dict[str, list[Finding]]: ... def encode_sql_rule_findings(*, findings: list[Finding]) -> str: ... diff --git a/src/sqlbuild/diagnostics/_helpers/logging.py b/src/sqlbuild/diagnostics/_helpers/logging.py index 30d3ff00d3..efe555958f 100644 --- a/src/sqlbuild/diagnostics/_helpers/logging.py +++ b/src/sqlbuild/diagnostics/_helpers/logging.py @@ -2,13 +2,12 @@ from __future__ import annotations -import hashlib import logging from collections.abc import Callable, Iterator from contextlib import contextmanager from contextvars import ContextVar, Token -from sqlbuild._native import SQL_TEXT_FIELD +from sqlbuild._native import debug_event_record_fields, sql_log_record from sqlbuild.runtime.observability.main.current_execution_identity import ( current_execution_identity, ) @@ -27,11 +26,11 @@ ) -def _current_identity_context() -> dict[str, object]: +def _current_identity_fields() -> dict[str, str | None] | None: identity: ExecutionIdentity | None = current_execution_identity() if identity is None: - return {} - return {f"sqlbuild_{key}": value for key, value in execution_identity_to_dict(identity).items()} + return None + return execution_identity_to_dict(identity) def log_sql( @@ -44,28 +43,15 @@ def log_sql( ) -> None: """Log SQL passed to an adapter or connection.""" - contextual_fields: dict[str, object] = { - key: value - for key, value in (_DIAGNOSTICS_CONTEXT.get() or _EMPTY_CONTEXT).items() - if key != SQL_TEXT_FIELD - } - sql_fields: dict[str, object] = { - "sqlbuild_sql_action": action, - "sqlbuild_sql_digest": f"sha256:{hashlib.sha256(sql.encode('utf-8')).hexdigest()}", - } - if intent is not None: - sql_fields["sqlbuild_sql_intent"] = intent - if query_id is not None: - sql_fields["sqlbuild_sql_query_id"] = query_id - record_fields: dict[str, object] = { - **contextual_fields, - **sql_fields, - **_current_identity_context(), - } - logger.debug( - f"{action} SQL", - extra=record_fields, + message, record_fields = sql_log_record( + context=_DIAGNOSTICS_CONTEXT.get(), + identity=_current_identity_fields(), + sql=sql, + action=action, + intent=intent, + query_id=query_id, ) + logger.debug(message, extra=record_fields) sql_sink: Callable[..., None] | None = _SQL_DIAGNOSTIC_SINK.get() if sql_sink is not None: sql_sink(logger=logger, sql=sql, fields=record_fields) @@ -89,11 +75,11 @@ def log_debug_event(*, logger: logging.Logger, message: str, **context: object) logger.debug( message, - extra={ - **(_DIAGNOSTICS_CONTEXT.get() or _EMPTY_CONTEXT), - **context, - **_current_identity_context(), - }, + extra=debug_event_record_fields( + context=_DIAGNOSTICS_CONTEXT.get(), + event_fields=context, + identity=_current_identity_fields(), + ), ) diff --git a/src/sqlbuild/diagnostics/_helpers/redaction.py b/src/sqlbuild/diagnostics/_helpers/redaction.py deleted file mode 100644 index 60e197fd1c..0000000000 --- a/src/sqlbuild/diagnostics/_helpers/redaction.py +++ /dev/null @@ -1,23 +0,0 @@ -"""Redacted structured fields of one logging record.""" - -from __future__ import annotations - -import logging -from typing import Final - -from sqlbuild._native import SQL_TEXT_FIELD, redact_diagnostic_value -from sqlbuild.diagnostics.constants import FORMATTER_RECORD_FIELDS - -_STANDARD_RECORD_FIELDS: Final[frozenset[str]] = frozenset(logging.makeLogRecord({}).__dict__) - - -def safe_extra_fields(record: logging.LogRecord) -> dict[str, object]: - """Collect redacted non-standard fields without evaluating arbitrary formatters.""" - - return { - key: redact_diagnostic_value(name=key, value=value) - for key, value in record.__dict__.items() - if key not in _STANDARD_RECORD_FIELDS - and key not in FORMATTER_RECORD_FIELDS - and key != SQL_TEXT_FIELD - } diff --git a/src/sqlbuild/diagnostics/classes/diagnostic_record_redactor.py b/src/sqlbuild/diagnostics/classes/diagnostic_record_redactor.py index d7a4a12f56..f3fd0216b1 100644 --- a/src/sqlbuild/diagnostics/classes/diagnostic_record_redactor.py +++ b/src/sqlbuild/diagnostics/classes/diagnostic_record_redactor.py @@ -4,8 +4,11 @@ import logging -from sqlbuild._native import redact_diagnostic_text, redact_diagnostic_value -from sqlbuild.diagnostics._helpers.redaction import safe_extra_fields +from sqlbuild._native import ( + redact_diagnostic_text, + redact_diagnostic_value, + safe_diagnostic_extra_fields, +) class DiagnosticRecordRedactor: @@ -27,4 +30,4 @@ def value(*, name: str, value: object) -> object: def extras(record: logging.LogRecord) -> dict[str, object]: """Collect redacted non-standard record fields.""" - return safe_extra_fields(record) + return safe_diagnostic_extra_fields(record) diff --git a/src/sqlbuild/diagnostics/classes/log_record_route_filter.py b/src/sqlbuild/diagnostics/classes/log_record_route_filter.py index 5f37e19531..b25ce3030b 100644 --- a/src/sqlbuild/diagnostics/classes/log_record_route_filter.py +++ b/src/sqlbuild/diagnostics/classes/log_record_route_filter.py @@ -4,13 +4,7 @@ import logging -from sqlbuild._native import LOGGER_PREFIX, SQL_TEXT_FIELD -from sqlbuild.diagnostics.classes.diagnostic_record_redactor import DiagnosticRecordRedactor -from sqlbuild.diagnostics.constants import ( - LOGGER_ROOT_NAME, - REDACTED_SQL_LOG_MESSAGE, - SQL_DIGEST_FIELD, -) +from sqlbuild._native import route_diagnostic_log_record class LogRecordRouteFilter(logging.Filter): @@ -31,40 +25,12 @@ def __init__( def filter(self, record: logging.LogRecord) -> logging.LogRecord | bool: """Return a safe copy when the record belongs at this destination.""" - internal: bool = record.name == LOGGER_ROOT_NAME or record.name.startswith(LOGGER_PREFIX) - if self._internal_only and not internal: + values: dict[str, object] | None = route_diagnostic_log_record( + record=record, + internal_only=self._internal_only, + allow_internal=self._allow_internal, + include_sql_text=self._include_sql_text, + ) + if values is None: return False - if internal and not self._allow_internal: - return False - if not internal and record.levelno < logging.INFO: - return False - if ( - self._include_sql_text - and SQL_DIGEST_FIELD in record.__dict__ - and SQL_TEXT_FIELD not in record.__dict__ - ): - return False - copied_values: dict[str, object] = dict(record.__dict__) - try: - copied_values["msg"] = DiagnosticRecordRedactor.text(record.getMessage()) - except Exception: - copied_values["msg"] = "" - copied_values["args"] = () - if SQL_DIGEST_FIELD in copied_values: - copied_values["msg"] = REDACTED_SQL_LOG_MESSAGE - if record.exc_info is not None: - try: - exception_text: str = logging.Formatter().formatException(record.exc_info) - copied_values["msg"] = ( - f"{copied_values['msg']}\n{DiagnosticRecordRedactor.text(exception_text)}" - ) - except Exception: - pass - copied_values["exc_info"] = None - copied_values["exc_text"] = None - for key, value in tuple(copied_values.items()): - if key.startswith("sqlbuild_") and key != SQL_TEXT_FIELD: - copied_values[key] = DiagnosticRecordRedactor.value(name=key, value=value) - if not self._include_sql_text: - copied_values.pop(SQL_TEXT_FIELD, None) - return logging.makeLogRecord(copied_values) + return logging.makeLogRecord(values) diff --git a/src/sqlbuild/diagnostics/constants.py b/src/sqlbuild/diagnostics/constants.py index 72c33e8eda..86d63524ae 100644 --- a/src/sqlbuild/diagnostics/constants.py +++ b/src/sqlbuild/diagnostics/constants.py @@ -1,8 +1,3 @@ """Diagnostic logging policy constants.""" -LOGGER_ROOT_NAME: str = "sqlbuild" -SQL_DIGEST_FIELD: str = "sqlbuild_sql_digest" -SQL_PRIVATE_RECORD_FIELD: str = "sqlbuild_private_sql_record" SQL_PRIVATE_RECORD_MARKER: object = object() -REDACTED_SQL_LOG_MESSAGE: str = "SQL diagnostic omitted" -FORMATTER_RECORD_FIELDS: frozenset[str] = frozenset(("message", "asctime")) diff --git a/src/sqlbuild/runtime/compute_logs/classes/diagnostic_log_handler.py b/src/sqlbuild/runtime/compute_logs/classes/diagnostic_log_handler.py index 03d3322de7..cc2b396af6 100644 --- a/src/sqlbuild/runtime/compute_logs/classes/diagnostic_log_handler.py +++ b/src/sqlbuild/runtime/compute_logs/classes/diagnostic_log_handler.py @@ -7,34 +7,11 @@ from datetime import UTC, datetime from functools import cache from importlib.metadata import PackageNotFoundError, version -from typing import cast +from typing import Any, cast -from sqlbuild._native import LOGGER_PREFIX, SQL_TEXT_FIELD -from sqlbuild.diagnostics.classes.diagnostic_record_redactor import DiagnosticRecordRedactor -from sqlbuild.diagnostics.constants import ( - LOGGER_ROOT_NAME, - REDACTED_SQL_LOG_MESSAGE, - SQL_DIGEST_FIELD, - SQL_PRIVATE_RECORD_FIELD, - SQL_PRIVATE_RECORD_MARKER, -) +from sqlbuild._native import CURRENT_DIAGNOSTIC_LOG_SCHEMA_VERSION, compute_diagnostic_log_fields from sqlbuild.runtime.compute_logs.types import ComputeLogStorage -from sqlbuild.runtime.observability.constants import CURRENT_DIAGNOSTIC_LOG_SCHEMA_VERSION from sqlbuild.runtime.observability.models import DiagnosticLog -from sqlbuild.runtime.observability.types import JSONValue - -_IDENTITY_FIELDS: tuple[str, ...] = ( - "invocation_id", - "run_id", - "resource_id", - "resource_attempt_id", - "operation_id", - "statement_id", - "log_stream_id", -) -_SQL_SAFE_FIELDS: frozenset[str] = frozenset( - ("sql_action", "sql_digest", "sql_intent", "sql_query_id") -) class ComputeDiagnosticLogHandler(logging.Handler): @@ -58,71 +35,21 @@ def __init__( def emit(self, record: logging.LogRecord) -> None: """Convert eligible records while excluding SQL and arbitrary parameter fields.""" - if self._failed or not isinstance(record.name, str): - return - internal: bool = record.name == LOGGER_ROOT_NAME or record.name.startswith(LOGGER_PREFIX) - if not internal and record.levelno < logging.INFO: + if self._failed: return - contains_sql: bool = ( - SQL_DIGEST_FIELD in record.__dict__ or SQL_TEXT_FIELD in record.__dict__ - ) - trusted_private_sql: bool = ( - record.__dict__.get(SQL_PRIVATE_RECORD_FIELD) is SQL_PRIVATE_RECORD_MARKER + fields: dict[str, object] | None = compute_diagnostic_log_fields( + record=record, + invocation_id=self._invocation_id, + include_sql_text=self._include_sql_text, ) - if self._include_sql_text and contains_sql and SQL_TEXT_FIELD not in record.__dict__: + if fields is None: return - structured: dict[str, object] = { - key: value - for key, value in record.__dict__.items() - if key.startswith("sqlbuild_") and key != SQL_TEXT_FIELD - } - identities: dict[str, str | None] = {} - for field_name in _IDENTITY_FIELDS: - raw_value: object = structured.get(f"sqlbuild_{field_name}") - identities[field_name] = str(raw_value) if raw_value is not None else None - identities["invocation_id"] = identities["invocation_id"] or self._invocation_id - fields: dict[str, JSONValue] = {} - for safe_name in ( - "sql_action", - "sql_digest", - "sql_intent", - "sql_query_id", - "channel", - "subscriber", - "error_type", - ): - raw_value = structured.get(f"sqlbuild_{safe_name}") - safe_value: object = DiagnosticRecordRedactor.value(name=safe_name, value=raw_value) - if isinstance(safe_value, (str, int, float, bool)) or safe_value is None: - if safe_value is not None: - fields[safe_name] = safe_value try: - message: str = ( - REDACTED_SQL_LOG_MESSAGE - if contains_sql - else DiagnosticRecordRedactor.text(record.getMessage()) - ) - logger_name: str = "sqlbuild.sql" if contains_sql else record.name - if contains_sql: - fields = {key: value for key, value in fields.items() if key in _SQL_SAFE_FIELDS} - if not trusted_private_sql: - identities = {field_name: None for field_name in _IDENTITY_FIELDS} - identities["invocation_id"] = self._invocation_id - if self._include_sql_text and SQL_TEXT_FIELD in record.__dict__: - fields["sql_text"] = str(record.__dict__[SQL_TEXT_FIELD]) - elif not internal: - fields = cast(dict[str, JSONValue], DiagnosticRecordRedactor.extras(record)) diagnostic: DiagnosticLog = DiagnosticLog( schema_version=CURRENT_DIAGNOSTIC_LOG_SCHEMA_VERSION, - producer="sqlbuild", producer_version=_sqlbuild_version(), occurred_at=datetime.fromtimestamp(record.created, tz=UTC), - severity=_severity(record.levelno), - logger=logger_name, - source="python_logging", - message=message, - fields=fields, - **identities, + **cast(dict[str, Any], fields), ) except Exception: return @@ -134,18 +61,6 @@ def emit(self, record: logging.LogRecord) -> None: self._failure_callback(error) -def _severity(level: int) -> str: - if level >= logging.CRITICAL: - return "critical" - if level >= logging.ERROR: - return "error" - if level >= logging.WARNING: - return "warning" - if level >= logging.INFO: - return "info" - return "debug" - - @cache def _sqlbuild_version() -> str: try: diff --git a/src/sqlbuild/runtime/observability/_helpers/factory.py b/src/sqlbuild/runtime/observability/_helpers/factory.py index ea883f9fd1..0901ade0ca 100644 --- a/src/sqlbuild/runtime/observability/_helpers/factory.py +++ b/src/sqlbuild/runtime/observability/_helpers/factory.py @@ -7,7 +7,7 @@ from types import MappingProxyType from uuid import uuid4 -from sqlbuild._native import CURRENT_LIFECYCLE_EVENT_SCHEMA_VERSION +from sqlbuild._native import CURRENT_LIFECYCLE_EVENT_SCHEMA_VERSION, validate_lifecycle_producer from sqlbuild.runtime.observability._helpers.identity import ( current_execution_identity, current_invocation_external_context, @@ -33,10 +33,7 @@ def create_lifecycle_event( raise ObservabilityValidationError( "cannot create a lifecycle event without an active invocation identity" ) - if producer != _DEFAULT_PRODUCER and producer_version is None: - raise ObservabilityValidationError( - "producer_version is required when producer is not 'sqlbuild'" - ) + validate_lifecycle_producer(producer=producer, producer_version=producer_version) return LifecycleEvent( event_id=uuid4().hex if event_id is None else event_id, event_type=event_type, diff --git a/src/sqlbuild/runtime/observability/_helpers/observability.py b/src/sqlbuild/runtime/observability/_helpers/observability.py index a2406ea77f..81ce115539 100644 --- a/src/sqlbuild/runtime/observability/_helpers/observability.py +++ b/src/sqlbuild/runtime/observability/_helpers/observability.py @@ -8,12 +8,12 @@ from typing import Any, cast from sqlbuild._native import ( + CURRENT_DIAGNOSTIC_LOG_SCHEMA_VERSION, lifecycle_event_catalogued, lifecycle_event_terminal, validate_lifecycle_schema_version, ) from sqlbuild.runtime.observability.constants import ( - CURRENT_DIAGNOSTIC_LOG_SCHEMA_VERSION, DIAGNOSTIC_ENVELOPE_FIELDS, LIFECYCLE_ENVELOPE_FIELDS_BY_VERSION, LIFECYCLE_INVOCATION_METADATA_SCHEMA_VERSION, diff --git a/src/sqlbuild/runtime/observability/classes/operation_lifecycle.py b/src/sqlbuild/runtime/observability/classes/operation_lifecycle.py index e071b3223b..89431b62b8 100644 --- a/src/sqlbuild/runtime/observability/classes/operation_lifecycle.py +++ b/src/sqlbuild/runtime/observability/classes/operation_lifecycle.py @@ -11,16 +11,15 @@ from typing import Any from sqlbuild._native import ( - HOOK_PHASES, - HOOK_TYPES, OPERATION_ADAPTERS, OPERATION_KINDS, - OPERATION_METADATA_FIELDS, OPERATION_NAMES, OPERATION_PHASES, OPERATION_SCOPES, OPERATION_STRATEGIES, OPERATION_TARGET_KINDS, + validated_operation_hook_fields, + validated_operation_metadata, ) from sqlbuild.runtime.observability._helpers.dispatcher import ( current_event_dispatcher, @@ -39,7 +38,6 @@ from sqlbuild.runtime.observability.types import JSONValue _ERROR_TOKEN_PATTERN: re.Pattern[str] = re.compile(r"^[A-Za-z0-9._:-]{1,128}$") -_HOOK_NAME_PATTERN: re.Pattern[str] = re.compile(r"^[A-Za-z_][A-Za-z0-9_]{0,127}$") class OperationLifecycle: @@ -78,9 +76,9 @@ def __init__( ) self._operation_kind: str = operation_kind self._operation_name: str = operation_name - self._metadata: dict[str, int] = _validated_metadata(metadata) + self._metadata: dict[str, int] = validated_operation_metadata(metadata=metadata) self._operation_id: str | None = operation_id - self._hook_fields: dict[str, JSONValue] = _validated_hook_fields( + self._hook_fields: dict[str, JSONValue] = validated_operation_hook_fields( hook_phase=hook_phase, hook_index=hook_index, hook_type=hook_type, @@ -239,7 +237,7 @@ def _terminal_payload( ) -> dict[str, JSONValue]: payload: dict[str, JSONValue] = self._base_payload() payload.update(_validated_attribute_fields(attributes)) - terminal_metadata: dict[str, int] = _validated_metadata(metadata) + terminal_metadata: dict[str, int] = validated_operation_metadata(metadata=metadata) if terminal_metadata: payload["metadata"] = {**self._metadata, **terminal_metadata} started: float = self._started_monotonic if self._started_monotonic is not None else 0.0 @@ -309,50 +307,6 @@ def _validated_attribute_fields( return fields -def _validated_metadata(metadata: Mapping[str, int] | None) -> dict[str, int]: - if metadata is None: - return {} - unexpected: set[str] = set(metadata) - OPERATION_METADATA_FIELDS - if unexpected: - raise ObservabilityValidationError("operation metadata contains a non-allowlisted field") - result: dict[str, int] = {} - for key, value in metadata.items(): - if isinstance(value, bool) or not isinstance(value, int) or value < 0: - raise ObservabilityValidationError( - "operation metadata values must be nonnegative integers excluding bool" - ) - result[key] = value - return result - - -def _validated_hook_fields( - *, - hook_phase: str | None, - hook_index: int | None, - hook_type: str | None, - hook_name: str | None, -) -> dict[str, JSONValue]: - supplied: bool = any( - value is not None for value in (hook_phase, hook_index, hook_type, hook_name) - ) - if not supplied: - return {} - if hook_phase not in HOOK_PHASES: - raise ObservabilityValidationError("hook_phase must be a catalogued value") - if hook_type not in HOOK_TYPES: - raise ObservabilityValidationError("hook_type must be a catalogued value") - if isinstance(hook_index, bool) or not isinstance(hook_index, int) or hook_index < 0: - raise ObservabilityValidationError("hook_index must be a nonnegative integer") - result: dict[str, JSONValue] = { - "hook_phase": hook_phase, - "hook_index": hook_index, - "hook_type": hook_type, - } - if hook_name is not None and _HOOK_NAME_PATTERN.fullmatch(hook_name) is not None: - result["hook_name"] = hook_name - return result - - def _safe_error_token(*, value: object, fallback: str) -> str: if not isinstance(value, str) or _ERROR_TOKEN_PATTERN.fullmatch(value) is None: return fallback diff --git a/src/sqlbuild/runtime/observability/classes/run_lifecycle.py b/src/sqlbuild/runtime/observability/classes/run_lifecycle.py index 887519d012..ee9f74ad2b 100644 --- a/src/sqlbuild/runtime/observability/classes/run_lifecycle.py +++ b/src/sqlbuild/runtime/observability/classes/run_lifecycle.py @@ -5,10 +5,10 @@ import time from types import TracebackType +from sqlbuild._native import validate_run_bounds from sqlbuild.runtime.observability._helpers.dispatcher import current_event_dispatcher from sqlbuild.runtime.observability._helpers.factory import create_lifecycle_event from sqlbuild.runtime.observability.classes.event_dispatcher import EventDispatcher -from sqlbuild.runtime.observability.exceptions import ObservabilityValidationError from sqlbuild.runtime.observability.types import JSONValue @@ -23,7 +23,7 @@ def __init__( configured_concurrency: int, worker_count: int, ) -> None: - _validate_run_bounds( + validate_run_bounds( selected_count=selected_count, configured_concurrency=configured_concurrency, worker_count=worker_count, @@ -108,25 +108,3 @@ def _publish(self, *, event_type: str, payload: dict[str, JSONValue]) -> None: self._dispatcher.publish_lifecycle( create_lifecycle_event(event_type=event_type, payload=payload) ) - - -def _validate_run_bounds( - *, selected_count: int, configured_concurrency: int, worker_count: int -) -> None: - values: tuple[tuple[str, int], ...] = ( - ("selected_count", selected_count), - ("configured_concurrency", configured_concurrency), - ("worker_count", worker_count), - ) - for field_name, value in values: - if isinstance(value, bool) or not isinstance(value, int): - raise ObservabilityValidationError(f"{field_name} must be an integer excluding bool") - if selected_count < 0: - raise ObservabilityValidationError("selected_count must be nonnegative") - if configured_concurrency < 1: - raise ObservabilityValidationError("configured_concurrency must be positive") - expected_worker_count: int = min(configured_concurrency, selected_count) - if worker_count != expected_worker_count: - raise ObservabilityValidationError( - "worker_count must equal min(configured_concurrency, selected_count)" - ) diff --git a/src/sqlbuild/runtime/observability/constants.py b/src/sqlbuild/runtime/observability/constants.py index 61ddeeeb6b..47ecb4b533 100644 --- a/src/sqlbuild/runtime/observability/constants.py +++ b/src/sqlbuild/runtime/observability/constants.py @@ -4,10 +4,6 @@ from types import MappingProxyType LIFECYCLE_INVOCATION_METADATA_SCHEMA_VERSION: int = 2 -CURRENT_DIAGNOSTIC_LOG_SCHEMA_VERSION: int = 1 -DIAGNOSTIC_SEVERITIES: frozenset[str] = frozenset( - {"trace", "debug", "info", "warning", "error", "critical"} -) RESOURCE_ATTEMPT_SKIPPED_EVENT: str = "resource_attempt_skipped" RENAME_OPERATION_STRATEGY: str = "rename" RETRY_SCHEDULED_EVENT: str = "retry_scheduled" diff --git a/src/sqlbuild/runtime/observability/models.py b/src/sqlbuild/runtime/observability/models.py index 1272ee6d80..d855c62daa 100644 --- a/src/sqlbuild/runtime/observability/models.py +++ b/src/sqlbuild/runtime/observability/models.py @@ -9,14 +9,12 @@ from typing import Literal from sqlbuild._native import ( - CURRENT_LIFECYCLE_EVENT_SCHEMA_VERSION, - LIFECYCLE_EVENT_SCHEMA_VERSIONS, freeze_lifecycle_json, - validate_known_lifecycle_event, + validate_diagnostic_log, + validate_lifecycle_event, + validate_lifecycle_invocation_sequence, validate_lifecycle_optional_text, validate_lifecycle_required_text, - validate_lifecycle_schema_version, - validate_lifecycle_timestamp, ) from sqlbuild.runtime.observability.exceptions import ObservabilityValidationError from sqlbuild.runtime.observability.types import JSONValue @@ -112,63 +110,12 @@ class LifecycleEvent: payload: Mapping[str, JSONValue] = field(default_factory=dict) def __post_init__(self) -> None: - - for field_name in ( - "event_id", - "event_type", - "producer", - "producer_version", - "invocation_id", - ): - validate_lifecycle_required_text(value=getattr(self, field_name), field_name=field_name) - validate_lifecycle_schema_version(value=self.schema_version) - if self.schema_version not in LIFECYCLE_EVENT_SCHEMA_VERSIONS: - raise ObservabilityValidationError( - "LifecycleEvent only represents known schema versions through " - f"{CURRENT_LIFECYCLE_EVENT_SCHEMA_VERSION}; decode other versions as " - "OpaqueLifecycleEvent" - ) - if self.schema_version == 1: - if self.invocation_sequence is not None or self.external_context: - raise ObservabilityValidationError( - "schema version 1 lifecycle events cannot contain invocation metadata" - ) - elif ( - isinstance(self.invocation_sequence, bool) - or not isinstance(self.invocation_sequence, int) - or self.invocation_sequence < 0 - ): - raise ObservabilityValidationError( - "invocation_sequence must be a non-negative integer excluding bool" - ) - validate_lifecycle_timestamp(value=self.occurred_at) - for field_name in ( - "run_id", - "resource_id", - "resource_attempt_id", - "operation_id", - "statement_id", - ): - validate_lifecycle_optional_text(value=getattr(self, field_name), field_name=field_name) - frozen_context: JSONValue = freeze_lifecycle_json( - value=self.external_context, path="external_context" - ) - if not isinstance(frozen_context, Mapping): - raise ObservabilityValidationError("external_context must be a JSON object") - object.__setattr__(self, "external_context", frozen_context) - frozen_payload: JSONValue = freeze_lifecycle_json(value=self.payload, path="payload") - if not isinstance(frozen_payload, Mapping): - raise ObservabilityValidationError("payload must be a JSON object") - object.__setattr__(self, "payload", frozen_payload) - validate_known_lifecycle_event(event=self) + validate_lifecycle_event(event=self) def with_invocation_sequence(self, sequence: int) -> LifecycleEvent: """Return this validated event with a dispatcher-assigned invocation sequence.""" - if isinstance(sequence, bool) or not isinstance(sequence, int) or sequence < 0: - raise ObservabilityValidationError( - "invocation_sequence must be a non-negative integer excluding bool" - ) + validate_lifecycle_invocation_sequence(value=sequence) sequenced: LifecycleEvent = copy.copy(self) object.__setattr__(sequenced, "invocation_sequence", sequence) return sequenced @@ -209,44 +156,7 @@ class DiagnosticLog: statement_id: str | None = None def __post_init__(self) -> None: - from sqlbuild.runtime.observability.constants import ( - CURRENT_DIAGNOSTIC_LOG_SCHEMA_VERSION, - DIAGNOSTIC_SEVERITIES, - ) - - validate_lifecycle_schema_version(value=self.schema_version) - if self.schema_version != CURRENT_DIAGNOSTIC_LOG_SCHEMA_VERSION: - raise ObservabilityValidationError( - f"diagnostic schema_version must be {CURRENT_DIAGNOSTIC_LOG_SCHEMA_VERSION}" - ) - for field_name in ( - "producer", - "producer_version", - "severity", - "logger", - "source", - "message", - ): - validate_lifecycle_required_text(value=getattr(self, field_name), field_name=field_name) - if self.severity not in DIAGNOSTIC_SEVERITIES: - raise ObservabilityValidationError( - "severity must be one of trace, debug, info, warning, error, critical" - ) - validate_lifecycle_timestamp(value=self.occurred_at) - for field_name in ( - "log_stream_id", - "invocation_id", - "run_id", - "resource_id", - "resource_attempt_id", - "operation_id", - "statement_id", - ): - validate_lifecycle_optional_text(value=getattr(self, field_name), field_name=field_name) - frozen_fields: JSONValue = freeze_lifecycle_json(value=self.fields, path="fields") - if not isinstance(frozen_fields, Mapping): - raise ObservabilityValidationError("fields must be a JSON object") - object.__setattr__(self, "fields", frozen_fields) + validate_diagnostic_log(record=self) @dataclass(frozen=True) From 9aa51143b50db31f7ee3dba08f33ad870cf86ea8 Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Fri, 16 Oct 2026 00:21:36 +0100 Subject: [PATCH 37/48] refactor: map adapter connections to dlt destinations natively The adapter-config to dlt destination mapping moves to sqlbuild-core and shares connection setting reads with the native ingestr destinations through a new loader_connection domain. build_dlt_destination keeps importing dlt and calling the destination factory. --- .../dlt_destination/_helpers/credentials.rs | 173 ++++++++++++++ .../src/dlt_destination/_helpers/mod.rs | 2 + .../src/dlt_destination/_helpers/request.rs | 48 ++++ .../src/dlt_destination/constants.rs | 36 +++ .../src/dlt_destination/errors.rs | 7 + .../main/dlt_destination_plan.rs | 40 ++++ .../src/dlt_destination/main/mod.rs | 1 + .../sqlbuild-core/src/dlt_destination/mod.rs | 10 + .../src/dlt_destination/models.rs | 49 ++++ .../src/dlt_destination/tests.rs | 4 + .../tests/test_dlt_destination_plan.rs | 108 +++++++++ .../src/dlt_destination/tests/test_types.rs | 11 + .../ingestr_command/_helpers/connection.rs | 29 +-- .../ingestr_command/_helpers/destinations.rs | 27 +-- .../src/ingestr_command/constants.rs | 6 +- .../main/ingestr_destination_uri.rs | 4 +- .../src/ingestr_command/models.rs | 11 - .../tests/test_ingestr_command.rs | 5 +- crates/sqlbuild-core/src/lib.rs | 2 + .../src/loader_connection/constants.rs | 8 + .../main/connection_setting.rs | 14 ++ .../src/loader_connection/main/mod.rs | 2 + .../main/required_connection_setting.rs | 18 ++ .../src/loader_connection/mod.rs | 5 + .../src/loader_connection/models.rs | 12 + .../bindings/_helpers/load/dlt_destination.rs | 123 ++++++++++ .../bindings/_helpers/load/ingestr_command.rs | 9 +- .../src/bindings/_helpers/load/mod.rs | 1 + .../bindings/_helpers/load/registration.rs | 3 +- python-boundary.toml | 2 +- src/sqlbuild/_native.pyi | 7 + .../integrations/dlt/_helpers/destination.py | 211 +----------------- 32 files changed, 730 insertions(+), 258 deletions(-) create mode 100644 crates/sqlbuild-core/src/dlt_destination/_helpers/credentials.rs create mode 100644 crates/sqlbuild-core/src/dlt_destination/_helpers/mod.rs create mode 100644 crates/sqlbuild-core/src/dlt_destination/_helpers/request.rs create mode 100644 crates/sqlbuild-core/src/dlt_destination/constants.rs create mode 100644 crates/sqlbuild-core/src/dlt_destination/errors.rs create mode 100644 crates/sqlbuild-core/src/dlt_destination/main/dlt_destination_plan.rs create mode 100644 crates/sqlbuild-core/src/dlt_destination/main/mod.rs create mode 100644 crates/sqlbuild-core/src/dlt_destination/mod.rs create mode 100644 crates/sqlbuild-core/src/dlt_destination/models.rs create mode 100644 crates/sqlbuild-core/src/dlt_destination/tests.rs create mode 100644 crates/sqlbuild-core/src/dlt_destination/tests/test_dlt_destination_plan.rs create mode 100644 crates/sqlbuild-core/src/dlt_destination/tests/test_types.rs create mode 100644 crates/sqlbuild-core/src/loader_connection/constants.rs create mode 100644 crates/sqlbuild-core/src/loader_connection/main/connection_setting.rs create mode 100644 crates/sqlbuild-core/src/loader_connection/main/mod.rs create mode 100644 crates/sqlbuild-core/src/loader_connection/main/required_connection_setting.rs create mode 100644 crates/sqlbuild-core/src/loader_connection/mod.rs create mode 100644 crates/sqlbuild-core/src/loader_connection/models.rs create mode 100644 crates/sqlbuild-python/src/bindings/_helpers/load/dlt_destination.rs diff --git a/crates/sqlbuild-core/src/dlt_destination/_helpers/credentials.rs b/crates/sqlbuild-core/src/dlt_destination/_helpers/credentials.rs new file mode 100644 index 0000000000..08bdb0c93a --- /dev/null +++ b/crates/sqlbuild-core/src/dlt_destination/_helpers/credentials.rs @@ -0,0 +1,173 @@ +//! Destination credentials per warehouse adapter. + +use crate::dlt_destination::constants::{ + BIGQUERY_ADAPTER, BIGQUERY_ARGUMENTS, CREDENTIALS_ARGUMENT, DATABRICKS_ADAPTER, DLT_LOADER, + MOTHERDUCK_ADAPTER, POSTGRES_ADAPTER, SNOWFLAKE_ACCOUNT_SUFFIX, SNOWFLAKE_ADAPTER, + SNOWFLAKE_OPTIONAL_SETTINGS, SQLSERVER_ADAPTER, SQLSERVER_DEFAULT_PORT, SQLSERVER_FACTORY, +}; +use crate::dlt_destination::errors::DltDestinationError; +use crate::dlt_destination::models::DltValue; +use crate::loader_connection::constants::{DEFAULT_HOST, MOTHERDUCK_PREFIX, POSTGRES_DEFAULT_PORT}; +use crate::loader_connection::main::connection_setting::connection_setting; +use crate::loader_connection::main::required_connection_setting::required_connection_setting; +use crate::loader_connection::models::LoaderConnection; + +/// Factory arguments, in call order. +type Arguments = Vec<(&'static str, DltValue)>; + +/// A required stripped connection setting of `adapter`. +pub(crate) fn required<'a>( + connection: &'a LoaderConnection, + key: &str, + adapter: &str, +) -> Result<&'a str, DltDestinationError> { + required_connection_setting(connection, key, adapter, DLT_LOADER) + .map_err(|message| DltDestinationError { message }) +} + +fn text(value: &str) -> DltValue { + DltValue::Text(value.to_owned()) +} + +/// The single `credentials` argument. +pub(crate) fn credentials(value: DltValue) -> Arguments { + vec![(CREDENTIALS_ARGUMENT, value)] +} + +fn snowflake_account(account: &str) -> &str { + account + .strip_suffix(SNOWFLAKE_ACCOUNT_SUFFIX) + .unwrap_or(account) +} + +fn motherduck(connection: &LoaderConnection) -> DltValue { + let database: &str = connection_setting(connection, "database").unwrap_or_default(); + let mut credentials: String = if database.starts_with(MOTHERDUCK_PREFIX) { + database.to_owned() + } else { + format!("{MOTHERDUCK_PREFIX}{database}") + }; + if let Some(token) = connection_setting(connection, "token") { + credentials = format!("{credentials}?motherduck_token={token}"); + } + DltValue::Text(credentials) +} + +fn postgres(connection: &LoaderConnection) -> Result { + let host: &str = connection_setting(connection, "host").unwrap_or(DEFAULT_HOST); + let port: &str = connection.port.as_deref().unwrap_or(POSTGRES_DEFAULT_PORT); + let user: &str = required(connection, "user", POSTGRES_ADAPTER)?; + let password: &str = connection_setting(connection, "password").unwrap_or_default(); + let database: &str = match connection_setting(connection, "dbname") { + Some(database) => database, + None => required(connection, "database", POSTGRES_ADAPTER)?, + }; + Ok(DltValue::Text(format!( + "postgresql://{user}:{password}@{host}:{port}/{database}" + ))) +} + +fn snowflake(connection: &LoaderConnection) -> Result { + let account: &str = required(connection, "account", SNOWFLAKE_ADAPTER)?; + let host: &str = connection_setting(connection, "host").unwrap_or(account); + let mut fields: Arguments = vec![ + ("account", text(snowflake_account(account))), + ("host", text(snowflake_account(host))), + ( + "user", + text(required(connection, "user", SNOWFLAKE_ADAPTER)?), + ), + ( + "database", + text(required(connection, "database", SNOWFLAKE_ADAPTER)?), + ), + ( + "warehouse", + text(required(connection, "warehouse", SNOWFLAKE_ADAPTER)?), + ), + ]; + for key in SNOWFLAKE_OPTIONAL_SETTINGS { + if let Some(value) = connection_setting(connection, key) { + fields.push((key, text(value))); + } + } + Ok(DltValue::Object(fields)) +} + +fn bigquery(connection: &LoaderConnection) -> Arguments { + let mut arguments: Arguments = Vec::new(); + for (argument, key) in BIGQUERY_ARGUMENTS { + if let Some(value) = connection_setting(connection, key) { + arguments.push((argument, text(value))); + } + } + arguments +} + +fn databricks(connection: &LoaderConnection) -> Result { + let mut fields: Arguments = vec![ + ( + "server_hostname", + text(required(connection, "server_hostname", DATABRICKS_ADAPTER)?), + ), + ( + "http_path", + text(required(connection, "http_path", DATABRICKS_ADAPTER)?), + ), + ( + "access_token", + text(required(connection, "token", DATABRICKS_ADAPTER)?), + ), + ]; + if let Some(catalog) = connection_setting(connection, "catalog") { + fields.push(("catalog", text(catalog))); + } + Ok(DltValue::Object(fields)) +} + +fn sqlserver(connection: &LoaderConnection) -> Result { + Ok(DltValue::Object(vec![ + ( + "host", + text(connection_setting(connection, "host").unwrap_or(DEFAULT_HOST)), + ), + ( + "port", + DltValue::Port { + default: SQLSERVER_DEFAULT_PORT, + }, + ), + ( + "database", + text(required(connection, "database", SQLSERVER_ADAPTER)?), + ), + ( + "username", + text(required(connection, "user", SQLSERVER_ADAPTER)?), + ), + ( + "password", + text(connection_setting(connection, "password").unwrap_or_default()), + ), + ( + "query", + DltValue::Object(vec![("TrustServerCertificate", text("yes"))]), + ), + ])) +} + +/// The factory and arguments of a non-DuckDB adapter, or `None` when dlt cannot load into it. +pub(crate) fn adapter_call( + adapter: &str, + connection: &LoaderConnection, +) -> Result, DltDestinationError> { + Ok(Some(match adapter { + MOTHERDUCK_ADAPTER => (MOTHERDUCK_ADAPTER, credentials(motherduck(connection))), + POSTGRES_ADAPTER => (POSTGRES_ADAPTER, credentials(postgres(connection)?)), + SNOWFLAKE_ADAPTER => (SNOWFLAKE_ADAPTER, credentials(snowflake(connection)?)), + BIGQUERY_ADAPTER => (BIGQUERY_ADAPTER, bigquery(connection)), + DATABRICKS_ADAPTER => (DATABRICKS_ADAPTER, credentials(databricks(connection)?)), + SQLSERVER_ADAPTER => (SQLSERVER_FACTORY, credentials(sqlserver(connection)?)), + _ => return Ok(None), + })) +} diff --git a/crates/sqlbuild-core/src/dlt_destination/_helpers/mod.rs b/crates/sqlbuild-core/src/dlt_destination/_helpers/mod.rs new file mode 100644 index 0000000000..53f1c45a32 --- /dev/null +++ b/crates/sqlbuild-core/src/dlt_destination/_helpers/mod.rs @@ -0,0 +1,2 @@ +pub(crate) mod credentials; +pub(crate) mod request; diff --git a/crates/sqlbuild-core/src/dlt_destination/_helpers/request.rs b/crates/sqlbuild-core/src/dlt_destination/_helpers/request.rs new file mode 100644 index 0000000000..91f9b45ff2 --- /dev/null +++ b/crates/sqlbuild-core/src/dlt_destination/_helpers/request.rs @@ -0,0 +1,48 @@ +//! Destination config and dataset rules checked before any adapter credentials. + +use crate::dlt_destination::constants::SQLBUILD_OWNED_DESTINATION_KEYS; +use crate::dlt_destination::errors::DltDestinationError; +use crate::dlt_destination::models::DltDatasetName; +use crate::text::main::python_strip::python_strip; + +/// Reject a destination config that sets keys SQLBuild owns. +pub(crate) fn check_destination_keys( + adapter: &str, + destination_keys: &[Option], +) -> Result<(), DltDestinationError> { + let mut blocked: Vec<&str> = destination_keys + .iter() + .flatten() + .map(String::as_str) + .filter(|key| SQLBUILD_OWNED_DESTINATION_KEYS.contains(key)) + .collect(); + if blocked.is_empty() { + return Ok(()); + } + blocked.sort_unstable(); + Err(DltDestinationError { + message: format!( + "Adapter '{adapter}' dlt destination config cannot define SQLBuild-owned key(s): {}", + blocked.join(", ") + ), + }) +} + +/// Reject a missing or blank dataset name, which raw landing requires. +pub(crate) fn check_dataset_name( + adapter: &str, + dataset_name: &DltDatasetName, +) -> Result<(), DltDestinationError> { + let named: bool = dataset_name + .text + .as_deref() + .is_some_and(|name| !python_strip(name).is_empty()); + if named { + return Ok(()); + } + Err(DltDestinationError { + message: format!( + "Adapter '{adapter}' requires an explicit dlt source schema for raw landing" + ), + }) +} diff --git a/crates/sqlbuild-core/src/dlt_destination/constants.rs b/crates/sqlbuild-core/src/dlt_destination/constants.rs new file mode 100644 index 0000000000..920e01ac55 --- /dev/null +++ b/crates/sqlbuild-core/src/dlt_destination/constants.rs @@ -0,0 +1,36 @@ +//! dlt destination defaults, adapters and the keys SQLBuild owns. + +/// The integration loader name dlt connection errors cite. +pub(crate) const DLT_LOADER: &str = "dlt"; +/// Destination options SQLBuild passes unless the destination config overrides them. +pub const DEFAULT_DESTINATION_OPTIONS: [(&str, &str); 1] = [("naming_convention", "sql_ci_v1")]; +/// Destination config keys SQLBuild sets itself. +pub(crate) const SQLBUILD_OWNED_DESTINATION_KEYS: [&str; 4] = [ + "credentials", + "dataset_name", + "default_schema_name", + "project_id", +]; +/// The dataset DuckDB destinations land in without an explicit one. +pub(crate) const DUCKDB_DEFAULT_DATASET: &str = "main"; +/// The suffix Snowflake account and host settings drop. +pub(crate) const SNOWFLAKE_ACCOUNT_SUFFIX: &str = ".snowflakecomputing.com"; +/// SQL Server's default port number. +pub(crate) const SQLSERVER_DEFAULT_PORT: i64 = 1433; +/// Snowflake settings copied into credentials when set, in order. +pub(crate) const SNOWFLAKE_OPTIONAL_SETTINGS: [&str; 4] = + ["password", "role", "authenticator", "token"]; +/// BigQuery factory arguments and the connection settings they come from, in order. +pub(crate) const BIGQUERY_ARGUMENTS: [(&str, &str); 2] = + [("project_id", "project"), ("location", "location")]; +pub(crate) const DUCKDB_ADAPTER: &str = "duckdb"; +pub(crate) const MOTHERDUCK_ADAPTER: &str = "motherduck"; +pub(crate) const POSTGRES_ADAPTER: &str = "postgres"; +pub(crate) const SNOWFLAKE_ADAPTER: &str = "snowflake"; +pub(crate) const BIGQUERY_ADAPTER: &str = "bigquery"; +pub(crate) const DATABRICKS_ADAPTER: &str = "databricks"; +pub(crate) const SQLSERVER_ADAPTER: &str = "sqlserver"; +/// The dlt destination factory of SQL Server. +pub(crate) const SQLSERVER_FACTORY: &str = "mssql"; +/// The factory argument holding destination credentials. +pub(crate) const CREDENTIALS_ARGUMENT: &str = "credentials"; diff --git a/crates/sqlbuild-core/src/dlt_destination/errors.rs b/crates/sqlbuild-core/src/dlt_destination/errors.rs new file mode 100644 index 0000000000..d0ca821392 --- /dev/null +++ b/crates/sqlbuild-core/src/dlt_destination/errors.rs @@ -0,0 +1,7 @@ +//! dlt destination construction errors. + +/// A destination the dlt integration cannot address, with its user-facing message. +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct DltDestinationError { + pub message: String, +} diff --git a/crates/sqlbuild-core/src/dlt_destination/main/dlt_destination_plan.rs b/crates/sqlbuild-core/src/dlt_destination/main/dlt_destination_plan.rs new file mode 100644 index 0000000000..3e06044eb3 --- /dev/null +++ b/crates/sqlbuild-core/src/dlt_destination/main/dlt_destination_plan.rs @@ -0,0 +1,40 @@ +//! `dlt_destination_plan`: the dlt destination call for one adapter's connection. + +use crate::dlt_destination::_helpers::credentials::{adapter_call, credentials, required}; +use crate::dlt_destination::_helpers::request::{check_dataset_name, check_destination_keys}; +use crate::dlt_destination::constants::{DUCKDB_ADAPTER, DUCKDB_DEFAULT_DATASET}; +use crate::dlt_destination::errors::DltDestinationError; +use crate::dlt_destination::models::{ + DltDataset, DltDestinationPlan, DltDestinationRequest, DltValue, +}; + +/// The dlt destination call for the request's adapter, connection and dataset. +pub fn dlt_destination_plan( + request: &DltDestinationRequest<'_>, +) -> Result { + let adapter: &str = request.adapter; + check_destination_keys(adapter, request.destination_keys)?; + if adapter == DUCKDB_ADAPTER { + let database: &str = required(request.connection, "database", DUCKDB_ADAPTER)?; + return Ok(DltDestinationPlan { + factory: DUCKDB_ADAPTER, + arguments: credentials(DltValue::Text(database.to_owned())), + dataset: if request.dataset_name.truthy { + DltDataset::Given + } else { + DltDataset::Default(DUCKDB_DEFAULT_DATASET) + }, + }); + } + check_dataset_name(adapter, request.dataset_name)?; + let Some((factory, arguments)) = adapter_call(adapter, request.connection)? else { + return Err(DltDestinationError { + message: format!("Adapter '{adapter}' does not support dlt integration loaders"), + }); + }; + Ok(DltDestinationPlan { + factory, + arguments, + dataset: DltDataset::Given, + }) +} diff --git a/crates/sqlbuild-core/src/dlt_destination/main/mod.rs b/crates/sqlbuild-core/src/dlt_destination/main/mod.rs new file mode 100644 index 0000000000..b8507d2941 --- /dev/null +++ b/crates/sqlbuild-core/src/dlt_destination/main/mod.rs @@ -0,0 +1 @@ +pub mod dlt_destination_plan; diff --git a/crates/sqlbuild-core/src/dlt_destination/mod.rs b/crates/sqlbuild-core/src/dlt_destination/mod.rs new file mode 100644 index 0000000000..4be5b95029 --- /dev/null +++ b/crates/sqlbuild-core/src/dlt_destination/mod.rs @@ -0,0 +1,10 @@ +//! The dlt destination factory and arguments for one SQLBuild warehouse adapter. + +pub(crate) mod _helpers; +pub mod constants; +pub mod errors; +pub mod main; +pub mod models; + +#[cfg(test)] +mod tests; diff --git a/crates/sqlbuild-core/src/dlt_destination/models.rs b/crates/sqlbuild-core/src/dlt_destination/models.rs new file mode 100644 index 0000000000..3e1d881496 --- /dev/null +++ b/crates/sqlbuild-core/src/dlt_destination/models.rs @@ -0,0 +1,49 @@ +//! A dlt destination factory call and the facts it is built from. + +use crate::loader_connection::models::LoaderConnection; + +/// One argument value of a dlt destination factory call. +#[derive(Clone, Debug, PartialEq, Eq)] +pub enum DltValue { + Text(String), + /// The connection's raw `port` setting, or `default` when unset. + Port { + default: i64, + }, + Object(Vec<(&'static str, DltValue)>), +} + +/// The dataset a dlt destination lands in. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum DltDataset { + /// The dataset name the caller gave. + Given, + Default(&'static str), +} + +/// The caller's dataset name. +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct DltDatasetName { + /// The name when it is a string. + pub text: Option, + /// Python truthiness of the name. + pub truthy: bool, +} + +/// One `dlt.destinations.(**arguments, **options)` call and its dataset. +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct DltDestinationPlan { + pub factory: &'static str, + pub arguments: Vec<(&'static str, DltValue)>, + pub dataset: DltDataset, +} + +/// One adapter's connection, destination config keys and dataset name. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub struct DltDestinationRequest<'a> { + pub adapter: &'a str, + pub connection: &'a LoaderConnection, + /// The destination config's keys, each `None` when not a string. + pub destination_keys: &'a [Option], + pub dataset_name: &'a DltDatasetName, +} diff --git a/crates/sqlbuild-core/src/dlt_destination/tests.rs b/crates/sqlbuild-core/src/dlt_destination/tests.rs new file mode 100644 index 0000000000..48fa2dd946 --- /dev/null +++ b/crates/sqlbuild-core/src/dlt_destination/tests.rs @@ -0,0 +1,4 @@ +#[path = "tests/test_dlt_destination_plan.rs"] +mod dlt_destination_plan; +#[path = "tests/test_types.rs"] +mod test_types; diff --git a/crates/sqlbuild-core/src/dlt_destination/tests/test_dlt_destination_plan.rs b/crates/sqlbuild-core/src/dlt_destination/tests/test_dlt_destination_plan.rs new file mode 100644 index 0000000000..418fd98879 --- /dev/null +++ b/crates/sqlbuild-core/src/dlt_destination/tests/test_dlt_destination_plan.rs @@ -0,0 +1,108 @@ +//! dlt destination calls match the Python integration's. + +use crate::dlt_destination::main::dlt_destination_plan::dlt_destination_plan; +use crate::dlt_destination::models::{ + DltDataset, DltDatasetName, DltDestinationPlan, DltDestinationRequest, DltValue, +}; +use crate::dlt_destination::tests::test_types::DestinationPlanTestCase; +use crate::loader_connection::models::LoaderConnection; + +#[test] +fn given_adapter_connection_when_planning_dlt_destination_then_matches_python_mapping() { + let test_cases = [ + DestinationPlanTestCase { + description: "duckdb without dataset lands in main", + adapter: "duckdb", + strings: &[("database", " warehouse.duckdb ")], + port: None, + destination_keys: &[], + dataset_name: None, + expected_plan: Ok(DltDestinationPlan { + factory: "duckdb", + arguments: vec![("credentials", DltValue::Text("warehouse.duckdb".to_owned()))], + dataset: DltDataset::Default("main"), + }), + }, + DestinationPlanTestCase { + description: "postgres connection string uses host default and port", + adapter: "postgres", + strings: &[("user", "u"), ("database", "d")], + port: Some("6543"), + destination_keys: &["staging"], + dataset_name: Some("raw"), + expected_plan: Ok(DltDestinationPlan { + factory: "postgres", + arguments: vec![( + "credentials", + DltValue::Text("postgresql://u:@localhost:6543/d".to_owned()), + )], + dataset: DltDataset::Given, + }), + }, + DestinationPlanTestCase { + description: "owned destination keys are sorted in the error", + adapter: "postgres", + strings: &[], + port: None, + destination_keys: &["dataset_name", "credentials"], + dataset_name: Some("raw"), + expected_plan: Err( + "Adapter 'postgres' dlt destination config cannot define SQLBuild-owned key(s): credentials, dataset_name", + ), + }, + DestinationPlanTestCase { + description: "blank dataset is rejected before credentials", + adapter: "snowflake", + strings: &[], + port: None, + destination_keys: &[], + dataset_name: Some(" "), + expected_plan: Err( + "Adapter 'snowflake' requires an explicit dlt source schema for raw landing", + ), + }, + DestinationPlanTestCase { + description: "missing required setting names the dlt loader", + adapter: "snowflake", + strings: &[], + port: None, + destination_keys: &[], + dataset_name: Some("raw"), + expected_plan: Err( + "Adapter 'snowflake' requires connection.account for dlt integration loaders", + ), + }, + ]; + + for test_case in test_cases { + let connection = LoaderConnection { + strings: test_case + .strings + .iter() + .map(|(key, value)| ((*key).to_owned(), (*value).to_owned())) + .collect(), + port: test_case.port.map(str::to_owned), + }; + let destination_keys: Vec> = test_case + .destination_keys + .iter() + .map(|key| Some((*key).to_owned())) + .collect(); + let dataset_name = DltDatasetName { + text: test_case.dataset_name.map(str::to_owned), + truthy: test_case.dataset_name.is_some_and(|name| !name.is_empty()), + }; + let request = DltDestinationRequest { + adapter: test_case.adapter, + connection: &connection, + destination_keys: &destination_keys, + dataset_name: &dataset_name, + }; + assert_eq!( + dlt_destination_plan(&request).map_err(|error| error.message), + test_case.expected_plan.map_err(str::to_owned), + "{}", + test_case.description + ); + } +} diff --git a/crates/sqlbuild-core/src/dlt_destination/tests/test_types.rs b/crates/sqlbuild-core/src/dlt_destination/tests/test_types.rs new file mode 100644 index 0000000000..ea416660fd --- /dev/null +++ b/crates/sqlbuild-core/src/dlt_destination/tests/test_types.rs @@ -0,0 +1,11 @@ +use crate::dlt_destination::models::DltDestinationPlan; + +pub(super) struct DestinationPlanTestCase { + pub(super) description: &'static str, + pub(super) adapter: &'static str, + pub(super) strings: &'static [(&'static str, &'static str)], + pub(super) port: Option<&'static str>, + pub(super) destination_keys: &'static [&'static str], + pub(super) dataset_name: Option<&'static str>, + pub(super) expected_plan: Result, +} diff --git a/crates/sqlbuild-core/src/ingestr_command/_helpers/connection.rs b/crates/sqlbuild-core/src/ingestr_command/_helpers/connection.rs index bd4661ff65..6a1951b26e 100644 --- a/crates/sqlbuild-core/src/ingestr_command/_helpers/connection.rs +++ b/crates/sqlbuild-core/src/ingestr_command/_helpers/connection.rs @@ -1,39 +1,34 @@ //! Connection settings read the way the ingestr destination expects them. +use crate::ingestr_command::constants::INGESTR_LOADER; use crate::ingestr_command::errors::IngestrCommandError; -use crate::ingestr_command::models::IngestrConnection; -use crate::text::main::python_strip::python_strip; +use crate::loader_connection::main::connection_setting::connection_setting; +use crate::loader_connection::main::required_connection_setting::required_connection_setting; +use crate::loader_connection::models::LoaderConnection; /// The stripped string setting `key`, or `None` when unset, not a string or blank. -pub(crate) fn optional<'a>(connection: &'a IngestrConnection, key: &str) -> Option<&'a str> { - connection - .strings - .get(key) - .map(|value| python_strip(value)) - .filter(|value| !value.is_empty()) +pub(crate) fn optional<'a>(connection: &'a LoaderConnection, key: &str) -> Option<&'a str> { + connection_setting(connection, key) } -/// The stripped string setting `key`, which the adapter's destination requires. +/// The stripped string setting `key`, which the adapter's ingestr destination requires. pub(crate) fn required<'a>( - connection: &'a IngestrConnection, + connection: &'a LoaderConnection, key: &str, adapter_name: &str, ) -> Result<&'a str, IngestrCommandError> { - optional(connection, key).ok_or_else(|| IngestrCommandError { - message: format!( - "Adapter '{adapter_name}' requires connection.{key} for ingestr integration loaders" - ), - }) + required_connection_setting(connection, key, adapter_name, INGESTR_LOADER) + .map_err(|message| IngestrCommandError { message }) } /// `(key, value)` when the setting `key` is set. -fn setting<'a>(connection: &'a IngestrConnection, key: &'a str) -> Option<(&'a str, &'a str)> { +fn setting<'a>(connection: &'a LoaderConnection, key: &'a str) -> Option<(&'a str, &'a str)> { optional(connection, key).map(|value| (key, value)) } /// The settings among `keys` that are set, in `keys` order. pub(crate) fn present<'a>( - connection: &'a IngestrConnection, + connection: &'a LoaderConnection, keys: &[&'a str], ) -> Vec<(&'a str, &'a str)> { keys.iter() diff --git a/crates/sqlbuild-core/src/ingestr_command/_helpers/destinations.rs b/crates/sqlbuild-core/src/ingestr_command/_helpers/destinations.rs index a8ffad57f9..86b6532aa8 100644 --- a/crates/sqlbuild-core/src/ingestr_command/_helpers/destinations.rs +++ b/crates/sqlbuild-core/src/ingestr_command/_helpers/destinations.rs @@ -3,15 +3,16 @@ use crate::ingestr_command::_helpers::connection::{optional, present, required}; use crate::ingestr_command::_helpers::url_quote::{query_suffix, quote, urlencode}; use crate::ingestr_command::constants::{ - BIGQUERY_PARAMETERS, DATABRICKS_PARAMETERS, MOTHERDUCK_PREFIX, POSTGRES_DEFAULT_PORT, - SNOWFLAKE_PARAMETERS, SQLSERVER_DEFAULT_DRIVER, SQLSERVER_DEFAULT_PORT, - SQLSERVER_DRIVER_PARAMETER, SQLSERVER_PARAMETERS, SQLSERVER_TRUST_SERVER_CERTIFICATE_PARAMETER, + BIGQUERY_PARAMETERS, DATABRICKS_PARAMETERS, SNOWFLAKE_PARAMETERS, SQLSERVER_DEFAULT_DRIVER, + SQLSERVER_DEFAULT_PORT, SQLSERVER_DRIVER_PARAMETER, SQLSERVER_PARAMETERS, + SQLSERVER_TRUST_SERVER_CERTIFICATE_PARAMETER, }; use crate::ingestr_command::errors::IngestrCommandError; -use crate::ingestr_command::models::IngestrConnection; +use crate::loader_connection::constants::{DEFAULT_HOST, MOTHERDUCK_PREFIX, POSTGRES_DEFAULT_PORT}; +use crate::loader_connection::models::LoaderConnection; /// DuckDB's database file URI. -pub(crate) fn duckdb(connection: &IngestrConnection) -> Result { +pub(crate) fn duckdb(connection: &LoaderConnection) -> Result { Ok(format!( "duckdb:///{}", required(connection, "database", "duckdb")? @@ -19,7 +20,7 @@ pub(crate) fn duckdb(connection: &IngestrConnection) -> Result String { +pub(crate) fn motherduck(connection: &LoaderConnection) -> String { let database: &str = optional(connection, "database").unwrap_or_default(); let database: &str = database.strip_prefix(MOTHERDUCK_PREFIX).unwrap_or(database); let token: Vec<(&str, &str)> = optional(connection, "token") @@ -30,8 +31,8 @@ pub(crate) fn motherduck(connection: &IngestrConnection) -> String { } /// Postgres's URI with its optional SSL mode. -pub(crate) fn postgres(connection: &IngestrConnection) -> Result { - let host: &str = optional(connection, "host").unwrap_or("localhost"); +pub(crate) fn postgres(connection: &LoaderConnection) -> Result { + let host: &str = optional(connection, "host").unwrap_or(DEFAULT_HOST); let port: &str = connection.port.as_deref().unwrap_or(POSTGRES_DEFAULT_PORT); let user: &str = required(connection, "user", "postgres")?; let password: &str = optional(connection, "password").unwrap_or_default(); @@ -50,7 +51,7 @@ pub(crate) fn postgres(connection: &IngestrConnection) -> Result Result { +pub(crate) fn bigquery(connection: &LoaderConnection) -> Result { let project: &str = match optional(connection, "project") { Some(project) => project, None => required(connection, "database", "bigquery")?, @@ -62,7 +63,7 @@ pub(crate) fn bigquery(connection: &IngestrConnection) -> Result Result { +pub(crate) fn snowflake(connection: &LoaderConnection) -> Result { let account: &str = required(connection, "account", "snowflake")?; let user: &str = required(connection, "user", "snowflake")?; let password: &str = optional(connection, "password").unwrap_or_default(); @@ -79,7 +80,7 @@ pub(crate) fn snowflake(connection: &IngestrConnection) -> Result Result { +pub(crate) fn databricks(connection: &LoaderConnection) -> Result { let host: &str = required(connection, "server_hostname", "databricks")?; let token: &str = required(connection, "token", "databricks")?; let mut parameters: Vec<(&str, &str)> = vec![( @@ -95,14 +96,14 @@ pub(crate) fn databricks(connection: &IngestrConnection) -> Result String { +pub(crate) fn sqlserver(connection: &LoaderConnection) -> String { let first = |keys: [&str; 2], default: &'static str| -> String { keys.iter() .find_map(|key| optional(connection, key)) .unwrap_or(default) .to_owned() }; - let host: String = first(["host", "server"], "localhost"); + let host: String = first(["host", "server"], DEFAULT_HOST); let port: &str = connection.port.as_deref().unwrap_or(SQLSERVER_DEFAULT_PORT); let user: String = first(["user", "username"], "sa"); let password: &str = optional(connection, "password").unwrap_or_default(); diff --git a/crates/sqlbuild-core/src/ingestr_command/constants.rs b/crates/sqlbuild-core/src/ingestr_command/constants.rs index 3de722e87b..12475e6949 100644 --- a/crates/sqlbuild-core/src/ingestr_command/constants.rs +++ b/crates/sqlbuild-core/src/ingestr_command/constants.rs @@ -6,8 +6,6 @@ pub(crate) const COMMAND_PREFIX: [&str; 2] = ["ingestr", "ingest"]; pub(crate) const COMMAND_PROGRESS: [&str; 3] = ["--yes", "--progress", "log"]; /// Characters Python's `urllib.parse.quote` never escapes. pub(crate) const URL_ALWAYS_SAFE: &[u8] = b"_.-~"; -/// Postgres's default port. -pub(crate) const POSTGRES_DEFAULT_PORT: &str = "5432"; /// SQL Server's default port. pub(crate) const SQLSERVER_DEFAULT_PORT: &str = "1433"; /// The SQL Server ODBC driver ingestr uses unless the connection names one. @@ -33,5 +31,5 @@ pub(crate) const SQLSERVER_PARAMETERS: [&str; 3] = pub(crate) const SQLSERVER_DRIVER_PARAMETER: &str = "driver"; /// The SQL Server certificate trust query parameter. pub(crate) const SQLSERVER_TRUST_SERVER_CERTIFICATE_PARAMETER: &str = "TrustServerCertificate"; -/// The MotherDuck database prefix the URI omits. -pub(crate) const MOTHERDUCK_PREFIX: &str = "md:"; +/// The integration loader name ingestr connection errors cite. +pub(crate) const INGESTR_LOADER: &str = "ingestr"; diff --git a/crates/sqlbuild-core/src/ingestr_command/main/ingestr_destination_uri.rs b/crates/sqlbuild-core/src/ingestr_command/main/ingestr_destination_uri.rs index 4320263bbb..198bf35e36 100644 --- a/crates/sqlbuild-core/src/ingestr_command/main/ingestr_destination_uri.rs +++ b/crates/sqlbuild-core/src/ingestr_command/main/ingestr_destination_uri.rs @@ -4,12 +4,12 @@ use crate::ingestr_command::_helpers::destinations::{ bigquery, databricks, duckdb, motherduck, postgres, snowflake, sqlserver, }; use crate::ingestr_command::errors::IngestrCommandError; -use crate::ingestr_command::models::IngestrConnection; +use crate::loader_connection::models::LoaderConnection; /// The ingestr destination URI for the connection of the adapter named `adapter_name`. pub fn ingestr_destination_uri( adapter_name: &str, - connection: &IngestrConnection, + connection: &LoaderConnection, ) -> Result { match adapter_name { "duckdb" => duckdb(connection), diff --git a/crates/sqlbuild-core/src/ingestr_command/models.rs b/crates/sqlbuild-core/src/ingestr_command/models.rs index 2ae053876d..bbfc6e6af1 100644 --- a/crates/sqlbuild-core/src/ingestr_command/models.rs +++ b/crates/sqlbuild-core/src/ingestr_command/models.rs @@ -1,16 +1,5 @@ //! Inputs of one ingestr command line. -use std::collections::BTreeMap; - -/// The adapter connection settings an ingestr destination URI reads. -#[derive(Clone, Debug, Default, PartialEq, Eq)] -pub struct IngestrConnection { - /// Connection settings whose values are Python strings, unstripped. - pub strings: BTreeMap, - /// `str(connection["port"])` when the connection sets a port. - pub port: Option, -} - /// A resolved ingestr source configuration. #[derive(Clone, Debug, Default, PartialEq, Eq)] pub struct IngestrSource { diff --git a/crates/sqlbuild-core/src/ingestr_command/tests/test_ingestr_command.rs b/crates/sqlbuild-core/src/ingestr_command/tests/test_ingestr_command.rs index 98287653ae..e35dd77651 100644 --- a/crates/sqlbuild-core/src/ingestr_command/tests/test_ingestr_command.rs +++ b/crates/sqlbuild-core/src/ingestr_command/tests/test_ingestr_command.rs @@ -4,8 +4,9 @@ use crate::ingestr_command::main::format_ingestr_command::format_ingestr_command use crate::ingestr_command::main::ingestr_command_arguments::ingestr_command_arguments; use crate::ingestr_command::main::ingestr_destination_table::ingestr_destination_table; use crate::ingestr_command::main::ingestr_destination_uri::ingestr_destination_uri; -use crate::ingestr_command::models::{IngestrConnection, IngestrDestination, IngestrSource}; +use crate::ingestr_command::models::{IngestrDestination, IngestrSource}; use crate::ingestr_command::tests::test_types::{CommandTestCase, DestinationUriTestCase}; +use crate::loader_connection::models::LoaderConnection; #[test] fn given_connection_when_building_destination_uri_then_matches_python_quoting() { @@ -61,7 +62,7 @@ fn given_connection_when_building_destination_uri_then_matches_python_quoting() }, ]; for test_case in test_cases { - let connection: IngestrConnection = IngestrConnection { + let connection: LoaderConnection = LoaderConnection { strings: test_case .strings .iter() diff --git a/crates/sqlbuild-core/src/lib.rs b/crates/sqlbuild-core/src/lib.rs index bfb6bfed61..1fcd518fa3 100644 --- a/crates/sqlbuild-core/src/lib.rs +++ b/crates/sqlbuild-core/src/lib.rs @@ -26,6 +26,7 @@ pub mod dbt_process; pub mod debug_report; pub mod diagnostic_routing; pub mod diagnostic_text; +pub mod dlt_destination; pub mod event_export; pub mod format_command; pub mod freshness_text; @@ -36,6 +37,7 @@ pub mod janitor_text; pub mod json; pub mod lifecycle_events; pub mod load_command; +pub mod loader_connection; pub mod migration_values; pub mod models; pub mod nested_progress; diff --git a/crates/sqlbuild-core/src/loader_connection/constants.rs b/crates/sqlbuild-core/src/loader_connection/constants.rs new file mode 100644 index 0000000000..115683dca6 --- /dev/null +++ b/crates/sqlbuild-core/src/loader_connection/constants.rs @@ -0,0 +1,8 @@ +//! Loader destination connection defaults. + +/// The host destinations connect to without one. +pub const DEFAULT_HOST: &str = "localhost"; +/// Postgres's default port. +pub const POSTGRES_DEFAULT_PORT: &str = "5432"; +/// The MotherDuck database prefix. +pub const MOTHERDUCK_PREFIX: &str = "md:"; diff --git a/crates/sqlbuild-core/src/loader_connection/main/connection_setting.rs b/crates/sqlbuild-core/src/loader_connection/main/connection_setting.rs new file mode 100644 index 0000000000..141c17c681 --- /dev/null +++ b/crates/sqlbuild-core/src/loader_connection/main/connection_setting.rs @@ -0,0 +1,14 @@ +//! `connection_setting`: one stripped string connection setting. + +use crate::loader_connection::models::LoaderConnection; +use crate::text::main::python_strip::python_strip; + +/// The stripped string setting `key`, or `None` when unset, not a string or blank. +#[must_use] +pub fn connection_setting<'a>(connection: &'a LoaderConnection, key: &str) -> Option<&'a str> { + connection + .strings + .get(key) + .map(|value| python_strip(value)) + .filter(|value| !value.is_empty()) +} diff --git a/crates/sqlbuild-core/src/loader_connection/main/mod.rs b/crates/sqlbuild-core/src/loader_connection/main/mod.rs new file mode 100644 index 0000000000..4ea9c16cda --- /dev/null +++ b/crates/sqlbuild-core/src/loader_connection/main/mod.rs @@ -0,0 +1,2 @@ +pub mod connection_setting; +pub mod required_connection_setting; diff --git a/crates/sqlbuild-core/src/loader_connection/main/required_connection_setting.rs b/crates/sqlbuild-core/src/loader_connection/main/required_connection_setting.rs new file mode 100644 index 0000000000..cb6ac0aecb --- /dev/null +++ b/crates/sqlbuild-core/src/loader_connection/main/required_connection_setting.rs @@ -0,0 +1,18 @@ +//! `required_connection_setting`: one connection setting a loader destination requires. + +use crate::loader_connection::main::connection_setting::connection_setting; +use crate::loader_connection::models::LoaderConnection; + +/// The stripped string setting `key`, which the adapter's `loader` destination requires. +pub fn required_connection_setting<'a>( + connection: &'a LoaderConnection, + key: &str, + adapter_name: &str, + loader: &str, +) -> Result<&'a str, String> { + connection_setting(connection, key).ok_or_else(|| { + format!( + "Adapter '{adapter_name}' requires connection.{key} for {loader} integration loaders" + ) + }) +} diff --git a/crates/sqlbuild-core/src/loader_connection/mod.rs b/crates/sqlbuild-core/src/loader_connection/mod.rs new file mode 100644 index 0000000000..28a7bc49f9 --- /dev/null +++ b/crates/sqlbuild-core/src/loader_connection/mod.rs @@ -0,0 +1,5 @@ +//! Warehouse connection settings that integration loader destinations read. + +pub mod constants; +pub mod main; +pub mod models; diff --git a/crates/sqlbuild-core/src/loader_connection/models.rs b/crates/sqlbuild-core/src/loader_connection/models.rs new file mode 100644 index 0000000000..ed4f4d15bd --- /dev/null +++ b/crates/sqlbuild-core/src/loader_connection/models.rs @@ -0,0 +1,12 @@ +//! The adapter connection settings a loader destination reads. + +use std::collections::BTreeMap; + +/// The adapter connection settings an ingestr or dlt destination reads. +#[derive(Clone, Debug, Default, PartialEq, Eq)] +pub struct LoaderConnection { + /// Connection settings whose values are Python strings, unstripped. + pub strings: BTreeMap, + /// `str(connection["port"])` when the connection sets a port. + pub port: Option, +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/load/dlt_destination.rs b/crates/sqlbuild-python/src/bindings/_helpers/load/dlt_destination.rs new file mode 100644 index 0000000000..47c179dd7c --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/load/dlt_destination.rs @@ -0,0 +1,123 @@ +//! The dlt destination factory, arguments and dataset for one SQLBuild warehouse adapter. + +use pyo3::prelude::{ + Bound, PyAny, PyAnyMethods, PyErr, PyModule, PyModuleMethods, PyResult, Python, +}; +use pyo3::types::{PyDict, PyDictMethods, PyString}; +use pyo3::{pyfunction, wrap_pyfunction}; +use sqlbuild_core::dlt_destination::constants::DEFAULT_DESTINATION_OPTIONS; +use sqlbuild_core::dlt_destination::main::dlt_destination_plan::dlt_destination_plan; +use sqlbuild_core::dlt_destination::models::{ + DltDataset, DltDatasetName, DltDestinationPlan, DltDestinationRequest, DltValue, +}; +use sqlbuild_core::loader_connection::models::LoaderConnection; + +use crate::bindings::_helpers::compile_driver::python_calls::imported; +use crate::bindings::_helpers::load::ingestr_command::connection; + +const DLT_ERROR: (&str, &str) = ( + "sqlbuild.integrations.dlt.exceptions", + "DltIntegrationError", +); + +/// `(factory, arguments, options, dataset_name)` of one dlt destination. +type DestinationCall<'py> = ( + &'static str, + Bound<'py, PyDict>, + Bound<'py, PyDict>, + Bound<'py, PyAny>, +); + +/// A `DltIntegrationError`, or the failure to construct one. +fn dlt_error(py: Python<'_>, message: String) -> PyErr { + imported(py, DLT_ERROR.0, DLT_ERROR.1) + .and_then(|class| class.call1((message,))) + .map_or_else(|error| error, PyErr::from_value) +} + +/// The text of a string value, or `None` for any other value. +fn string(value: &Bound<'_, PyAny>) -> PyResult> { + if value.is_instance_of::() { + return Ok(Some(value.extract()?)); + } + Ok(None) +} + +/// One factory argument as a Python value. +fn argument<'py>( + value: &DltValue, + connection_config: &Bound<'py, PyAny>, +) -> PyResult> { + let py: Python<'py> = connection_config.py(); + match value { + DltValue::Text(text) => Ok(PyString::new(py, text).into_any()), + DltValue::Port { default } => connection_config.call_method1("get", ("port", *default)), + DltValue::Object(fields) => Ok(arguments(fields, connection_config)?.into_any()), + } +} + +fn arguments<'py>( + fields: &[(&str, DltValue)], + connection_config: &Bound<'py, PyAny>, +) -> PyResult> { + let values: Bound<'py, PyDict> = PyDict::new(connection_config.py()); + for (key, value) in fields { + values.set_item(*key, argument(value, connection_config)?)?; + } + Ok(values) +} + +/// `(factory, arguments, options, dataset_name)`; callers unpack `arguments` and `options` apart. +#[pyfunction] +#[pyo3(signature = (*, adapter_name, connection_config, destination_config, dataset_name))] +fn dlt_destination_arguments<'py>( + adapter_name: &str, + connection_config: &Bound<'py, PyAny>, + destination_config: &Bound<'py, PyAny>, + dataset_name: &Bound<'py, PyAny>, +) -> PyResult> { + let py: Python<'py> = connection_config.py(); + let dict: Bound<'py, PyAny> = imported(py, "builtins", "dict")?; + let destination: Bound<'py, PyDict> = if destination_config.is_truthy()? { + dict.call1((destination_config,))?.downcast_into()? + } else { + PyDict::new(py) + }; + let mut destination_keys: Vec> = Vec::new(); + for key in destination.keys() { + destination_keys.push(string(&key)?); + } + let settings: Bound<'py, PyDict> = dict.call1((connection_config,))?.downcast_into()?; + let name = DltDatasetName { + text: string(dataset_name)?, + truthy: dataset_name.is_truthy()?, + }; + let connection: LoaderConnection = connection(&settings)?; + let request = DltDestinationRequest { + adapter: adapter_name, + connection: &connection, + destination_keys: &destination_keys, + dataset_name: &name, + }; + let plan: DltDestinationPlan = + dlt_destination_plan(&request).map_err(|error| dlt_error(py, error.message))?; + let options: Bound<'py, PyDict> = PyDict::new(py); + for (key, value) in DEFAULT_DESTINATION_OPTIONS { + options.set_item(key, value)?; + } + options.update(destination.as_mapping())?; + let dataset: Bound<'py, PyAny> = match plan.dataset { + DltDataset::Given => dataset_name.clone(), + DltDataset::Default(name) => PyString::new(py, name).into_any(), + }; + Ok(( + plan.factory, + arguments(&plan.arguments, connection_config)?, + options, + dataset, + )) +} + +pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { + module.add_function(wrap_pyfunction!(dlt_destination_arguments, module)?) +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/load/ingestr_command.rs b/crates/sqlbuild-python/src/bindings/_helpers/load/ingestr_command.rs index 3e64a70e39..11a990d48a 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/load/ingestr_command.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/load/ingestr_command.rs @@ -11,9 +11,8 @@ use sqlbuild_core::ingestr_command::main::format_ingestr_command::format_ingestr use sqlbuild_core::ingestr_command::main::ingestr_command_arguments::ingestr_command_arguments; use sqlbuild_core::ingestr_command::main::ingestr_destination_table::ingestr_destination_table; use sqlbuild_core::ingestr_command::main::ingestr_destination_uri::ingestr_destination_uri; -use sqlbuild_core::ingestr_command::models::{ - IngestrConnection, IngestrDestination, IngestrSource, -}; +use sqlbuild_core::ingestr_command::models::{IngestrDestination, IngestrSource}; +use sqlbuild_core::loader_connection::models::LoaderConnection; use sqlbuild_model_config::templates::models::TemplateOptions; use crate::bindings::_helpers::boundary::keyword_request::Request; @@ -157,7 +156,7 @@ fn resolved_source( } /// The string settings of a connection and its rendered port. -fn connection(connection_config: &Bound<'_, PyDict>) -> PyResult { +pub(crate) fn connection(connection_config: &Bound<'_, PyDict>) -> PyResult { let mut strings: BTreeMap = BTreeMap::new(); for (key, value) in connection_config.iter() { if let (Ok(key), Ok(value)) = (key.extract::(), value.downcast::()) { @@ -168,7 +167,7 @@ fn connection(connection_config: &Bound<'_, PyDict>) -> PyResult) -> PyResult<()> { + dlt_destination::register(module)?; ingestr_command::register(module)?; load_command::register(module)?; load_progress::register(module)?; diff --git a/python-boundary.toml b/python-boundary.toml index 7ce5069489..c566c7b3ce 100644 --- a/python-boundary.toml +++ b/python-boundary.toml @@ -380,7 +380,7 @@ note = "Documented orchestrator integration and rule 4 Rivers glue: translator i [[stays]] path = "integrations/dlt" rule = 4 -note = "Loader runtime that runs inside user loaders; synthetic loader record building and naming are native." +note = "Loader runtime that runs inside user loaders and calls the dlt destination factories; synthetic loader record building, naming and adapter-config to destination mapping are native (dlt_destination_arguments)." [[stays]] path = "integrations/ingestr" diff --git a/src/sqlbuild/_native.pyi b/src/sqlbuild/_native.pyi index 9485ccd8c9..5c1193c4db 100644 --- a/src/sqlbuild/_native.pyi +++ b/src/sqlbuild/_native.pyi @@ -4003,6 +4003,13 @@ def build_ingestr_command( def format_ingestr_command(command: Sequence[str], /) -> str: ... def format_clone_execution_json( *, result: CloneExecutionResult, resource_types_by_name: Mapping[str, str] +def dlt_destination_arguments( + *, + adapter_name: str, + connection_config: Mapping[str, object], + destination_config: Mapping[str, object] | None, + dataset_name: str | None, +) -> tuple[str, dict[str, Any], dict[str, Any], str | None]: ... ) -> str: ... def format_test_execution_json(*, results: tuple[SqlTestExecutionResult, ...]) -> str: ... def format_audit_execution_json( diff --git a/src/sqlbuild/integrations/dlt/_helpers/destination.py b/src/sqlbuild/integrations/dlt/_helpers/destination.py index 9c875dee7b..3553745ec4 100644 --- a/src/sqlbuild/integrations/dlt/_helpers/destination.py +++ b/src/sqlbuild/integrations/dlt/_helpers/destination.py @@ -2,17 +2,10 @@ from __future__ import annotations -from typing import Any, cast - -from sqlbuild.adapter.contract.types import BuiltinAdapter +from sqlbuild._native import dlt_destination_arguments from sqlbuild.integrations.dlt.exceptions import DltIntegrationError from sqlbuild.integrations.dlt.models import DltDestinationConfig -_DEFAULT_DESTINATION_OPTIONS: dict[str, object] = {"naming_convention": "sql_ci_v1"} -_SQLBUILD_OWNED_DESTINATION_KEYS: frozenset[str] = frozenset( - {"credentials", "dataset_name", "default_schema_name", "project_id"} -) - def build_dlt_destination( *, @@ -31,199 +24,13 @@ def build_dlt_destination( "pip install 'sqlbuild[dlt]'" ) from error - destination_options: dict[str, object] = _destination_options( - adapter_name=adapter_name, destination_config=destination_config or {} - ) - if adapter_name == BuiltinAdapter.DUCKDB.value: - return DltDestinationConfig( - destination=cast(Any, destinations.duckdb)( - credentials=_required_string( - config=connection_config, - key="database", - adapter_name=BuiltinAdapter.DUCKDB.value, - ), - **destination_options, - ), - dataset_name=dataset_name or "main", - ) - _require_dataset_name(adapter_name=adapter_name, dataset_name=dataset_name) - if adapter_name == BuiltinAdapter.MOTHERDUCK.value: - database: str = _optional_string(config=connection_config, key="database") or "" - token: str | None = _optional_string(config=connection_config, key="token") - credentials: str = database if database.startswith("md:") else f"md:{database}" - if token is not None: - credentials = f"{credentials}?motherduck_token={token}" - return DltDestinationConfig( - destination=cast(Any, destinations.motherduck)( - credentials=credentials, **destination_options - ), - dataset_name=dataset_name, - ) - if adapter_name == BuiltinAdapter.POSTGRES.value: - return DltDestinationConfig( - destination=cast(Any, destinations.postgres)( - credentials=_postgres_connection_string(connection_config), **destination_options - ), - dataset_name=dataset_name, - ) - if adapter_name == BuiltinAdapter.SNOWFLAKE.value: - return DltDestinationConfig( - destination=cast(Any, destinations.snowflake)( - credentials=_snowflake_credentials(connection_config), **destination_options - ), - dataset_name=dataset_name, - ) - if adapter_name == BuiltinAdapter.BIGQUERY.value: - return DltDestinationConfig( - destination=cast(Any, destinations.bigquery)( - **_bigquery_config(connection_config), **destination_options - ), - dataset_name=dataset_name, - ) - if adapter_name == BuiltinAdapter.DATABRICKS.value: - return DltDestinationConfig( - destination=cast(Any, destinations.databricks)( - credentials=_databricks_credentials(connection_config), **destination_options - ), - dataset_name=dataset_name, - ) - if adapter_name == BuiltinAdapter.SQLSERVER.value: - return DltDestinationConfig( - destination=cast(Any, destinations.mssql)( - credentials=_sqlserver_credentials(connection_config), **destination_options - ), - dataset_name=dataset_name, - ) - raise DltIntegrationError(f"Adapter '{adapter_name}' does not support dlt integration loaders") - - -def _destination_options( - *, adapter_name: str, destination_config: dict[str, object] -) -> dict[str, object]: - blocked_keys: tuple[str, ...] = tuple( - key for key in destination_config if key in _SQLBUILD_OWNED_DESTINATION_KEYS + factory, arguments, options, resolved_dataset_name = dlt_destination_arguments( + adapter_name=adapter_name, + connection_config=connection_config, + destination_config=destination_config, + dataset_name=dataset_name, ) - if blocked_keys: - blocked: str = ", ".join(sorted(blocked_keys)) - raise DltIntegrationError( - f"Adapter '{adapter_name}' dlt destination config cannot define SQLBuild-owned key(s): " - f"{blocked}" - ) - return {**_DEFAULT_DESTINATION_OPTIONS, **destination_config} - - -def _postgres_connection_string(config: dict[str, object]) -> str: - host: str = _optional_string(config=config, key="host") or "localhost" - port: object = config.get("port", 5432) - user: str = _required_string( - config=config, key="user", adapter_name=BuiltinAdapter.POSTGRES.value + return DltDestinationConfig( + destination=getattr(destinations, factory)(**arguments, **options), + dataset_name=resolved_dataset_name, ) - password: str = _optional_string(config=config, key="password") or "" - database: str = _optional_string(config=config, key="dbname") or _required_string( - config=config, key="database", adapter_name=BuiltinAdapter.POSTGRES.value - ) - return f"postgresql://{user}:{password}@{host}:{port}/{database}" - - -def _snowflake_credentials(config: dict[str, object]) -> dict[str, object]: - account: str = _required_string( - config=config, key="account", adapter_name=BuiltinAdapter.SNOWFLAKE.value - ) - credentials: dict[str, object] = { - "account": _snowflake_account(account), - "host": _snowflake_account(_optional_string(config=config, key="host") or account), - "user": _required_string( - config=config, key="user", adapter_name=BuiltinAdapter.SNOWFLAKE.value - ), - "database": _required_string( - config=config, key="database", adapter_name=BuiltinAdapter.SNOWFLAKE.value - ), - "warehouse": _required_string( - config=config, key="warehouse", adapter_name=BuiltinAdapter.SNOWFLAKE.value - ), - } - credentials = _copy_optional(source=config, destination=credentials, key="password") - credentials = _copy_optional(source=config, destination=credentials, key="role") - credentials = _copy_optional(source=config, destination=credentials, key="authenticator") - credentials = _copy_optional(source=config, destination=credentials, key="token") - return credentials - - -def _bigquery_config(config: dict[str, object]) -> dict[str, object]: - options: dict[str, object] = {} - project_id: str | None = _optional_string(config=config, key="project") - location: str | None = _optional_string(config=config, key="location") - if project_id is not None: - options["project_id"] = project_id - if location is not None: - options["location"] = location - return options - - -def _snowflake_account(account: str) -> str: - return account.removesuffix(".snowflakecomputing.com") - - -def _databricks_credentials(config: dict[str, object]) -> dict[str, object]: - credentials: dict[str, object] = { - "server_hostname": _required_string( - config=config, key="server_hostname", adapter_name=BuiltinAdapter.DATABRICKS.value - ), - "http_path": _required_string( - config=config, key="http_path", adapter_name=BuiltinAdapter.DATABRICKS.value - ), - "access_token": _required_string( - config=config, key="token", adapter_name=BuiltinAdapter.DATABRICKS.value - ), - } - catalog: str | None = _optional_string(config=config, key="catalog") - if catalog is not None: - credentials["catalog"] = catalog - return credentials - - -def _sqlserver_credentials(config: dict[str, object]) -> dict[str, object]: - return { - "host": _optional_string(config=config, key="host") or "localhost", - "port": config.get("port", 1433), - "database": _required_string( - config=config, key="database", adapter_name=BuiltinAdapter.SQLSERVER.value - ), - "username": _required_string( - config=config, key="user", adapter_name=BuiltinAdapter.SQLSERVER.value - ), - "password": _optional_string(config=config, key="password") or "", - "query": {"TrustServerCertificate": "yes"}, - } - - -def _require_dataset_name(*, adapter_name: str, dataset_name: str | None) -> None: - if not isinstance(dataset_name, str) or not dataset_name.strip(): - raise DltIntegrationError( - f"Adapter '{adapter_name}' requires an explicit dlt source schema for raw landing" - ) - - -def _copy_optional( - *, source: dict[str, object], destination: dict[str, object], key: str -) -> dict[str, object]: - value: str | None = _optional_string(config=source, key=key) - if value is not None: - destination[key] = value - return destination - - -def _required_string(*, config: dict[str, object], key: str, adapter_name: str) -> str: - value: object | None = config.get(key) - if not isinstance(value, str) or not value.strip(): - raise DltIntegrationError( - f"Adapter '{adapter_name}' requires connection.{key} for dlt integration loaders" - ) - return value.strip() - - -def _optional_string(*, config: dict[str, object], key: str) -> str | None: - value: object | None = config.get(key) - if not isinstance(value, str) or not value.strip(): - return None - return value.strip() From 4fd70f75842dcacb59fc9c46e497042294db8044 Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Fri, 16 Oct 2026 00:21:36 +0100 Subject: [PATCH 38/48] refactor: coerce dbt profile as_bool and as_number values natively --- .../src/dbt_profile/constants.rs | 6 ++ .../src/dbt_profile/main/dbt_bool_text.rs | 17 ++++ .../src/dbt_profile/main/dbt_number_text.rs | 18 +++++ .../src/dbt_profile/main/mod.rs | 2 + .../sqlbuild-config/src/dbt_profile/models.rs | 7 ++ .../sqlbuild-config/src/dbt_profile/tests.rs | 2 + .../dbt_profile/tests/test_dbt_coercion.rs | 79 +++++++++++++++++++ .../src/dbt_profile/tests/test_types.rs | 16 +++- .../src/bindings/_helpers/dbt_interop/mod.rs | 1 + .../_helpers/dbt_interop/profile_coercion.rs | 71 +++++++++++++++++ .../_helpers/dbt_interop/registration.rs | 1 + python-boundary.toml | 2 +- src/sqlbuild/_native.pyi | 44 ++++++----- .../dbt/_helpers/profile/render.py | 42 ++-------- 14 files changed, 250 insertions(+), 58 deletions(-) create mode 100644 crates/sqlbuild-config/src/dbt_profile/main/dbt_bool_text.rs create mode 100644 crates/sqlbuild-config/src/dbt_profile/main/dbt_number_text.rs create mode 100644 crates/sqlbuild-config/src/dbt_profile/tests/test_dbt_coercion.rs create mode 100644 crates/sqlbuild-python/src/bindings/_helpers/dbt_interop/profile_coercion.rs diff --git a/crates/sqlbuild-config/src/dbt_profile/constants.rs b/crates/sqlbuild-config/src/dbt_profile/constants.rs index cbfd030d3a..e705f661d8 100644 --- a/crates/sqlbuild-config/src/dbt_profile/constants.rs +++ b/crates/sqlbuild-config/src/dbt_profile/constants.rs @@ -51,3 +51,9 @@ pub(crate) const BIGQUERY_METHOD_ERROR_SUFFIX: &str = "is not supported by SQLBu /// The dbt target path used when `dbt_project.yml` sets none. pub(crate) const DEFAULT_DBT_TARGET_PATH: &str = "target"; +/// Text dbt's `as_bool` reads as true, after stripping and lowercasing. +pub(crate) const DBT_TRUE_VALUES: [&str; 5] = ["true", "1", "yes", "y", "on"]; +/// Text dbt's `as_bool` reads as false, after stripping and lowercasing. +pub(crate) const DBT_FALSE_VALUES: [&str; 5] = ["false", "0", "no", "n", "off"]; +/// The separator that makes dbt's `as_number` read text as a float. +pub(crate) const DBT_NUMBER_DECIMAL_SEPARATOR: char = '.'; diff --git a/crates/sqlbuild-config/src/dbt_profile/main/dbt_bool_text.rs b/crates/sqlbuild-config/src/dbt_profile/main/dbt_bool_text.rs new file mode 100644 index 0000000000..4f2d6b8b35 --- /dev/null +++ b/crates/sqlbuild-config/src/dbt_profile/main/dbt_bool_text.rs @@ -0,0 +1,17 @@ +//! `dbt_bool_text`: the boolean dbt's `as_bool` reads from text. + +use sqlbuild_core::text::main::python_strip::python_strip; + +use crate::dbt_profile::constants::{DBT_FALSE_VALUES, DBT_TRUE_VALUES}; + +/// The boolean dbt's `as_bool` reads from stripped, lowercased text, or `None`. +#[must_use] +pub fn dbt_bool_text(text: &str) -> Option { + let normalized: String = python_strip(text).to_lowercase(); + if DBT_TRUE_VALUES.contains(&normalized.as_str()) { + return Some(true); + } + DBT_FALSE_VALUES + .contains(&normalized.as_str()) + .then_some(false) +} diff --git a/crates/sqlbuild-config/src/dbt_profile/main/dbt_number_text.rs b/crates/sqlbuild-config/src/dbt_profile/main/dbt_number_text.rs new file mode 100644 index 0000000000..c01d1d66c6 --- /dev/null +++ b/crates/sqlbuild-config/src/dbt_profile/main/dbt_number_text.rs @@ -0,0 +1,18 @@ +//! `dbt_number_text`: how dbt's `as_number` parses text. + +use sqlbuild_core::text::main::python_strip::python_strip; + +use crate::dbt_profile::constants::DBT_NUMBER_DECIMAL_SEPARATOR; +use crate::dbt_profile::models::DbtNumberParse; + +/// The stripped text dbt's `as_number` parses and whether it parses it as a float. +#[must_use] +pub fn dbt_number_text(text: &str) -> (&str, DbtNumberParse) { + let stripped: &str = python_strip(text); + let parse: DbtNumberParse = if stripped.contains(DBT_NUMBER_DECIMAL_SEPARATOR) { + DbtNumberParse::Float + } else { + DbtNumberParse::Integer + }; + (stripped, parse) +} diff --git a/crates/sqlbuild-config/src/dbt_profile/main/mod.rs b/crates/sqlbuild-config/src/dbt_profile/main/mod.rs index 640790dd39..480aa81680 100644 --- a/crates/sqlbuild-config/src/dbt_profile/main/mod.rs +++ b/crates/sqlbuild-config/src/dbt_profile/main/mod.rs @@ -1,4 +1,6 @@ +pub mod dbt_bool_text; pub mod dbt_init_project_toml; +pub mod dbt_number_text; pub mod dbt_profile_outputs; pub mod dbt_project_metadata; pub mod dbt_yaml_mapping; diff --git a/crates/sqlbuild-config/src/dbt_profile/models.rs b/crates/sqlbuild-config/src/dbt_profile/models.rs index a4e5264201..21c1b86b0e 100644 --- a/crates/sqlbuild-config/src/dbt_profile/models.rs +++ b/crates/sqlbuild-config/src/dbt_profile/models.rs @@ -100,3 +100,10 @@ pub struct DbtProfileOutputs { /// Stripped output names in first-seen order; a repeated name keeps its last entry. pub outputs: Vec<(String, usize)>, } + +/// How dbt's `as_number` parses one stripped text. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum DbtNumberParse { + Integer, + Float, +} diff --git a/crates/sqlbuild-config/src/dbt_profile/tests.rs b/crates/sqlbuild-config/src/dbt_profile/tests.rs index 003e0bb7d0..9ae346d0e2 100644 --- a/crates/sqlbuild-config/src/dbt_profile/tests.rs +++ b/crates/sqlbuild-config/src/dbt_profile/tests.rs @@ -1,3 +1,5 @@ +#[path = "tests/test_dbt_coercion.rs"] +mod dbt_coercion; #[path = "tests/test_dbt_profile.rs"] mod dbt_profile; #[path = "tests/helpers.rs"] diff --git a/crates/sqlbuild-config/src/dbt_profile/tests/test_dbt_coercion.rs b/crates/sqlbuild-config/src/dbt_profile/tests/test_dbt_coercion.rs new file mode 100644 index 0000000000..c915b76167 --- /dev/null +++ b/crates/sqlbuild-config/src/dbt_profile/tests/test_dbt_coercion.rs @@ -0,0 +1,79 @@ +use crate::dbt_profile::main::dbt_bool_text::dbt_bool_text; +use crate::dbt_profile::main::dbt_number_text::dbt_number_text; +use crate::dbt_profile::models::DbtNumberParse; +use crate::dbt_profile::tests::test_types::{DbtBoolTestCase, DbtNumberTestCase}; + +#[test] +fn given_profile_text_when_coercing_to_bool_then_dbt_spellings_are_read() { + let test_cases = [ + DbtBoolTestCase { + description: "padded mixed-case yes", + text: " Yes ", + expected_bool: Some(true), + }, + DbtBoolTestCase { + description: "uppercase on", + text: "ON", + expected_bool: Some(true), + }, + DbtBoolTestCase { + description: "whitespace-padded no", + text: "\tno\n", + expected_bool: Some(false), + }, + DbtBoolTestCase { + description: "zero", + text: "0", + expected_bool: Some(false), + }, + DbtBoolTestCase { + description: "other digit", + text: "2", + expected_bool: None, + }, + DbtBoolTestCase { + description: "empty", + text: "", + expected_bool: None, + }, + ]; + + for test_case in test_cases { + assert_eq!( + dbt_bool_text(test_case.text), + test_case.expected_bool, + "{}", + test_case.description + ); + } +} + +#[test] +fn given_profile_text_when_coercing_to_number_then_decimal_separator_selects_float() { + let test_cases = [ + DbtNumberTestCase { + description: "padded negative integer", + text: " -3 ", + expected_parse: ("-3", DbtNumberParse::Integer), + }, + DbtNumberTestCase { + description: "decimal", + text: "1.5", + expected_parse: ("1.5", DbtNumberParse::Float), + }, + DbtNumberTestCase { + description: "exponent without separator parses as integer", + text: "1e3", + expected_parse: ("1e3", DbtNumberParse::Integer), + }, + ]; + + for test_case in test_cases { + assert_eq!( + dbt_number_text(test_case.text), + test_case.expected_parse, + "{}", + test_case.description + ); + } +} diff --git a/crates/sqlbuild-config/src/dbt_profile/tests/test_types.rs b/crates/sqlbuild-config/src/dbt_profile/tests/test_types.rs index 752c57b53d..e4e7b2742a 100644 --- a/crates/sqlbuild-config/src/dbt_profile/tests/test_types.rs +++ b/crates/sqlbuild-config/src/dbt_profile/tests/test_types.rs @@ -1,7 +1,7 @@ use crate::dbt_profile::errors::DbtProfileError; use crate::dbt_profile::models::{ - DbtInitConfig, DbtProfileOutputs, DbtProjectMetadata, DbtYamlValue, LoadedDbtProfile, - NormalizedProfile, ProfileEntry, + DbtInitConfig, DbtNumberParse, DbtProfileOutputs, DbtProjectMetadata, DbtYamlValue, + LoadedDbtProfile, NormalizedProfile, ProfileEntry, }; pub(super) struct NormalizeTestCase { @@ -34,3 +34,15 @@ pub(super) struct SelectTargetTestCase { pub(super) requested: Option<&'static str>, pub(super) expected_result: Result<(String, usize), DbtProfileError>, } + +pub(super) struct DbtBoolTestCase { + pub(super) description: &'static str, + pub(super) text: &'static str, + pub(super) expected_bool: Option, +} + +pub(super) struct DbtNumberTestCase { + pub(super) description: &'static str, + pub(super) text: &'static str, + pub(super) expected_parse: (&'static str, DbtNumberParse), +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/dbt_interop/mod.rs b/crates/sqlbuild-python/src/bindings/_helpers/dbt_interop/mod.rs index 79b76ab4ca..a6dc7f2cbd 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/dbt_interop/mod.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/dbt_interop/mod.rs @@ -11,6 +11,7 @@ pub(crate) mod pipeline_prologue; pub(crate) mod plan; pub(crate) mod plan_output; pub(crate) mod profile; +pub(crate) mod profile_coercion; pub(crate) mod profile_connection; pub(crate) mod profile_load; pub(crate) mod reference_resolver; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/dbt_interop/profile_coercion.rs b/crates/sqlbuild-python/src/bindings/_helpers/dbt_interop/profile_coercion.rs new file mode 100644 index 0000000000..1c679f6324 --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/dbt_interop/profile_coercion.rs @@ -0,0 +1,71 @@ +//! dbt profile `as_bool` and `as_number` Jinja filters. + +use pyo3::exceptions::PyValueError; +use pyo3::prelude::{ + Bound, PyAny, PyAnyMethods, PyErr, PyModule, PyModuleMethods, PyResult, Python, +}; +use pyo3::types::{PyBool, PyFloat, PyInt, PyString, PyStringMethods}; +use pyo3::{pyfunction, wrap_pyfunction}; +use sqlbuild_config::dbt_profile::main::dbt_bool_text::dbt_bool_text; +use sqlbuild_config::dbt_profile::main::dbt_number_text::dbt_number_text; +use sqlbuild_config::dbt_profile::models::DbtNumberParse; + +use crate::bindings::_helpers::dbt_interop::profile_connection::profile_error; + +/// A `DbtProfileError` for a value that does not coerce to `target`. +fn coercion_error(value: &Bound<'_, PyAny>, target: &str) -> PyResult { + let rendered: String = value.repr()?.to_str()?.to_owned(); + Ok(profile_error( + value.py(), + format!("Cannot coerce {rendered} to {target}"), + )) +} + +/// A bool, or a string dbt reads as one. +#[pyfunction] +#[pyo3(signature = (value))] +fn coerce_dbt_profile_bool<'py>(value: &Bound<'py, PyAny>) -> PyResult> { + if value.is_instance_of::() { + return Ok(value.clone()); + } + if let Ok(text) = value.downcast::() + && let Some(flag) = dbt_bool_text(text.to_str()?) + { + return Ok(PyBool::new(value.py(), flag).to_owned().into_any()); + } + Err(coercion_error(value, "bool")?) +} + +/// An int or float, or a string parsed as one. +#[pyfunction] +#[pyo3(signature = (value))] +fn coerce_dbt_profile_number<'py>(value: &Bound<'py, PyAny>) -> PyResult> { + let py: Python<'py> = value.py(); + if value.is_instance_of::() { + return Err(coercion_error(value, "number")?); + } + if value.is_instance_of::() || value.is_instance_of::() { + return Ok(value.clone()); + } + let Ok(text) = value.downcast::() else { + return Err(coercion_error(value, "number")?); + }; + let (stripped, parse): (&str, DbtNumberParse) = dbt_number_text(text.to_str()?); + let parsed: PyResult> = match parse { + DbtNumberParse::Float => py.get_type::().call1((stripped,)), + DbtNumberParse::Integer => py.get_type::().call1((stripped,)), + }; + match parsed { + Err(error) if error.is_instance_of::(py) => { + let raised: PyErr = coercion_error(value, "number")?; + raised.set_cause(py, Some(error)); + Err(raised) + } + parsed => parsed, + } +} + +pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { + module.add_function(wrap_pyfunction!(coerce_dbt_profile_bool, module)?)?; + module.add_function(wrap_pyfunction!(coerce_dbt_profile_number, module)?) +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/dbt_interop/registration.rs b/crates/sqlbuild-python/src/bindings/_helpers/dbt_interop/registration.rs index 02bb776394..f188970794 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/dbt_interop/registration.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/dbt_interop/registration.rs @@ -14,6 +14,7 @@ pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { orchestration::register(module)?; plan::register(module)?; profile::register(module)?; + super::profile_coercion::register(module)?; super::profile_connection::register(module)?; super::reference_resolver::register(module)?; super::runtime_config::register(module)?; diff --git a/python-boundary.toml b/python-boundary.toml index c566c7b3ce..9dca15b98a 100644 --- a/python-boundary.toml +++ b/python-boundary.toml @@ -395,7 +395,7 @@ note = "Rule 4: PyYAML glue only; parses dbt_project.yml and profiles.yml with y [[stays]] path = "integrations/dbt/_helpers/profile/render.py" rule = 4 -note = "Renders the selected dbt profile output with Jinja, env_var and var as dbt does." +note = "Renders the selected dbt profile output with Jinja, env_var and var as dbt does; as_bool and as_number coercions are native." [[stays]] path = "cli/commands/main/entrypoint/_dispatch_with_compute_logs.py" diff --git a/src/sqlbuild/_native.pyi b/src/sqlbuild/_native.pyi index 5c1193c4db..8b9c38fee1 100644 --- a/src/sqlbuild/_native.pyi +++ b/src/sqlbuild/_native.pyi @@ -2909,6 +2909,8 @@ def validate_preview_target( def normalize_dbt_profile_connection( *, resolved: ResolvedDbtProfileOutput ) -> NormalizedDbtProfileConnection: ... +def coerce_dbt_profile_bool(value: object) -> bool: ... +def coerce_dbt_profile_number(value: object) -> int | float: ... def build_dbt_init_project_toml( *, project_name: str, @@ -4001,8 +4003,6 @@ def build_ingestr_command( is_reload: bool, ) -> tuple[str, ...]: ... def format_ingestr_command(command: Sequence[str], /) -> str: ... -def format_clone_execution_json( - *, result: CloneExecutionResult, resource_types_by_name: Mapping[str, str] def dlt_destination_arguments( *, adapter_name: str, @@ -4010,6 +4010,8 @@ def dlt_destination_arguments( destination_config: Mapping[str, object] | None, dataset_name: str | None, ) -> tuple[str, dict[str, Any], dict[str, Any], str | None]: ... +def format_clone_execution_json( + *, result: CloneExecutionResult, resource_types_by_name: Mapping[str, str] ) -> str: ... def format_test_execution_json(*, results: tuple[SqlTestExecutionResult, ...]) -> str: ... def format_audit_execution_json( @@ -5092,6 +5094,7 @@ def build_microbatch_create_index_sqls( ) -> tuple[str, ...]: ... CURRENT_LIFECYCLE_EVENT_SCHEMA_VERSION: int +CURRENT_DIAGNOSTIC_LOG_SCHEMA_VERSION: int LIFECYCLE_EVENT_SCHEMA_VERSIONS: frozenset[int] HOOK_PHASES: frozenset[str] HOOK_TYPES: frozenset[str] @@ -5101,7 +5104,6 @@ OPERATION_METADATA_FIELDS: frozenset[str] OPERATION_NAMES: frozenset[str] OPERATION_PHASES: frozenset[str] OPERATION_SCOPES: frozenset[str] -CURRENT_DIAGNOSTIC_LOG_SCHEMA_VERSION: int OPERATION_STRATEGIES: frozenset[str] OPERATION_TARGET_KINDS: frozenset[str] RESOURCE_SKIP_CODES: frozenset[str] @@ -5115,15 +5117,6 @@ def validate_lifecycle_required_text(*, value: object, field_name: str) -> None: def validate_lifecycle_optional_text(*, value: object, field_name: str) -> None: ... def validate_lifecycle_timestamp(*, value: object) -> None: ... def validate_known_lifecycle_event(*, event: LifecycleEvent) -> None: ... -def render_state_table_create_sql( - *, - qualified_name: str, - columns: tuple[str, ...], - column_types: Mapping[str, StateSqlValueType], - required_columns: frozenset[str], - render_framework_type: Callable[[FrameworkType], str], -) -> str: ... -def render_insert_source_freshness_records_sql( def validate_lifecycle_event(*, event: LifecycleEvent) -> None: ... def validate_lifecycle_invocation_sequence(*, value: object) -> None: ... def validate_diagnostic_log(*, record: DiagnosticLog) -> None: ... @@ -5139,6 +5132,15 @@ def validated_operation_hook_fields( hook_name: str | None, ) -> dict[str, JSONValue]: ... def validate_lifecycle_producer(*, producer: str, producer_version: str | None) -> None: ... +def render_state_table_create_sql( + *, + qualified_name: str, + columns: tuple[str, ...], + column_types: Mapping[str, StateSqlValueType], + required_columns: frozenset[str], + render_framework_type: Callable[[FrameworkType], str], +) -> str: ... +def render_insert_source_freshness_records_sql( *, database: str | None, schema: str, @@ -6651,15 +6653,6 @@ def build_direct_source_freshness_propagation_result( ) -> DirectSourceFreshnessPropagationResult: ... def redact_diagnostic_text(value: str) -> str: ... def redact_diagnostic_value(*, name: str, value: object) -> object: ... -def sql_rule_lint_key(*, config: LintConfig) -> str: ... -def sql_model_rule_keys( - *, lint_key: str, codes: list[str], models: list[tuple[str, str, str, str | None]] -) -> dict[str, str]: ... -def sql_file_rule_key( - *, lint_key: str, codes: list[str], relative_path: str, contents: str, expanded: bool -) -> str | None: ... -def load_sql_rule_cache(*, path: Path) -> dict[str, tuple[str, str]]: ... -def lookup_sql_rule_cache( def sql_log_record( *, context: Mapping[str, object] | None, @@ -6686,6 +6679,15 @@ def safe_diagnostic_extra_fields(record: logging.LogRecord) -> dict[str, object] def compute_diagnostic_log_fields( *, record: logging.LogRecord, invocation_id: str, include_sql_text: bool ) -> dict[str, object] | None: ... +def sql_rule_lint_key(*, config: LintConfig) -> str: ... +def sql_model_rule_keys( + *, lint_key: str, codes: list[str], models: list[tuple[str, str, str, str | None]] +) -> dict[str, str]: ... +def sql_file_rule_key( + *, lint_key: str, codes: list[str], relative_path: str, contents: str, expanded: bool +) -> str | None: ... +def load_sql_rule_cache(*, path: Path) -> dict[str, tuple[str, str]]: ... +def lookup_sql_rule_cache( *, entries: dict[str, tuple[str, str]], keys: list[tuple[str, str]] ) -> dict[str, list[Finding]]: ... def encode_sql_rule_findings(*, findings: list[Finding]) -> str: ... diff --git a/src/sqlbuild/integrations/dbt/_helpers/profile/render.py b/src/sqlbuild/integrations/dbt/_helpers/profile/render.py index 902ac42703..97a519e8c9 100644 --- a/src/sqlbuild/integrations/dbt/_helpers/profile/render.py +++ b/src/sqlbuild/integrations/dbt/_helpers/profile/render.py @@ -8,13 +8,15 @@ from pathlib import Path from typing import Any, cast -from sqlbuild._native import ResolvedDbtProfileOutput, SelectedDbtProfileOutput +from sqlbuild._native import ( + ResolvedDbtProfileOutput, + SelectedDbtProfileOutput, + coerce_dbt_profile_bool, + coerce_dbt_profile_number, +) from sqlbuild.integrations.dbt.exceptions import DbtProfileError -_DBT_FALSE_VALUES: frozenset[str] = frozenset({"false", "0", "no", "n", "off"}) _DBT_JINJA_MARKERS: tuple[str, ...] = ("{{", "{%", "{#") -_DBT_NUMBER_DECIMAL_SEPARATOR: str = "." -_DBT_TRUE_VALUES: frozenset[str] = frozenset({"true", "1", "yes", "y", "on"}) class _Missing: @@ -137,8 +139,8 @@ def var(name: str, default: object = _MISSING) -> object: globals_map["target"] = target environment.filters.update( { - "as_bool": _as_bool, - "as_number": _as_number, + "as_bool": coerce_dbt_profile_bool, + "as_number": coerce_dbt_profile_number, "as_native": _as_native, } ) @@ -152,33 +154,5 @@ def var(name: str, default: object = _MISSING) -> object: ) from error -def _as_bool(value: object) -> bool: - if isinstance(value, bool): - return value - if isinstance(value, str): - normalized: str = value.strip().lower() - if normalized in _DBT_TRUE_VALUES: - return True - if normalized in _DBT_FALSE_VALUES: - return False - raise DbtProfileError(f"Cannot coerce {value!r} to bool") - - -def _as_number(value: object) -> int | float: - if isinstance(value, bool): - raise DbtProfileError(f"Cannot coerce {value!r} to number") - if isinstance(value, int | float): - return value - if isinstance(value, str): - stripped: str = value.strip() - try: - if _DBT_NUMBER_DECIMAL_SEPARATOR in stripped: - return float(stripped) - return int(stripped) - except ValueError as error: - raise DbtProfileError(f"Cannot coerce {value!r} to number") from error - raise DbtProfileError(f"Cannot coerce {value!r} to number") - - def _as_native(value: object) -> object: return value From 2b1924ffe6966ad7d548dd92f24d79813ab0e60e Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Fri, 16 Oct 2026 00:47:43 +0100 Subject: [PATCH 39/48] refactor: single-source diagnostic log record field and logger constants in Rust sqlbuild_sql, sqlbuild_sql_digest, sqlbuild_private_sql_record and the SQLBuild logger root and prefix are defined once in sqlbuild-core diagnostic_routing. Python reads the field names it still needs from sqlbuild._native; diagnostics/constants.py keeps only the private record marker object. --- .../src/diagnostic_text/_helpers/record_parts.rs | 4 ++-- crates/sqlbuild-core/src/diagnostic_text/constants.rs | 6 +----- .../src/bindings/_helpers/diagnostics/console_records.rs | 4 ++-- .../bindings/_helpers/exported_constants/constants.rs | 9 ++++++--- python-boundary.toml | 2 +- src/sqlbuild/_native.pyi | 3 ++- src/sqlbuild/diagnostics/_helpers/logging.py | 1 - .../diagnostics/classes/invocation_diagnostic_routing.py | 4 ++-- .../classes/test_scoped_compute_log_capture.py | 3 +-- 9 files changed, 17 insertions(+), 19 deletions(-) diff --git a/crates/sqlbuild-core/src/diagnostic_text/_helpers/record_parts.rs b/crates/sqlbuild-core/src/diagnostic_text/_helpers/record_parts.rs index fee887e958..841ed92da6 100644 --- a/crates/sqlbuild-core/src/diagnostic_text/_helpers/record_parts.rs +++ b/crates/sqlbuild-core/src/diagnostic_text/_helpers/record_parts.rs @@ -1,7 +1,7 @@ //! The header pieces of a `--debug` console line. use crate::cli_text::models::CliStyle; -use crate::diagnostic_text::constants::DIAGNOSTIC_LOGGER_PREFIX; +use crate::diagnostic_routing::constants::LOGGER_PREFIX; use crate::diagnostic_text::models::DiagnosticRecordText; /// Whether Python's `str.isspace()` holds for `character`. @@ -43,7 +43,7 @@ pub(crate) fn header(record: &DiagnosticRecordText, style: CliStyle) -> String { let context: String = context_message(record, style).unwrap_or_else(|| { let scope: &str = record .logger_name - .strip_prefix(DIAGNOSTIC_LOGGER_PREFIX) + .strip_prefix(LOGGER_PREFIX) .unwrap_or(&record.logger_name); format!( "{} {}", diff --git a/crates/sqlbuild-core/src/diagnostic_text/constants.rs b/crates/sqlbuild-core/src/diagnostic_text/constants.rs index fa2933b9d4..536adb4da1 100644 --- a/crates/sqlbuild-core/src/diagnostic_text/constants.rs +++ b/crates/sqlbuild-core/src/diagnostic_text/constants.rs @@ -1,9 +1,5 @@ -//! Prefixes, statements and widths of `--debug` console lines. +//! Statements and widths of `--debug` console lines. -/// The logger prefix diagnostics console headers drop from SQLBuild logger names. -pub const DIAGNOSTIC_LOGGER_PREFIX: &str = "sqlbuild."; -/// The log record attribute carrying a diagnostic's SQL text. -pub const SQL_TEXT_LOG_FIELD: &str = "sqlbuild_sql"; /// Statements a diagnostics console line shows inline instead of in a block. pub(crate) const INLINE_SQL_STATEMENTS: [&str; 3] = ["BEGIN", "COMMIT", "ROLLBACK"]; /// The rule above and below a block of logged SQL. diff --git a/crates/sqlbuild-python/src/bindings/_helpers/diagnostics/console_records.rs b/crates/sqlbuild-python/src/bindings/_helpers/diagnostics/console_records.rs index c1432c8c1f..e2e9943643 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/diagnostics/console_records.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/diagnostics/console_records.rs @@ -4,7 +4,7 @@ use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyModule, PyModuleMethods, PyRes use pyo3::types::{PyInt, PyString}; use pyo3::{pyfunction, wrap_pyfunction}; use sqlbuild_core::cli_text::models::CliStyle; -use sqlbuild_core::diagnostic_text::constants::SQL_TEXT_LOG_FIELD; +use sqlbuild_core::diagnostic_routing::constants::SQL_TEXT_FIELD; use sqlbuild_core::diagnostic_text::main::format_diagnostic_record::format_diagnostic_record; use sqlbuild_core::diagnostic_text::models::DiagnosticRecordText; @@ -62,7 +62,7 @@ fn format_diagnostics_console_record( event: text_field(record, "sqlbuild_event")?, tokens, duration_ms, - sql: text_field(record, SQL_TEXT_LOG_FIELD)?, + sql: text_field(record, SQL_TEXT_FIELD)?, }; Ok(format_diagnostic_record(&text, CliStyle { use_color })) } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/exported_constants/constants.rs b/crates/sqlbuild-python/src/bindings/_helpers/exported_constants/constants.rs index 01ed8cecef..0718d4e3f7 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/exported_constants/constants.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/exported_constants/constants.rs @@ -26,7 +26,9 @@ use sqlbuild_core::constants::{ SETTINGS_SECTION, SOURCE_FRESHNESS_TABLE_NAME, STAGED_PROMOTION_MODE, TABLE_PROMOTION_MODE_SETTING_KEY, ZERO_DAY_DURATION, }; -use sqlbuild_core::diagnostic_text::constants::{DIAGNOSTIC_LOGGER_PREFIX, SQL_TEXT_LOG_FIELD}; +use sqlbuild_core::diagnostic_routing::constants::{ + SQL_DIGEST_FIELD, SQL_PRIVATE_RECORD_FIELD, SQL_TEXT_FIELD, +}; use sqlbuild_rules::constants::MIN_CUSTOM_RULE_TEST_CASES_KEY; use sqlbuild_rules::custom_host_pool::constants::CUSTOM_RULE_PROJECT_SUBJECT; use sqlbuild_state::execution_history::constants::{ @@ -103,8 +105,9 @@ pub(crate) const EXPORTED_TEXTS: &[(&str, &str)] = &[ ("LAUNCH_RECORD_OPTION", LAUNCH_RECORD_OPTION), ("ZERO_DAY_CURSOR_DURATION", ZERO_DAY_DURATION), ("AST_CTE_KIND", CTE_AST_KIND), - ("LOGGER_PREFIX", DIAGNOSTIC_LOGGER_PREFIX), - ("SQL_TEXT_FIELD", SQL_TEXT_LOG_FIELD), + ("SQL_TEXT_FIELD", SQL_TEXT_FIELD), + ("SQL_DIGEST_FIELD", SQL_DIGEST_FIELD), + ("SQL_PRIVATE_RECORD_FIELD", SQL_PRIVATE_RECORD_FIELD), ("MIN_CUSTOM_RULE_TEST_CASES", MIN_CUSTOM_RULE_TEST_CASES_KEY), ]; /// Integer constants exported under their Python names. diff --git a/python-boundary.toml b/python-boundary.toml index 9dca15b98a..eb97c109a2 100644 --- a/python-boundary.toml +++ b/python-boundary.toml @@ -730,7 +730,7 @@ note = "Shape rule: data exceptions only (DbtProfileError), no logic; consumed b [[stays]] path = "diagnostics/constants.py" rule = 4 -note = "Shape rule: data constants only (SQL_DIGEST_FIELD, SQL_TEXT_FIELD, SQL_PRIVATE_RECORD_FIELD, SQL_PRIVATE_RECORD_MARKER), no logic; the marker object identifies trusted private SQL records for the native compute log handler; consumed by diagnostics/classes/invocation_diagnostic_routing.py." +note = "Shape rule: the SQL_PRIVATE_RECORD_MARKER object() sentinel only, no logic; its identity marks trusted private SQL records for the native compute log handler; the record field names are native (sqlbuild._native); consumed by diagnostics/classes/invocation_diagnostic_routing.py." [[stays]] path = "cursor_algebra/exceptions.py" diff --git a/src/sqlbuild/_native.pyi b/src/sqlbuild/_native.pyi index 8b9c38fee1..138bac52ba 100644 --- a/src/sqlbuild/_native.pyi +++ b/src/sqlbuild/_native.pyi @@ -216,8 +216,9 @@ CONTRACT_ENFORCED: str LAUNCH_RECORD_OPTION: str ZERO_DAY_CURSOR_DURATION: str AST_CTE_KIND: str -LOGGER_PREFIX: str SQL_TEXT_FIELD: str +SQL_DIGEST_FIELD: str +SQL_PRIVATE_RECORD_FIELD: str MIN_CUSTOM_RULE_TEST_CASES: str DEFAULT_PAGE_LIMIT: int MAX_PAGE_LIMIT: int diff --git a/src/sqlbuild/diagnostics/_helpers/logging.py b/src/sqlbuild/diagnostics/_helpers/logging.py index efe555958f..b9f33d612c 100644 --- a/src/sqlbuild/diagnostics/_helpers/logging.py +++ b/src/sqlbuild/diagnostics/_helpers/logging.py @@ -16,7 +16,6 @@ ) from sqlbuild.runtime.observability.models import ExecutionIdentity -_LOGGER_ROOT_NAME: str = "sqlbuild" _EMPTY_CONTEXT: dict[str, object] = {} _DIAGNOSTICS_CONTEXT: ContextVar[dict[str, object] | None] = ContextVar( "sqlbuild_diagnostics_context", default=None diff --git a/src/sqlbuild/diagnostics/classes/invocation_diagnostic_routing.py b/src/sqlbuild/diagnostics/classes/invocation_diagnostic_routing.py index cf684cac9f..a0e24e3c17 100644 --- a/src/sqlbuild/diagnostics/classes/invocation_diagnostic_routing.py +++ b/src/sqlbuild/diagnostics/classes/invocation_diagnostic_routing.py @@ -7,12 +7,12 @@ from contextvars import Token from types import TracebackType -from sqlbuild._native import SQL_TEXT_FIELD +from sqlbuild._native import SQL_PRIVATE_RECORD_FIELD, SQL_TEXT_FIELD from sqlbuild.diagnostics._helpers.logging import set_sql_diagnostic_sink from sqlbuild.diagnostics.classes.diagnostics_console_formatter import DiagnosticsConsoleFormatter from sqlbuild.diagnostics.classes.dynamic_stderr_handler import DynamicStderrHandler from sqlbuild.diagnostics.classes.log_record_route_filter import LogRecordRouteFilter -from sqlbuild.diagnostics.constants import SQL_PRIVATE_RECORD_FIELD, SQL_PRIVATE_RECORD_MARKER +from sqlbuild.diagnostics.constants import SQL_PRIVATE_RECORD_MARKER from sqlbuild.diagnostics.models import DiagnosticRoutingOptions from sqlbuild.runtime.compute_logs.classes.diagnostic_log_handler import ( ComputeDiagnosticLogHandler, diff --git a/tests/unit/src/sqlbuild/runtime/compute_logs/classes/test_scoped_compute_log_capture.py b/tests/unit/src/sqlbuild/runtime/compute_logs/classes/test_scoped_compute_log_capture.py index f35e26dba5..38fc3c29a7 100644 --- a/tests/unit/src/sqlbuild/runtime/compute_logs/classes/test_scoped_compute_log_capture.py +++ b/tests/unit/src/sqlbuild/runtime/compute_logs/classes/test_scoped_compute_log_capture.py @@ -10,14 +10,13 @@ import pytest -from sqlbuild._native import LocalFilesystemComputeLogStorage +from sqlbuild._native import SQL_DIGEST_FIELD, LocalFilesystemComputeLogStorage from sqlbuild.compute_logs import ( CaptureMetadata, CaptureStateError, ComputeLogStream, FinalCaptureMetadata, ) -from sqlbuild.diagnostics.constants import SQL_DIGEST_FIELD from sqlbuild.diagnostics.main.log_debug_event import log_debug_event from sqlbuild.diagnostics.main.log_sql import log_sql from sqlbuild.diagnostics.models import DiagnosticRoutingOptions From c6df205557af11146bdb626b38dce013312fc041 Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Fri, 16 Oct 2026 00:47:43 +0100 Subject: [PATCH 40/48] fix: format any integer debug console duration like Python without raising The diagnostics console formatter read sqlbuild_duration_ms as i64 for every record, so integers above 2**63 raised OverflowError from logging. The duration is now Python's ms / 1000, read only when sqlbuild_subject is a string, and omitted when the quotient does not fit a float. --- .../diagnostic_text/_helpers/record_parts.rs | 4 +- .../src/diagnostic_text/models.rs | 4 +- .../src/diagnostic_text/tests/test_format.rs | 2 +- .../_helpers/diagnostics/console_records.rs | 34 ++++++-- .../src/sqlbuild/diagnostics/_test_types.py | 8 ++ .../diagnostics/test_console_formatter.py | 78 +++++++++++++++++++ 6 files changed, 117 insertions(+), 13 deletions(-) create mode 100644 tests/unit/src/sqlbuild/diagnostics/test_console_formatter.py diff --git a/crates/sqlbuild-core/src/diagnostic_text/_helpers/record_parts.rs b/crates/sqlbuild-core/src/diagnostic_text/_helpers/record_parts.rs index 841ed92da6..3a347899cc 100644 --- a/crates/sqlbuild-core/src/diagnostic_text/_helpers/record_parts.rs +++ b/crates/sqlbuild-core/src/diagnostic_text/_helpers/record_parts.rs @@ -27,8 +27,8 @@ fn context_message(record: &DiagnosticRecordText, style: CliStyle) -> Option, /// Each `label=value` context token whose record field is a string, in display order. pub tokens: Vec<(&'static str, String)>, - /// `sqlbuild_duration_ms` when it is an integer. - pub duration_ms: Option, + /// `sqlbuild_duration_ms / 1000` when the field is an integer, as Python divides it. + pub duration_seconds: Option, /// `sqlbuild_sql` when it is a string. pub sql: Option, } diff --git a/crates/sqlbuild-core/src/diagnostic_text/tests/test_format.rs b/crates/sqlbuild-core/src/diagnostic_text/tests/test_format.rs index e947c420f1..624bebd3e9 100644 --- a/crates/sqlbuild-core/src/diagnostic_text/tests/test_format.rs +++ b/crates/sqlbuild-core/src/diagnostic_text/tests/test_format.rs @@ -26,7 +26,7 @@ fn given_log_record_fields_when_formatting_then_matches_console_line() { subject: Some("model".to_owned()), name: Some("orders".to_owned()), tokens: vec![("phase", "run".to_owned())], - duration_ms: Some(1005), + duration_seconds: Some(1.005), ..DiagnosticRecordText::default() }, use_color: true, diff --git a/crates/sqlbuild-python/src/bindings/_helpers/diagnostics/console_records.rs b/crates/sqlbuild-python/src/bindings/_helpers/diagnostics/console_records.rs index e2e9943643..e8e8571bbe 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/diagnostics/console_records.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/diagnostics/console_records.rs @@ -1,5 +1,6 @@ //! The `--debug` stderr line of one Python log record. +use pyo3::exceptions::PyOverflowError; use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyModule, PyModuleMethods, PyResult}; use pyo3::types::{PyInt, PyString}; use pyo3::{pyfunction, wrap_pyfunction}; @@ -8,6 +9,10 @@ use sqlbuild_core::diagnostic_routing::constants::SQL_TEXT_FIELD; use sqlbuild_core::diagnostic_text::main::format_diagnostic_record::format_diagnostic_record; use sqlbuild_core::diagnostic_text::models::DiagnosticRecordText; +/// The record field holding an operation's duration in milliseconds. +const DURATION_FIELD: &str = "sqlbuild_duration_ms"; +const MILLISECONDS_PER_SECOND: i64 = 1000; + /// Each context field shown as a `label=value` token, in display order. const CONTEXT_TOKENS: [(&str, &str); 7] = [ ("sqlbuild_phase", "phase"), @@ -31,6 +36,22 @@ fn text_field(record: &Bound<'_, PyAny>, field: &str) -> PyResult Ok(None) } +/// Python's `sqlbuild_duration_ms / 1000` for an integer field that fits a float, else `None`. +fn duration_seconds(record: &Bound<'_, PyAny>) -> PyResult> { + if !record.hasattr(DURATION_FIELD)? { + return Ok(None); + } + let value: Bound<'_, PyAny> = record.getattr(DURATION_FIELD)?; + if !value.is_instance_of::() { + return Ok(None); + } + match value.div(MILLISECONDS_PER_SECOND) { + Ok(seconds) => Ok(Some(seconds.extract()?)), + Err(error) if error.is_instance_of::(record.py()) => Ok(None), + Err(error) => Err(error), + } +} + /// The console line of `record` without its exception text. #[pyfunction] #[pyo3(signature = (record, *, use_color))] @@ -44,12 +65,9 @@ fn format_diagnostics_console_record( tokens.push((label, value)); } } - let duration_ms: Option = if record.hasattr("sqlbuild_duration_ms")? { - let value: Bound<'_, PyAny> = record.getattr("sqlbuild_duration_ms")?; - value - .is_instance_of::() - .then(|| value.extract()) - .transpose()? + let subject: Option = text_field(record, "sqlbuild_subject")?; + let duration_seconds: Option = if subject.is_some() { + duration_seconds(record)? } else { None }; @@ -57,11 +75,11 @@ fn format_diagnostics_console_record( level_name: record.getattr("levelname")?.extract()?, logger_name: record.getattr("name")?.extract()?, message: record.call_method0("getMessage")?.extract()?, - subject: text_field(record, "sqlbuild_subject")?, + subject, name: text_field(record, "sqlbuild_name")?, event: text_field(record, "sqlbuild_event")?, tokens, - duration_ms, + duration_seconds, sql: text_field(record, SQL_TEXT_FIELD)?, }; Ok(format_diagnostic_record(&text, CliStyle { use_color })) diff --git a/tests/unit/src/sqlbuild/diagnostics/_test_types.py b/tests/unit/src/sqlbuild/diagnostics/_test_types.py index f7ab2e6602..eb51dbd1fe 100644 --- a/tests/unit/src/sqlbuild/diagnostics/_test_types.py +++ b/tests/unit/src/sqlbuild/diagnostics/_test_types.py @@ -29,3 +29,11 @@ class ProcessResourceTestCase: expected_user_cpu_seconds: float expected_system_cpu_seconds: float expected_max_rss_bytes: int | None + + +@dataclass(frozen=True) +class ConsoleDurationTestCase: + description: str + subject: object + duration_ms: object + expected_line: str diff --git a/tests/unit/src/sqlbuild/diagnostics/test_console_formatter.py b/tests/unit/src/sqlbuild/diagnostics/test_console_formatter.py new file mode 100644 index 0000000000..4a08a8eaaf --- /dev/null +++ b/tests/unit/src/sqlbuild/diagnostics/test_console_formatter.py @@ -0,0 +1,78 @@ +"""Tests for the diagnostics console formatter's duration token.""" + +from __future__ import annotations + +import logging + +import pytest + +from sqlbuild.diagnostics.classes.diagnostics_console_formatter import ( + DiagnosticsConsoleFormatter, +) +from tests.unit.src.sqlbuild.diagnostics._test_types import ConsoleDurationTestCase + + +@pytest.mark.parametrize( + "test_case", + [ + ConsoleDurationTestCase( + description="small duration rounds like Python", + subject="model", + duration_ms=1005, + expected_line="[info] model duration=1.00s done", + ), + ConsoleDurationTestCase( + description="duration above 2**63 formats like Python", + subject="model", + duration_ms=2**70, + expected_line="[info] model duration=1180591620717411328.00s done", + ), + ConsoleDurationTestCase( + description="negative duration below -2**63 formats like Python", + subject="model", + duration_ms=-(2**80), + expected_line="[info] model duration=-1208925819614629199872.00s done", + ), + ConsoleDurationTestCase( + description="bool duration is an int", + subject="model", + duration_ms=True, + expected_line="[info] model duration=0.00s done", + ), + ConsoleDurationTestCase( + description="non-integer duration is ignored", + subject="model", + duration_ms=1.5, + expected_line="[info] model done", + ), + ConsoleDurationTestCase( + description="non-string subject ignores an oversized duration", + subject=5, + duration_ms=2**70, + expected_line="[info] exec done", + ), + ConsoleDurationTestCase( + description="duration too large for a float is omitted instead of raising", + subject="model", + duration_ms=10**400, + expected_line="[info] model done", + ), + ], + ids=lambda case: case.description, +) +def test_given_duration_field_when_formatting_console_line_then_matches_python_rendering( + test_case: ConsoleDurationTestCase, +) -> None: + record: logging.LogRecord = logging.makeLogRecord( + { + "name": "sqlbuild.exec", + "levelname": "INFO", + "msg": "done", + "sqlbuild_subject": test_case.subject, + "sqlbuild_duration_ms": test_case.duration_ms, + } + ) + + line: str = DiagnosticsConsoleFormatter().format(record) + + assert line == test_case.expected_line, test_case.description From bfa7e19bcc0ed0d4470de25045490a5b5506c81d Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Fri, 16 Oct 2026 01:08:10 +0100 Subject: [PATCH 41/48] refactor: move flagged rule, Python-node, Dagster translator and startup-stamp logic to native --- .../analysis_session/_helpers/cte_facts.rs | 32 +-- crates/sqlbuild-analysis/src/lib.rs | 1 + .../main/conditional_result_nullability.rs | 19 ++ .../main/first_arg_nullability.rs | 8 + .../src/nullability_rules/main/mod.rs | 3 + .../main/safe_cast_nullability.rs | 12 + .../src/nullability_rules/mod.rs | 4 + .../src/nullability_rules/tests.rs | 4 + .../tests/test_nullability_rules.rs | 69 +++++ .../src/nullability_rules/tests/test_types.rs | 7 + crates/sqlbuild-cache/src/lib.rs | 1 + .../src/startup_inputs/_helpers/mod.rs | 1 + .../src/startup_inputs/_helpers/pure_paths.rs | 55 ++++ .../src/startup_inputs/constants.rs | 8 + .../src/startup_inputs/main/mod.rs | 1 + .../src/startup_inputs/main/startup_stamps.rs | 66 +++++ .../sqlbuild-cache/src/startup_inputs/mod.rs | 9 + .../src/startup_inputs/models.rs | 22 ++ .../src/startup_inputs/tests.rs | 4 + .../tests/test_startup_stamps.rs | 66 +++++ .../src/startup_inputs/tests/test_types.rs | 7 + .../src/project/_helpers/help.rs | 15 +- crates/sqlbuild-config/src/toml/main/mod.rs | 1 + .../src/toml/main/toml_literal.rs | 20 ++ crates/sqlbuild-config/src/toml/mod.rs | 1 + crates/sqlbuild-config/src/toml/models.rs | 11 + crates/sqlbuild-config/src/toml/tests.rs | 2 + .../src/toml/tests/test_toml_literal.rs | 50 ++++ .../src/toml/tests/test_types.rs | 6 + .../dagster_invocation/_helpers/markdown.rs | 22 ++ .../src/dagster_invocation/_helpers/mod.rs | 2 + .../src/dagster_invocation/_helpers/names.rs | 19 ++ .../src/dagster_invocation/constants.rs | 12 + .../main/asset_description.rs | 19 ++ .../main/dagster_name_segment.rs | 18 ++ .../main/dagster_tag_key.rs | 16 ++ .../main/fallback_asset_description.rs | 23 ++ .../src/dagster_invocation/main/mod.rs | 4 + .../src/dagster_invocation/tests.rs | 2 + .../tests/test_translator.rs | 143 +++++++++++ .../dagster_invocation/tests/test_types.rs | 24 ++ .../main/is_lowercase_identifier.rs | 9 + .../src/identifier_text/main/mod.rs | 1 + .../sqlbuild-core/src/identifier_text/mod.rs | 6 + .../src/identifier_text/tests.rs | 4 + .../tests/test_lowercase_identifier.rs | 76 ++++++ .../src/identifier_text/tests/test_types.rs | 5 + crates/sqlbuild-core/src/lib.rs | 1 + .../sqlbuild-core/src/posix_path/constants.rs | 1 + .../sqlbuild-core/src/posix_path/main/mod.rs | 1 + .../src/posix_path/main/posix_escapes_root.rs | 12 + .../src/posix_path/tests/test_posix_path.rs | 67 ++++- .../src/posix_path/tests/test_types.rs | 6 + .../src/rule_evidence/constants.rs | 34 +++ .../main/allowed_custom_rule_import.rs | 13 + .../rule_evidence/main/disallowed_import.rs | 49 ++++ .../src/rule_evidence/main/mod.rs | 2 + .../src/rule_evidence/models.rs | 11 + .../src/rule_evidence/tests.rs | 2 + .../rule_evidence/tests/test_hermeticity.rs | 121 +++++++++ .../src/rule_evidence/tests/test_types.rs | 15 ++ .../bindings/_helpers/authored_values/mod.rs | 1 + .../_helpers/authored_values/python_nodes.rs | 209 +++++++++++++++ .../_helpers/authored_values/registration.rs | 3 +- .../bindings/_helpers/compile_command/mod.rs | 1 + .../_helpers/compile_command/registration.rs | 4 +- .../compile_command/startup_inputs.rs | 69 +++++ .../_helpers/dagster_invocation/mod.rs | 1 + .../dagster_invocation/registration.rs | 29 ++- .../_helpers/dagster_invocation/translator.rs | 238 ++++++++++++++++++ .../src/bindings/_helpers/rules/catalogue.rs | 10 +- .../_helpers/rules/custom_rule_evidence.rs | 119 ++++++++- .../src/bindings/_helpers/rules/mod.rs | 1 + .../bindings/_helpers/rules/registration.rs | 5 +- .../src/bindings/_helpers/rules/rule_api.rs | 226 +++++++++++++++++ .../bindings/_helpers/rules/rule_dispatch.rs | 7 +- .../src/bindings/_helpers/type_system/mod.rs | 1 + .../_helpers/type_system/nullability.rs | 75 ++++++ .../_helpers/type_system/registration.rs | 3 +- crates/sqlbuild-rules/src/lib.rs | 1 + .../src/project_tree/_helpers/entries.rs | 28 +++ .../src/project_tree/_helpers/files.rs | 64 +++++ .../src/project_tree/_helpers/mod.rs | 3 + .../project_tree/_helpers/relative_paths.rs | 20 ++ .../src/project_tree/constants.rs | 16 ++ .../sqlbuild-rules/src/project_tree/errors.rs | 20 ++ .../src/project_tree/main/mod.rs | 3 + .../project_tree/main/project_tree_entries.rs | 38 +++ .../project_tree/main/project_tree_file.rs | 32 +++ .../main/tracked_text_suffixes.rs | 9 + crates/sqlbuild-rules/src/project_tree/mod.rs | 10 + .../sqlbuild-rules/src/project_tree/models.rs | 17 ++ .../sqlbuild-rules/src/project_tree/tests.rs | 6 + .../src/project_tree/tests/helpers.rs | 24 ++ .../project_tree/tests/test_project_tree.rs | 113 +++++++++ .../src/project_tree/tests/test_types.rs | 13 + .../src/rules/_helpers/evaluation.rs | 13 +- python-boundary.toml | 23 +- src/sqlbuild/_native.pyi | 44 ++++ .../type_system/_helpers/inference_rules.py | 41 --- .../main/conditional_result_nullability.py | 6 +- .../type_system/main/first_arg_nullability.py | 6 +- .../type_system/main/safe_cast_nullability.py | 6 +- .../compile_reuse/_helpers/startup_inputs.py | 30 +-- .../classes/sqlbuild_dagster_translator.py | 123 ++------- .../_helpers/dependency_normalization.py | 16 -- src/sqlbuild/python_nodes/main/apply_hook.py | 2 +- .../main/calculate_retry_delay.py | 12 +- src/sqlbuild/python_nodes/models.py | 64 ++--- .../_helpers/engine/hermeticity.py | 52 ---- .../_helpers/engine/rule_harness.py | 16 +- .../rule_engine/classes/project_tree.py | 87 +------ .../rule_engine/classes/rule_context.py | 11 +- .../rule_engine/classes/runtime_guard.py | 3 +- src/sqlbuild/rule_engine/constants.py | 35 --- .../adapter/type_system/_test_types.py | 17 ++ .../type_system/test_nullability_rules.py | 92 +++++++ .../cli/compile_reuse/_helpers/_test_types.py | 22 ++ .../_helpers/test_startup_inputs.py | 85 +++++++ .../sqlbuild/python_nodes/main/_test_types.py | 19 ++ .../main/test_calculate_retry_delay.py | 93 +++++++ .../_helpers/engine/test_hermeticity.py | 3 +- 122 files changed, 3063 insertions(+), 522 deletions(-) create mode 100644 crates/sqlbuild-analysis/src/nullability_rules/main/conditional_result_nullability.rs create mode 100644 crates/sqlbuild-analysis/src/nullability_rules/main/first_arg_nullability.rs create mode 100644 crates/sqlbuild-analysis/src/nullability_rules/main/mod.rs create mode 100644 crates/sqlbuild-analysis/src/nullability_rules/main/safe_cast_nullability.rs create mode 100644 crates/sqlbuild-analysis/src/nullability_rules/mod.rs create mode 100644 crates/sqlbuild-analysis/src/nullability_rules/tests.rs create mode 100644 crates/sqlbuild-analysis/src/nullability_rules/tests/test_nullability_rules.rs create mode 100644 crates/sqlbuild-analysis/src/nullability_rules/tests/test_types.rs create mode 100644 crates/sqlbuild-cache/src/startup_inputs/_helpers/mod.rs create mode 100644 crates/sqlbuild-cache/src/startup_inputs/_helpers/pure_paths.rs create mode 100644 crates/sqlbuild-cache/src/startup_inputs/constants.rs create mode 100644 crates/sqlbuild-cache/src/startup_inputs/main/mod.rs create mode 100644 crates/sqlbuild-cache/src/startup_inputs/main/startup_stamps.rs create mode 100644 crates/sqlbuild-cache/src/startup_inputs/mod.rs create mode 100644 crates/sqlbuild-cache/src/startup_inputs/models.rs create mode 100644 crates/sqlbuild-cache/src/startup_inputs/tests.rs create mode 100644 crates/sqlbuild-cache/src/startup_inputs/tests/test_startup_stamps.rs create mode 100644 crates/sqlbuild-cache/src/startup_inputs/tests/test_types.rs create mode 100644 crates/sqlbuild-config/src/toml/main/toml_literal.rs create mode 100644 crates/sqlbuild-config/src/toml/models.rs create mode 100644 crates/sqlbuild-config/src/toml/tests/test_toml_literal.rs create mode 100644 crates/sqlbuild-core/src/dagster_invocation/_helpers/markdown.rs create mode 100644 crates/sqlbuild-core/src/dagster_invocation/_helpers/names.rs create mode 100644 crates/sqlbuild-core/src/dagster_invocation/main/asset_description.rs create mode 100644 crates/sqlbuild-core/src/dagster_invocation/main/dagster_name_segment.rs create mode 100644 crates/sqlbuild-core/src/dagster_invocation/main/dagster_tag_key.rs create mode 100644 crates/sqlbuild-core/src/dagster_invocation/main/fallback_asset_description.rs create mode 100644 crates/sqlbuild-core/src/dagster_invocation/tests/test_translator.rs create mode 100644 crates/sqlbuild-core/src/identifier_text/main/is_lowercase_identifier.rs create mode 100644 crates/sqlbuild-core/src/identifier_text/main/mod.rs create mode 100644 crates/sqlbuild-core/src/identifier_text/mod.rs create mode 100644 crates/sqlbuild-core/src/identifier_text/tests.rs create mode 100644 crates/sqlbuild-core/src/identifier_text/tests/test_lowercase_identifier.rs create mode 100644 crates/sqlbuild-core/src/identifier_text/tests/test_types.rs create mode 100644 crates/sqlbuild-core/src/posix_path/main/posix_escapes_root.rs create mode 100644 crates/sqlbuild-pysource/src/rule_evidence/main/allowed_custom_rule_import.rs create mode 100644 crates/sqlbuild-pysource/src/rule_evidence/main/disallowed_import.rs create mode 100644 crates/sqlbuild-pysource/src/rule_evidence/tests/test_hermeticity.rs create mode 100644 crates/sqlbuild-python/src/bindings/_helpers/authored_values/python_nodes.rs create mode 100644 crates/sqlbuild-python/src/bindings/_helpers/compile_command/startup_inputs.rs create mode 100644 crates/sqlbuild-python/src/bindings/_helpers/dagster_invocation/translator.rs create mode 100644 crates/sqlbuild-python/src/bindings/_helpers/rules/rule_api.rs create mode 100644 crates/sqlbuild-python/src/bindings/_helpers/type_system/nullability.rs create mode 100644 crates/sqlbuild-rules/src/project_tree/_helpers/entries.rs create mode 100644 crates/sqlbuild-rules/src/project_tree/_helpers/files.rs create mode 100644 crates/sqlbuild-rules/src/project_tree/_helpers/mod.rs create mode 100644 crates/sqlbuild-rules/src/project_tree/_helpers/relative_paths.rs create mode 100644 crates/sqlbuild-rules/src/project_tree/constants.rs create mode 100644 crates/sqlbuild-rules/src/project_tree/errors.rs create mode 100644 crates/sqlbuild-rules/src/project_tree/main/mod.rs create mode 100644 crates/sqlbuild-rules/src/project_tree/main/project_tree_entries.rs create mode 100644 crates/sqlbuild-rules/src/project_tree/main/project_tree_file.rs create mode 100644 crates/sqlbuild-rules/src/project_tree/main/tracked_text_suffixes.rs create mode 100644 crates/sqlbuild-rules/src/project_tree/mod.rs create mode 100644 crates/sqlbuild-rules/src/project_tree/models.rs create mode 100644 crates/sqlbuild-rules/src/project_tree/tests.rs create mode 100644 crates/sqlbuild-rules/src/project_tree/tests/helpers.rs create mode 100644 crates/sqlbuild-rules/src/project_tree/tests/test_project_tree.rs create mode 100644 crates/sqlbuild-rules/src/project_tree/tests/test_types.rs delete mode 100644 src/sqlbuild/adapter/type_system/_helpers/inference_rules.py delete mode 100644 src/sqlbuild/rule_engine/_helpers/engine/hermeticity.py create mode 100644 tests/unit/src/sqlbuild/adapter/type_system/test_nullability_rules.py create mode 100644 tests/unit/src/sqlbuild/cli/compile_reuse/_helpers/test_startup_inputs.py create mode 100644 tests/unit/src/sqlbuild/python_nodes/main/test_calculate_retry_delay.py diff --git a/crates/sqlbuild-analysis/src/assembly/analysis_session/_helpers/cte_facts.rs b/crates/sqlbuild-analysis/src/assembly/analysis_session/_helpers/cte_facts.rs index 6d937aa4e6..3fff43f207 100644 --- a/crates/sqlbuild-analysis/src/assembly/analysis_session/_helpers/cte_facts.rs +++ b/crates/sqlbuild-analysis/src/assembly/analysis_session/_helpers/cte_facts.rs @@ -32,6 +32,9 @@ use crate::assembly::analysis_session::constants::{ use crate::assembly::analysis_session::models::NullabilityCallback; use crate::assembly::analysis_session::models::{ColumnFact, LineageRow}; use crate::assembly::analysis_session::types::{Pairs, Shapes}; +use crate::nullability_rules::main::conditional_result_nullability::conditional_result_nullability; +use crate::nullability_rules::main::first_arg_nullability::first_arg_nullability; +use crate::nullability_rules::main::safe_cast_nullability::safe_cast_nullability; use crate::type_system::main::normalize_type::normalize_type; use crate::type_system::models::{NormalizedType, TypeFamily}; @@ -1020,36 +1023,13 @@ fn combined(node_kind: &str, rule: Option<&str>, arguments: &[&'static str]) -> return UNKNOWN_NULLABILITY; } match rule { - Some(FIRST_ARG_RULE) => arguments.first().copied().unwrap_or(UNKNOWN_NULLABILITY), - Some(CONDITIONAL_RESULT_RULE) => conditional_result(arguments), - Some(SAFE_CAST_RULE) => safe_cast(arguments), + Some(FIRST_ARG_RULE) => first_arg_nullability(arguments), + Some(CONDITIONAL_RESULT_RULE) => conditional_result_nullability(arguments), + Some(SAFE_CAST_RULE) => safe_cast_nullability(arguments), _ => UNKNOWN_NULLABILITY, } } -/// Python's `safe_cast_nullability`. -fn safe_cast(arguments: &[&'static str]) -> &'static str { - if arguments.first() == Some(&NULLABLE_NULLABILITY) { - NULLABLE_NULLABILITY - } else { - UNKNOWN_NULLABILITY - } -} - -/// Python's `conditional_result_nullability`. -fn conditional_result(arguments: &[&'static str]) -> &'static str { - let [_, when_true, when_false, ..] = arguments else { - return UNKNOWN_NULLABILITY; - }; - if *when_true == NON_NULL_NULLABILITY && *when_false == NON_NULL_NULLABILITY { - return NON_NULL_NULLABILITY; - } - if *when_true == NULLABLE_NULLABILITY || *when_false == NULLABLE_NULLABILITY { - return NULLABLE_NULLABILITY; - } - UNKNOWN_NULLABILITY -} - /// Python's `_infer_polyglot_column_nullability`. fn column_nullability( column: &Expression, diff --git a/crates/sqlbuild-analysis/src/lib.rs b/crates/sqlbuild-analysis/src/lib.rs index 53e6d5545c..3f2e538f9d 100644 --- a/crates/sqlbuild-analysis/src/lib.rs +++ b/crates/sqlbuild-analysis/src/lib.rs @@ -21,6 +21,7 @@ pub mod lineage; pub mod lineage_command; pub mod lineage_graph; pub mod manifest; +pub mod nullability_rules; pub mod planner_changes; pub mod planner_fixtures; pub mod planner_identity; diff --git a/crates/sqlbuild-analysis/src/nullability_rules/main/conditional_result_nullability.rs b/crates/sqlbuild-analysis/src/nullability_rules/main/conditional_result_nullability.rs new file mode 100644 index 0000000000..67b44465be --- /dev/null +++ b/crates/sqlbuild-analysis/src/nullability_rules/main/conditional_result_nullability.rs @@ -0,0 +1,19 @@ +//! The nullability of an IF/IFF-style conditional from its two result arguments. + +use crate::assembly::analysis_session::constants::{ + NON_NULL_NULLABILITY, NULLABLE_NULLABILITY, UNKNOWN_NULLABILITY, +}; + +/// Non-null when both results are, nullable when either is, otherwise unknown. +pub fn conditional_result_nullability(arguments: &[&str]) -> &'static str { + let [_, when_true, when_false, ..] = arguments else { + return UNKNOWN_NULLABILITY; + }; + if *when_true == NON_NULL_NULLABILITY && *when_false == NON_NULL_NULLABILITY { + return NON_NULL_NULLABILITY; + } + if *when_true == NULLABLE_NULLABILITY || *when_false == NULLABLE_NULLABILITY { + return NULLABLE_NULLABILITY; + } + UNKNOWN_NULLABILITY +} diff --git a/crates/sqlbuild-analysis/src/nullability_rules/main/first_arg_nullability.rs b/crates/sqlbuild-analysis/src/nullability_rules/main/first_arg_nullability.rs new file mode 100644 index 0000000000..f4d72700b4 --- /dev/null +++ b/crates/sqlbuild-analysis/src/nullability_rules/main/first_arg_nullability.rs @@ -0,0 +1,8 @@ +//! The nullability of a null-propagating unary function. + +use crate::assembly::analysis_session::constants::UNKNOWN_NULLABILITY; + +/// The first argument's nullability, or unknown without arguments. +pub fn first_arg_nullability<'a>(arguments: &[&'a str]) -> &'a str { + arguments.first().copied().unwrap_or(UNKNOWN_NULLABILITY) +} diff --git a/crates/sqlbuild-analysis/src/nullability_rules/main/mod.rs b/crates/sqlbuild-analysis/src/nullability_rules/main/mod.rs new file mode 100644 index 0000000000..8efe47be41 --- /dev/null +++ b/crates/sqlbuild-analysis/src/nullability_rules/main/mod.rs @@ -0,0 +1,3 @@ +pub mod conditional_result_nullability; +pub mod first_arg_nullability; +pub mod safe_cast_nullability; diff --git a/crates/sqlbuild-analysis/src/nullability_rules/main/safe_cast_nullability.rs b/crates/sqlbuild-analysis/src/nullability_rules/main/safe_cast_nullability.rs new file mode 100644 index 0000000000..90cf26c4d0 --- /dev/null +++ b/crates/sqlbuild-analysis/src/nullability_rules/main/safe_cast_nullability.rs @@ -0,0 +1,12 @@ +//! The nullability of a SAFE_CAST/TRY_CAST-style conversion. + +use crate::assembly::analysis_session::constants::{NULLABLE_NULLABILITY, UNKNOWN_NULLABILITY}; + +/// Nullable when the converted value is nullable; otherwise unknown, since conversion may fail. +pub fn safe_cast_nullability(arguments: &[&str]) -> &'static str { + if arguments.first() == Some(&NULLABLE_NULLABILITY) { + NULLABLE_NULLABILITY + } else { + UNKNOWN_NULLABILITY + } +} diff --git a/crates/sqlbuild-analysis/src/nullability_rules/mod.rs b/crates/sqlbuild-analysis/src/nullability_rules/mod.rs new file mode 100644 index 0000000000..e29b572755 --- /dev/null +++ b/crates/sqlbuild-analysis/src/nullability_rules/mod.rs @@ -0,0 +1,4 @@ +pub mod main; + +#[cfg(test)] +mod tests; diff --git a/crates/sqlbuild-analysis/src/nullability_rules/tests.rs b/crates/sqlbuild-analysis/src/nullability_rules/tests.rs new file mode 100644 index 0000000000..5c5151c2b0 --- /dev/null +++ b/crates/sqlbuild-analysis/src/nullability_rules/tests.rs @@ -0,0 +1,4 @@ +#[path = "tests/test_nullability_rules.rs"] +mod nullability_rules; +#[path = "tests/test_types.rs"] +mod test_types; diff --git a/crates/sqlbuild-analysis/src/nullability_rules/tests/test_nullability_rules.rs b/crates/sqlbuild-analysis/src/nullability_rules/tests/test_nullability_rules.rs new file mode 100644 index 0000000000..31b8270d7c --- /dev/null +++ b/crates/sqlbuild-analysis/src/nullability_rules/tests/test_nullability_rules.rs @@ -0,0 +1,69 @@ +use crate::nullability_rules::main::conditional_result_nullability::conditional_result_nullability; +use crate::nullability_rules::main::first_arg_nullability::first_arg_nullability; +use crate::nullability_rules::main::safe_cast_nullability::safe_cast_nullability; +use crate::nullability_rules::tests::test_types::NullabilityRuleTestCase; + +#[test] +fn given_argument_nullabilities_when_inferred_then_each_rule_matches_its_contract() { + let test_cases = [ + NullabilityRuleTestCase { + description: "no arguments", + arguments: vec![], + expected_first_arg: "unknown", + expected_conditional: "unknown", + expected_safe_cast: "unknown", + }, + NullabilityRuleTestCase { + description: "a nullable value only", + arguments: vec!["nullable"], + expected_first_arg: "nullable", + expected_conditional: "unknown", + expected_safe_cast: "nullable", + }, + NullabilityRuleTestCase { + description: "two non-null results", + arguments: vec!["unknown", "non_null", "non_null"], + expected_first_arg: "unknown", + expected_conditional: "non_null", + expected_safe_cast: "unknown", + }, + NullabilityRuleTestCase { + description: "one nullable result", + arguments: vec!["non_null", "non_null", "nullable", "nullable"], + expected_first_arg: "non_null", + expected_conditional: "nullable", + expected_safe_cast: "unknown", + }, + NullabilityRuleTestCase { + description: "an unproven result", + arguments: vec!["nullable", "non_null", "unknown"], + expected_first_arg: "nullable", + expected_conditional: "unknown", + expected_safe_cast: "nullable", + }, + NullabilityRuleTestCase { + description: "an unrecognised value", + arguments: vec!["other", "other", "non_null"], + expected_first_arg: "other", + expected_conditional: "unknown", + expected_safe_cast: "unknown", + }, + ]; + for test_case in test_cases { + let actual = ( + first_arg_nullability(&test_case.arguments), + conditional_result_nullability(&test_case.arguments), + safe_cast_nullability(&test_case.arguments), + ); + assert_eq!( + actual, + ( + test_case.expected_first_arg, + test_case.expected_conditional, + test_case.expected_safe_cast + ), + "{}", + test_case.description + ); + } +} diff --git a/crates/sqlbuild-analysis/src/nullability_rules/tests/test_types.rs b/crates/sqlbuild-analysis/src/nullability_rules/tests/test_types.rs new file mode 100644 index 0000000000..11caf17093 --- /dev/null +++ b/crates/sqlbuild-analysis/src/nullability_rules/tests/test_types.rs @@ -0,0 +1,7 @@ +pub(crate) struct NullabilityRuleTestCase { + pub(crate) description: &'static str, + pub(crate) arguments: Vec<&'static str>, + pub(crate) expected_first_arg: &'static str, + pub(crate) expected_conditional: &'static str, + pub(crate) expected_safe_cast: &'static str, +} diff --git a/crates/sqlbuild-cache/src/lib.rs b/crates/sqlbuild-cache/src/lib.rs index c11728d69f..6899e432d6 100644 --- a/crates/sqlbuild-cache/src/lib.rs +++ b/crates/sqlbuild-cache/src/lib.rs @@ -10,4 +10,5 @@ pub mod module_digests; pub mod project_reuse; pub mod project_snapshot; pub mod stage_capture; +pub mod startup_inputs; pub mod store; diff --git a/crates/sqlbuild-cache/src/startup_inputs/_helpers/mod.rs b/crates/sqlbuild-cache/src/startup_inputs/_helpers/mod.rs new file mode 100644 index 0000000000..eb1eb4d0ff --- /dev/null +++ b/crates/sqlbuild-cache/src/startup_inputs/_helpers/mod.rs @@ -0,0 +1 @@ +pub(crate) mod pure_paths; diff --git a/crates/sqlbuild-cache/src/startup_inputs/_helpers/pure_paths.rs b/crates/sqlbuild-cache/src/startup_inputs/_helpers/pure_paths.rs new file mode 100644 index 0000000000..2cc65747f7 --- /dev/null +++ b/crates/sqlbuild-cache/src/startup_inputs/_helpers/pure_paths.rs @@ -0,0 +1,55 @@ +//! `pathlib.PurePosixPath` spelling and ordering over raw path bytes. + +use std::ffi::OsString; +use std::os::unix::ffi::{OsStrExt, OsStringExt}; +use std::path::{Path, PathBuf}; + +use crate::startup_inputs::constants::{CURRENT, SEPARATOR}; + +/// The root a POSIX path keeps: two leading slashes stay two, more collapse to one. +fn root(bytes: &[u8]) -> &[u8] { + match bytes { + [SEPARATOR, SEPARATOR, rest @ ..] if rest.first() != Some(&SEPARATOR) => &bytes[..2], + [SEPARATOR, ..] => &bytes[..1], + _ => &[], + } +} + +/// `str(PurePosixPath(path, *children))`. +pub(crate) fn pure(path: &Path, children: &[&[u8]]) -> PathBuf { + let bytes: &[u8] = path.as_os_str().as_bytes(); + let mut text: Vec = root(bytes).to_vec(); + let parts = std::iter::once(bytes) + .chain(children.iter().copied()) + .flat_map(|segment| segment.split(|byte| *byte == SEPARATOR)) + .filter(|part| !part.is_empty() && *part != CURRENT); + let mut first: bool = true; + for part in parts { + if !first { + text.push(SEPARATOR); + } + text.extend_from_slice(part); + first = false; + } + if text.is_empty() { + text.extend_from_slice(CURRENT); + } + PathBuf::from(OsString::from_vec(text)) +} + +/// `str(Path(path).absolute())` against the working directory `cwd`. +pub(crate) fn absolute(path: &Path, cwd: &Path) -> PathBuf { + if root(path.as_os_str().as_bytes()).is_empty() { + pure(cwd, &[path.as_os_str().as_bytes()]) + } else { + pure(path, &[]) + } +} + +/// The `/`-separated parts `PurePosixPath` orders paths by. +pub(crate) fn order_key(path: &Path) -> Vec<&[u8]> { + path.as_os_str() + .as_bytes() + .split(|byte| *byte == SEPARATOR) + .collect() +} diff --git a/crates/sqlbuild-cache/src/startup_inputs/constants.rs b/crates/sqlbuild-cache/src/startup_inputs/constants.rs new file mode 100644 index 0000000000..20ba4b2d99 --- /dev/null +++ b/crates/sqlbuild-cache/src/startup_inputs/constants.rs @@ -0,0 +1,8 @@ +//! Names the startup stamps read. + +/// The virtual-environment configuration file of an interpreter prefix. +pub(crate) const VENV_CONFIG: &[u8] = b"pyvenv.cfg"; +/// The suffix of the path configuration files `site` processes. +pub(crate) const PATH_FILE_SUFFIX: &[u8] = b".pth"; +pub(crate) const SEPARATOR: u8 = b'/'; +pub(crate) const CURRENT: &[u8] = b"."; diff --git a/crates/sqlbuild-cache/src/startup_inputs/main/mod.rs b/crates/sqlbuild-cache/src/startup_inputs/main/mod.rs new file mode 100644 index 0000000000..44325627ba --- /dev/null +++ b/crates/sqlbuild-cache/src/startup_inputs/main/mod.rs @@ -0,0 +1 @@ +pub mod startup_stamps; diff --git a/crates/sqlbuild-cache/src/startup_inputs/main/startup_stamps.rs b/crates/sqlbuild-cache/src/startup_inputs/main/startup_stamps.rs new file mode 100644 index 0000000000..e7384af4f6 --- /dev/null +++ b/crates/sqlbuild-cache/src/startup_inputs/main/startup_stamps.rs @@ -0,0 +1,66 @@ +//! Stamp every startup directory, its `.pth` files, the interpreter and its venv configuration. + +use std::collections::BTreeSet; +use std::ffi::OsString; +use std::io; +use std::os::unix::ffi::OsStrExt; +use std::os::unix::fs::MetadataExt; +use std::path::{Path, PathBuf}; + +use crate::startup_inputs::_helpers::pure_paths::{absolute, order_key, pure}; +use crate::startup_inputs::constants::{PATH_FILE_SUFFIX, VENV_CONFIG}; +use crate::startup_inputs::models::{StartupLocations, StartupStamp}; + +/// Sorted stamps of the startup paths; missing paths are unstamped and other stat failures fail. +pub fn startup_stamps( + locations: &StartupLocations, + cwd: &Path, +) -> Result, (PathBuf, io::Error)> { + let directories: BTreeSet = locations + .directories + .iter() + .map(|directory| absolute(directory, cwd).into_os_string()) + .collect(); + let mut paths: BTreeSet = directories.clone(); + paths.extend( + locations + .files + .iter() + .map(|file| pure(file, &[]).into_os_string()), + ); + paths.extend( + locations + .prefixes + .iter() + .map(|prefix| pure(prefix, &[VENV_CONFIG]).into_os_string()), + ); + for directory in &directories { + let directory: &Path = Path::new(directory); + let Ok(listing) = std::fs::read_dir(directory) else { + continue; + }; + for entry in listing.flatten() { + if entry.file_name().as_bytes().ends_with(PATH_FILE_SUFFIX) { + paths.insert(pure(directory, &[entry.file_name().as_bytes()]).into_os_string()); + } + } + } + let mut ordered: Vec = paths.into_iter().map(PathBuf::from).collect(); + ordered.sort_by(|left, right| order_key(left).cmp(&order_key(right))); + ordered + .into_iter() + .map(|path| match std::fs::metadata(&path) { + Ok(metadata) => Ok(StartupStamp { + modified_ns: Some(metadata.mtime() * 1_000_000_000 + metadata.mtime_nsec()), + size: Some(metadata.len()), + path, + }), + Err(error) if error.kind() == io::ErrorKind::NotFound => Ok(StartupStamp { + path, + modified_ns: None, + size: None, + }), + Err(error) => Err((path, error)), + }) + .collect() +} diff --git a/crates/sqlbuild-cache/src/startup_inputs/mod.rs b/crates/sqlbuild-cache/src/startup_inputs/mod.rs new file mode 100644 index 0000000000..ab8bbc79c9 --- /dev/null +++ b/crates/sqlbuild-cache/src/startup_inputs/mod.rs @@ -0,0 +1,9 @@ +//! Interpreter and site-path stamps that must be unchanged before Python startup is skipped. + +pub(crate) mod _helpers; +pub(crate) mod constants; +pub mod main; +pub mod models; + +#[cfg(test)] +mod tests; diff --git a/crates/sqlbuild-cache/src/startup_inputs/models.rs b/crates/sqlbuild-cache/src/startup_inputs/models.rs new file mode 100644 index 0000000000..e14607c8d4 --- /dev/null +++ b/crates/sqlbuild-cache/src/startup_inputs/models.rs @@ -0,0 +1,22 @@ +//! One startup input and its file stamp. + +use std::path::PathBuf; + +/// The interpreter locations the stamps cover. +#[derive(Clone, Debug, Default, PartialEq, Eq)] +pub struct StartupLocations { + /// Import-path and site-packages directories, made absolute. + pub directories: Vec, + /// Interpreter files, as given. + pub files: Vec, + /// Interpreter prefixes whose `pyvenv.cfg` is stamped. + pub prefixes: Vec, +} + +/// A path with its modification time in nanoseconds and size, both absent when it is missing. +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct StartupStamp { + pub path: PathBuf, + pub modified_ns: Option, + pub size: Option, +} diff --git a/crates/sqlbuild-cache/src/startup_inputs/tests.rs b/crates/sqlbuild-cache/src/startup_inputs/tests.rs new file mode 100644 index 0000000000..7d8de14dac --- /dev/null +++ b/crates/sqlbuild-cache/src/startup_inputs/tests.rs @@ -0,0 +1,4 @@ +#[path = "tests/test_startup_stamps.rs"] +mod startup_stamps; +#[path = "tests/test_types.rs"] +mod test_types; diff --git a/crates/sqlbuild-cache/src/startup_inputs/tests/test_startup_stamps.rs b/crates/sqlbuild-cache/src/startup_inputs/tests/test_startup_stamps.rs new file mode 100644 index 0000000000..690d1ce3b5 --- /dev/null +++ b/crates/sqlbuild-cache/src/startup_inputs/tests/test_startup_stamps.rs @@ -0,0 +1,66 @@ +use std::path::{Path, PathBuf}; + +use crate::startup_inputs::main::startup_stamps::startup_stamps; +use crate::startup_inputs::models::{StartupLocations, StartupStamp}; +use crate::startup_inputs::tests::test_types::StartupStampsTestCase; + +#[test] +fn given_startup_locations_when_stamped_then_paths_are_absolute_listed_and_sorted_by_parts() { + let test_cases = [StartupStampsTestCase { + description: "relative entries, path files, missing entries and the venv configuration", + directories: vec!["site", "./a//", "a-b", "a/b", "missing"], + expected_stamps: vec![ + ("a", true), + ("a/b", true), + ("a/q.pth", true), + ("a-b", true), + ("a-b/x.pth", true), + ("missing", false), + ("pyvenv.cfg", false), + ("site", true), + ("site/.pth", true), + ("site/one.pth", true), + ], + }]; + for test_case in test_cases { + let directory = tempfile::tempdir().expect("temporary directory"); + let root: PathBuf = std::fs::canonicalize(directory.path()).expect("resolved"); + for folder in ["a", "a-b", "a/b", "site"] { + std::fs::create_dir_all(root.join(folder)).expect("folder"); + } + for file in [ + "site/one.pth", + "site/.pth", + "site/two.pth.bak", + "a/q.pth", + "a-b/x.pth", + ] { + std::fs::write(root.join(file), "x\n").expect("file"); + } + let locations = StartupLocations { + directories: test_case.directories.iter().map(PathBuf::from).collect(), + files: vec![root.join("site/one.pth")], + prefixes: vec![root.clone()], + }; + let stamps: Vec = startup_stamps(&locations, &root).expect("stamps"); + let listed: Vec<(String, bool)> = stamps + .iter() + .map(|stamp| { + let relative: &Path = stamp.path.strip_prefix(&root).unwrap_or(&stamp.path); + ( + relative.to_string_lossy().into_owned(), + stamp.size.is_some(), + ) + }) + .collect(); + let listed: Vec<(&str, bool)> = listed + .iter() + .map(|(path, present)| (path.as_str(), *present)) + .collect(); + assert_eq!( + listed, test_case.expected_stamps, + "{}", + test_case.description + ); + } +} diff --git a/crates/sqlbuild-cache/src/startup_inputs/tests/test_types.rs b/crates/sqlbuild-cache/src/startup_inputs/tests/test_types.rs new file mode 100644 index 0000000000..57ad24a56c --- /dev/null +++ b/crates/sqlbuild-cache/src/startup_inputs/tests/test_types.rs @@ -0,0 +1,7 @@ +pub(crate) struct StartupStampsTestCase { + pub(crate) description: &'static str, + /// Import-path entries, relative to the temporary root unless absolute-looking. + pub(crate) directories: Vec<&'static str>, + /// Stamped paths under the root and whether each exists, in order. + pub(crate) expected_stamps: Vec<(&'static str, bool)>, +} diff --git a/crates/sqlbuild-config/src/project/_helpers/help.rs b/crates/sqlbuild-config/src/project/_helpers/help.rs index 9cdb500837..027276dd83 100644 --- a/crates/sqlbuild-config/src/project/_helpers/help.rs +++ b/crates/sqlbuild-config/src/project/_helpers/help.rs @@ -8,6 +8,8 @@ use crate::project::constants::{ REPLAY_ON_CHANGE_VALUES, SETTING_SNIPPET_INDENT, }; use crate::project::errors::ConfigFailure; +use crate::toml::main::toml_literal::toml_literal; +use crate::toml::models::TomlLiteral; /// A setting value as TOML shows it. #[derive(Clone, Copy, Debug, PartialEq, Eq)] @@ -28,14 +30,11 @@ pub(crate) struct Setting<'a> { } fn toml_value(value: SettingValue<'_>) -> String { - match value { - SettingValue::Bool(true) => "true".to_owned(), - SettingValue::Bool(false) => "false".to_owned(), - SettingValue::Integer(digits) => digits.to_owned(), - SettingValue::Text(text) => { - format!("\"{}\"", text.replace('\\', "\\\\").replace('"', "\\\"")) - } - } + toml_literal(&match value { + SettingValue::Bool(flag) => TomlLiteral::Bool(flag), + SettingValue::Integer(digits) => TomlLiteral::Integer(digits.to_owned()), + SettingValue::Text(text) => TomlLiteral::Text(text.to_owned()), + }) } /// `setting_help`: `, set this in :` and the TOML section and key. diff --git a/crates/sqlbuild-config/src/toml/main/mod.rs b/crates/sqlbuild-config/src/toml/main/mod.rs index 04683b5110..f50eb8ba0e 100644 --- a/crates/sqlbuild-config/src/toml/main/mod.rs +++ b/crates/sqlbuild-config/src/toml/main/mod.rs @@ -1 +1,2 @@ pub mod load_toml; +pub mod toml_literal; diff --git a/crates/sqlbuild-config/src/toml/main/toml_literal.rs b/crates/sqlbuild-config/src/toml/main/toml_literal.rs new file mode 100644 index 0000000000..e04b5835c1 --- /dev/null +++ b/crates/sqlbuild-config/src/toml/main/toml_literal.rs @@ -0,0 +1,20 @@ +//! A TOML literal as SQLBuild writes it into configuration text and help. + +use crate::toml::models::TomlLiteral; + +/// `true`/`false`, the integer digits, a `"`-quoted string escaping `\` and `"`, or `[a, b]`. +#[must_use] +pub fn toml_literal(value: &TomlLiteral) -> String { + match value { + TomlLiteral::Bool(true) => "true".to_owned(), + TomlLiteral::Bool(false) => "false".to_owned(), + TomlLiteral::Integer(digits) => digits.clone(), + TomlLiteral::Text(text) => { + format!("\"{}\"", text.replace('\\', "\\\\").replace('"', "\\\"")) + } + TomlLiteral::Array(items) => { + let items: Vec = items.iter().map(toml_literal).collect(); + format!("[{}]", items.join(", ")) + } + } +} diff --git a/crates/sqlbuild-config/src/toml/mod.rs b/crates/sqlbuild-config/src/toml/mod.rs index 37f9081f4f..1e64f3d116 100644 --- a/crates/sqlbuild-config/src/toml/mod.rs +++ b/crates/sqlbuild-config/src/toml/mod.rs @@ -1,6 +1,7 @@ pub(crate) mod _helpers; pub(crate) mod constants; pub mod main; +pub mod models; #[cfg(test)] mod tests; diff --git a/crates/sqlbuild-config/src/toml/models.rs b/crates/sqlbuild-config/src/toml/models.rs new file mode 100644 index 0000000000..8273c88fe3 --- /dev/null +++ b/crates/sqlbuild-config/src/toml/models.rs @@ -0,0 +1,11 @@ +//! TOML values as SQLBuild writes them into configuration text. + +/// A boolean, integer, string or array literal. +#[derive(Clone, Debug, PartialEq, Eq)] +pub enum TomlLiteral { + Bool(bool), + /// An integer, as its decimal text. + Integer(String), + Text(String), + Array(Vec), +} diff --git a/crates/sqlbuild-config/src/toml/tests.rs b/crates/sqlbuild-config/src/toml/tests.rs index 55b6a54c6d..3722e7e455 100644 --- a/crates/sqlbuild-config/src/toml/tests.rs +++ b/crates/sqlbuild-config/src/toml/tests.rs @@ -4,3 +4,5 @@ mod helpers; mod load_toml; #[path = "tests/test_types.rs"] mod test_types; +#[path = "tests/test_toml_literal.rs"] +mod toml_literal; diff --git a/crates/sqlbuild-config/src/toml/tests/test_toml_literal.rs b/crates/sqlbuild-config/src/toml/tests/test_toml_literal.rs new file mode 100644 index 0000000000..b4cdc468e5 --- /dev/null +++ b/crates/sqlbuild-config/src/toml/tests/test_toml_literal.rs @@ -0,0 +1,50 @@ +use crate::toml::main::toml_literal::toml_literal; +use crate::toml::models::TomlLiteral; +use crate::toml::tests::test_types::TomlLiteralTestCase; + +#[test] +fn given_literals_when_written_then_toml_text_matches_the_rule_harness_encoding() { + let test_cases = [ + TomlLiteralTestCase { + description: "a boolean", + value: TomlLiteral::Bool(true), + expected_text: "true", + }, + TomlLiteralTestCase { + description: "an integer keeps its digits", + value: TomlLiteral::Integer("-5".to_owned()), + expected_text: "-5", + }, + TomlLiteralTestCase { + description: "a plain string", + value: TomlLiteral::Text("plain".to_owned()), + expected_text: "\"plain\"", + }, + TomlLiteralTestCase { + description: "quotes and backslashes are escaped", + value: TomlLiteral::Text("a\"b\\c".to_owned()), + expected_text: "\"a\\\"b\\\\c\"", + }, + TomlLiteralTestCase { + description: "an empty array", + value: TomlLiteral::Array(Vec::new()), + expected_text: "[]", + }, + TomlLiteralTestCase { + description: "nested arrays", + value: TomlLiteral::Array(vec![ + TomlLiteral::Integer("1".to_owned()), + TomlLiteral::Array(vec![TomlLiteral::Bool(false)]), + ]), + expected_text: "[1, [false]]", + }, + ]; + for test_case in test_cases { + assert_eq!( + toml_literal(&test_case.value), + test_case.expected_text, + "{}", + test_case.description + ); + } +} diff --git a/crates/sqlbuild-config/src/toml/tests/test_types.rs b/crates/sqlbuild-config/src/toml/tests/test_types.rs index c33becd42d..fab0f70cdb 100644 --- a/crates/sqlbuild-config/src/toml/tests/test_types.rs +++ b/crates/sqlbuild-config/src/toml/tests/test_types.rs @@ -24,3 +24,9 @@ pub(super) struct DeepTomlTestCase { pub(super) text: String, pub(super) expected_value: ConfigValue, } + +pub(super) struct TomlLiteralTestCase { + pub(super) description: &'static str, + pub(super) value: crate::toml::models::TomlLiteral, + pub(super) expected_text: &'static str, +} diff --git a/crates/sqlbuild-core/src/dagster_invocation/_helpers/markdown.rs b/crates/sqlbuild-core/src/dagster_invocation/_helpers/markdown.rs new file mode 100644 index 0000000000..34809b51a5 --- /dev/null +++ b/crates/sqlbuild-core/src/dagster_invocation/_helpers/markdown.rs @@ -0,0 +1,22 @@ +//! A Markdown code span that shows any text verbatim. + +use crate::dagster_invocation::constants::{CODE_SPAN_MARK, CODE_SPAN_PADDED_EDGES}; + +/// `value` fenced by one more backtick than its longest run, padded at a backtick or space edge. +pub(crate) fn markdown_code_span(value: &str) -> String { + let mut longest: usize = 0; + let mut current: usize = 0; + for character in value.chars() { + current = if character == CODE_SPAN_MARK { + current + 1 + } else { + 0 + }; + longest = longest.max(current); + } + let delimiter: String = CODE_SPAN_MARK.to_string().repeat(longest + 1); + let padded: bool = + value.starts_with(CODE_SPAN_PADDED_EDGES) || value.ends_with(CODE_SPAN_PADDED_EDGES); + let padding: &str = if padded { " " } else { "" }; + format!("{delimiter}{padding}{value}{padding}{delimiter}") +} diff --git a/crates/sqlbuild-core/src/dagster_invocation/_helpers/mod.rs b/crates/sqlbuild-core/src/dagster_invocation/_helpers/mod.rs index 07ebd31772..c23e22da7f 100644 --- a/crates/sqlbuild-core/src/dagster_invocation/_helpers/mod.rs +++ b/crates/sqlbuild-core/src/dagster_invocation/_helpers/mod.rs @@ -1 +1,3 @@ pub(crate) mod graph; +pub(crate) mod markdown; +pub(crate) mod names; diff --git a/crates/sqlbuild-core/src/dagster_invocation/_helpers/names.rs b/crates/sqlbuild-core/src/dagster_invocation/_helpers/names.rs new file mode 100644 index 0000000000..a4c719165c --- /dev/null +++ b/crates/sqlbuild-core/src/dagster_invocation/_helpers/names.rs @@ -0,0 +1,19 @@ +//! Runs of rejected characters collapsed to one separator. + +use crate::dagster_invocation::constants::NAME_SEPARATOR; + +/// `value` with every maximal run of characters `kept` rejects replaced by one `_`. +pub(crate) fn replace_runs(value: &str, kept: impl Fn(char) -> bool) -> String { + let mut replaced: String = String::with_capacity(value.len()); + let mut in_run: bool = false; + for character in value.chars() { + if kept(character) { + replaced.push(character); + in_run = false; + } else if !in_run { + replaced.push(NAME_SEPARATOR); + in_run = true; + } + } + replaced +} diff --git a/crates/sqlbuild-core/src/dagster_invocation/constants.rs b/crates/sqlbuild-core/src/dagster_invocation/constants.rs index ede32be2a3..ba8a89311c 100644 --- a/crates/sqlbuild-core/src/dagster_invocation/constants.rs +++ b/crates/sqlbuild-core/src/dagster_invocation/constants.rs @@ -21,3 +21,15 @@ pub const DEFAULT_CHECK_NAME: &str = "check"; pub const MAX_DIAGNOSTIC_CHARS: usize = 4_000; /// Indentation of SQL in failure text. pub const SQL_INDENT: &str = " "; +/// Separator replacing characters a Dagster tag key or name segment cannot hold. +pub const NAME_SEPARATOR: char = '_'; +/// ASCII punctuation a Dagster tag key keeps besides letters, digits and `_`. +pub const TAG_KEY_PUNCTUATION: [char; 3] = ['.', '-', '/']; +/// The Markdown code-span delimiter. +pub const CODE_SPAN_MARK: char = '`'; +/// Edge characters that need padding inside a Markdown code span. +pub(crate) const CODE_SPAN_PADDED_EDGES: [char; 2] = [CODE_SPAN_MARK, ' ']; +/// The default translator's description kind of a node without one. +pub const DEFAULT_ASSET_KIND: &str = "asset"; +/// The default translator's description name of a node without one. +pub const DEFAULT_ASSET_NAME: &str = "unknown"; diff --git a/crates/sqlbuild-core/src/dagster_invocation/main/asset_description.rs b/crates/sqlbuild-core/src/dagster_invocation/main/asset_description.rs new file mode 100644 index 0000000000..ab85c819be --- /dev/null +++ b/crates/sqlbuild-core/src/dagster_invocation/main/asset_description.rs @@ -0,0 +1,19 @@ +//! The default Dagster description: the node's text followed by its SQL or source file. + +use crate::dagster_invocation::_helpers::markdown::markdown_code_span; +use crate::text::main::python_strip::python_strip; + +/// `description` followed by its stripped SQL block, or else by its source file code span. +#[must_use] +pub fn asset_description(description: &str, sql: Option<&str>, path: Option<&str>) -> String { + if let Some(sql) = sql.map(python_strip).filter(|sql| !sql.is_empty()) { + return format!("{description}\n\n**SQLBuild SQL:**\n```sql\n{sql}\n```"); + } + if let Some(path) = path.filter(|path| !path.is_empty()) { + return format!( + "{description}\n\n**Source file:** {}", + markdown_code_span(path) + ); + } + description.to_owned() +} diff --git a/crates/sqlbuild-core/src/dagster_invocation/main/dagster_name_segment.rs b/crates/sqlbuild-core/src/dagster_invocation/main/dagster_name_segment.rs new file mode 100644 index 0000000000..737e54cf01 --- /dev/null +++ b/crates/sqlbuild-core/src/dagster_invocation/main/dagster_name_segment.rs @@ -0,0 +1,18 @@ +//! The default Dagster translator's check-name segment. + +use crate::dagster_invocation::_helpers::names::replace_runs; +use crate::dagster_invocation::constants::{DEFAULT_CHECK_NAME, NAME_SEPARATOR}; + +/// `re.sub(r"[^A-Za-z0-9_]+", "_", value).strip("_") or "check"`. +#[must_use] +pub fn dagster_name_segment(value: &str) -> String { + let replaced: String = replace_runs(value, |character| { + character.is_ascii_alphanumeric() || character == NAME_SEPARATOR + }); + let trimmed: &str = replaced.trim_matches(NAME_SEPARATOR); + if trimmed.is_empty() { + DEFAULT_CHECK_NAME.to_owned() + } else { + trimmed.to_owned() + } +} diff --git a/crates/sqlbuild-core/src/dagster_invocation/main/dagster_tag_key.rs b/crates/sqlbuild-core/src/dagster_invocation/main/dagster_tag_key.rs new file mode 100644 index 0000000000..2b54e34160 --- /dev/null +++ b/crates/sqlbuild-core/src/dagster_invocation/main/dagster_tag_key.rs @@ -0,0 +1,16 @@ +//! The default Dagster translator's tag key for one SQLBuild tag. + +use crate::dagster_invocation::_helpers::names::replace_runs; +use crate::dagster_invocation::constants::{NAME_SEPARATOR, TAG_KEY_PUNCTUATION}; + +/// `re.sub(r"[^A-Za-z0-9_.\-/]+", "_", value).strip("_")`. +#[must_use] +pub fn dagster_tag_key(value: &str) -> String { + replace_runs(value, |character| { + character.is_ascii_alphanumeric() + || character == NAME_SEPARATOR + || TAG_KEY_PUNCTUATION.contains(&character) + }) + .trim_matches(NAME_SEPARATOR) + .to_owned() +} diff --git a/crates/sqlbuild-core/src/dagster_invocation/main/fallback_asset_description.rs b/crates/sqlbuild-core/src/dagster_invocation/main/fallback_asset_description.rs new file mode 100644 index 0000000000..a0ea7f4cc4 --- /dev/null +++ b/crates/sqlbuild-core/src/dagster_invocation/main/fallback_asset_description.rs @@ -0,0 +1,23 @@ +//! The default Dagster description of a node that has none. + +/// `SQLBuild ``[ materialized as ``][ at ``].`, kind `_` as spaces. +#[must_use] +pub fn fallback_asset_description( + kind: &str, + name: &str, + qualified_name: &str, + materialization: Option<&str>, +) -> String { + let kind: String = kind.replace('_', " "); + let relation: String = if qualified_name.is_empty() { + String::new() + } else { + format!(" at `{qualified_name}`") + }; + match materialization { + Some(materialization) => { + format!("SQLBuild {kind} `{name}` materialized as `{materialization}`{relation}.") + } + None => format!("SQLBuild {kind} `{name}`{relation}."), + } +} diff --git a/crates/sqlbuild-core/src/dagster_invocation/main/mod.rs b/crates/sqlbuild-core/src/dagster_invocation/main/mod.rs index 38e4f2ce5e..4c3765b5a1 100644 --- a/crates/sqlbuild-core/src/dagster_invocation/main/mod.rs +++ b/crates/sqlbuild-core/src/dagster_invocation/main/mod.rs @@ -1,6 +1,10 @@ +pub mod asset_description; pub mod asset_failure_text; pub mod caller_json_output_path; +pub mod dagster_name_segment; +pub mod dagster_tag_key; pub mod diagnostic_tail; +pub mod fallback_asset_description; pub mod has_explicit_scenario_selector; pub mod json_output_requested; pub mod normalize_check_name; diff --git a/crates/sqlbuild-core/src/dagster_invocation/tests.rs b/crates/sqlbuild-core/src/dagster_invocation/tests.rs index 969cb00f66..d2b5a423e7 100644 --- a/crates/sqlbuild-core/src/dagster_invocation/tests.rs +++ b/crates/sqlbuild-core/src/dagster_invocation/tests.rs @@ -4,5 +4,7 @@ mod helpers; mod test_arguments; #[path = "tests/test_texts.rs"] mod test_texts; +#[path = "tests/test_translator.rs"] +mod test_translator; #[path = "tests/test_types.rs"] mod test_types; diff --git a/crates/sqlbuild-core/src/dagster_invocation/tests/test_translator.rs b/crates/sqlbuild-core/src/dagster_invocation/tests/test_translator.rs new file mode 100644 index 0000000000..4f5e6c003b --- /dev/null +++ b/crates/sqlbuild-core/src/dagster_invocation/tests/test_translator.rs @@ -0,0 +1,143 @@ +//! The default Dagster translator's names and descriptions match its Python spelling. + +use crate::dagster_invocation::main::asset_description::asset_description; +use crate::dagster_invocation::main::dagster_name_segment::dagster_name_segment; +use crate::dagster_invocation::main::dagster_tag_key::dagster_tag_key; +use crate::dagster_invocation::main::fallback_asset_description::fallback_asset_description; +use crate::dagster_invocation::tests::test_types::{ + DescriptionTestCase, FallbackDescriptionTestCase, NameNormalizationTestCase, +}; + +#[test] +fn given_tags_and_names_when_normalized_then_rejected_runs_become_one_separator() { + let test_cases = [ + NameNormalizationTestCase { + description: "a plain name", + value: "pii", + expected_tag_key: "pii", + expected_segment: "pii", + }, + NameNormalizationTestCase { + description: "punctuation runs collapse", + value: "a b!c", + expected_tag_key: "a_b_c", + expected_segment: "a_b_c", + }, + NameNormalizationTestCase { + description: "outer separators are trimmed", + value: "__x__", + expected_tag_key: "x", + expected_segment: "x", + }, + NameNormalizationTestCase { + description: "non-ASCII letters are rejected", + value: "\u{65e5}\u{672c}", + expected_tag_key: "", + expected_segment: "check", + }, + NameNormalizationTestCase { + description: "tag keys keep dots, dashes and slashes", + value: "ok/x.y-z", + expected_tag_key: "ok/x.y-z", + expected_segment: "ok_x_y_z", + }, + ]; + for test_case in test_cases { + assert_eq!( + ( + dagster_tag_key(test_case.value), + dagster_name_segment(test_case.value) + ), + ( + test_case.expected_tag_key.to_owned(), + test_case.expected_segment.to_owned() + ), + "{}", + test_case.description + ); + } +} + +#[test] +fn given_node_text_when_described_then_sql_wins_over_the_source_file() { + let test_cases = [ + DescriptionTestCase { + description: "stripped SQL is fenced", + text: "D", + sql: Some(" select 1 "), + path: Some("m.sql"), + expected_description: "D\n\n**SQLBuild SQL:**\n```sql\nselect 1\n```", + }, + DescriptionTestCase { + description: "blank SQL falls back to the source file", + text: "D", + sql: Some(" "), + path: Some("m.sql"), + expected_description: "D\n\n**Source file:** `m.sql`", + }, + DescriptionTestCase { + description: "backticks in the path widen the fence", + text: "D", + sql: None, + path: Some("a`b`` c.py"), + expected_description: "D\n\n**Source file:** ```a`b`` c.py```", + }, + DescriptionTestCase { + description: "an edge backtick pads the span", + text: "D", + sql: None, + path: Some("`start"), + expected_description: "D\n\n**Source file:** `` `start ``", + }, + DescriptionTestCase { + description: "no SQL or path keeps the text", + text: "D", + sql: None, + path: Some(""), + expected_description: "D", + }, + ]; + for test_case in test_cases { + assert_eq!( + asset_description(test_case.text, test_case.sql, test_case.path), + test_case.expected_description, + "{}", + test_case.description + ); + } +} + +#[test] +fn given_undocumented_node_when_described_then_kind_name_and_relation_are_named() { + let test_cases = [ + FallbackDescriptionTestCase { + description: "a materialized relation", + kind: "python_asset", + name: "o", + qualified_name: "db.o", + materialization: Some("table"), + expected_description: "SQLBuild python asset `o` materialized as `table` at `db.o`.", + }, + FallbackDescriptionTestCase { + description: "no relation or materialization", + kind: "seed", + name: "s", + qualified_name: "", + materialization: None, + expected_description: "SQLBuild seed `s`.", + }, + ]; + for test_case in test_cases { + assert_eq!( + fallback_asset_description( + test_case.kind, + test_case.name, + test_case.qualified_name, + test_case.materialization + ), + test_case.expected_description, + "{}", + test_case.description + ); + } +} diff --git a/crates/sqlbuild-core/src/dagster_invocation/tests/test_types.rs b/crates/sqlbuild-core/src/dagster_invocation/tests/test_types.rs index 1f7b24722c..e63e23e7d4 100644 --- a/crates/sqlbuild-core/src/dagster_invocation/tests/test_types.rs +++ b/crates/sqlbuild-core/src/dagster_invocation/tests/test_types.rs @@ -35,3 +35,27 @@ pub(super) struct FailureTextTestCase { pub failure: AssetFailure, pub expected_text: &'static str, } + +pub(super) struct NameNormalizationTestCase { + pub description: &'static str, + pub value: &'static str, + pub expected_tag_key: &'static str, + pub expected_segment: &'static str, +} + +pub(super) struct DescriptionTestCase { + pub description: &'static str, + pub text: &'static str, + pub sql: Option<&'static str>, + pub path: Option<&'static str>, + pub expected_description: &'static str, +} + +pub(super) struct FallbackDescriptionTestCase { + pub description: &'static str, + pub kind: &'static str, + pub name: &'static str, + pub qualified_name: &'static str, + pub materialization: Option<&'static str>, + pub expected_description: &'static str, +} diff --git a/crates/sqlbuild-core/src/identifier_text/main/is_lowercase_identifier.rs b/crates/sqlbuild-core/src/identifier_text/main/is_lowercase_identifier.rs new file mode 100644 index 0000000000..6dabcf061c --- /dev/null +++ b/crates/sqlbuild-core/src/identifier_text/main/is_lowercase_identifier.rs @@ -0,0 +1,9 @@ +//! Lowercase ASCII identifiers: `re.fullmatch(r"^[a-z][a-z0-9_]*$", value)`. + +/// Whether `value` is a lowercase ASCII letter followed by lowercase letters, digits or `_`. +#[must_use] +pub fn is_lowercase_identifier(value: &str) -> bool { + let mut bytes = value.bytes(); + bytes.next().is_some_and(|byte| byte.is_ascii_lowercase()) + && bytes.all(|byte| byte.is_ascii_lowercase() || byte.is_ascii_digit() || byte == b'_') +} diff --git a/crates/sqlbuild-core/src/identifier_text/main/mod.rs b/crates/sqlbuild-core/src/identifier_text/main/mod.rs new file mode 100644 index 0000000000..a29dc793b1 --- /dev/null +++ b/crates/sqlbuild-core/src/identifier_text/main/mod.rs @@ -0,0 +1 @@ +pub mod is_lowercase_identifier; diff --git a/crates/sqlbuild-core/src/identifier_text/mod.rs b/crates/sqlbuild-core/src/identifier_text/mod.rs new file mode 100644 index 0000000000..5db00853a7 --- /dev/null +++ b/crates/sqlbuild-core/src/identifier_text/mod.rs @@ -0,0 +1,6 @@ +//! Identifier spellings authored names must follow. + +pub mod main; + +#[cfg(test)] +mod tests; diff --git a/crates/sqlbuild-core/src/identifier_text/tests.rs b/crates/sqlbuild-core/src/identifier_text/tests.rs new file mode 100644 index 0000000000..149a53f52e --- /dev/null +++ b/crates/sqlbuild-core/src/identifier_text/tests.rs @@ -0,0 +1,4 @@ +#[path = "tests/test_lowercase_identifier.rs"] +mod lowercase_identifier; +#[path = "tests/test_types.rs"] +mod test_types; diff --git a/crates/sqlbuild-core/src/identifier_text/tests/test_lowercase_identifier.rs b/crates/sqlbuild-core/src/identifier_text/tests/test_lowercase_identifier.rs new file mode 100644 index 0000000000..b06b3563da --- /dev/null +++ b/crates/sqlbuild-core/src/identifier_text/tests/test_lowercase_identifier.rs @@ -0,0 +1,76 @@ +use crate::identifier_text::main::is_lowercase_identifier::is_lowercase_identifier; +use crate::identifier_text::tests::test_types::LowercaseIdentifierTestCase; + +#[test] +fn given_names_when_checked_then_only_lowercase_ascii_identifiers_pass() { + let test_cases = [ + LowercaseIdentifierTestCase { + description: "a plain name", + value: "orders", + expected_valid: true, + }, + LowercaseIdentifierTestCase { + description: "one letter", + value: "a", + expected_valid: true, + }, + LowercaseIdentifierTestCase { + description: "digits and underscores", + value: "a1_b", + expected_valid: true, + }, + LowercaseIdentifierTestCase { + description: "a trailing underscore", + value: "abc_", + expected_valid: true, + }, + LowercaseIdentifierTestCase { + description: "an empty name", + value: "", + expected_valid: false, + }, + LowercaseIdentifierTestCase { + description: "an uppercase letter", + value: "A", + expected_valid: false, + }, + LowercaseIdentifierTestCase { + description: "a leading digit", + value: "1a", + expected_valid: false, + }, + LowercaseIdentifierTestCase { + description: "a leading underscore", + value: "_a", + expected_valid: false, + }, + LowercaseIdentifierTestCase { + description: "a hyphen", + value: "a-b", + expected_valid: false, + }, + LowercaseIdentifierTestCase { + description: "a trailing newline", + value: "ab\n", + expected_valid: false, + }, + LowercaseIdentifierTestCase { + description: "an accented letter", + value: "\u{e9}", + expected_valid: false, + }, + LowercaseIdentifierTestCase { + description: "a dotless i", + value: "a\u{131}", + expected_valid: false, + }, + ]; + for test_case in test_cases { + assert_eq!( + is_lowercase_identifier(test_case.value), + test_case.expected_valid, + "{}", + test_case.description + ); + } +} diff --git a/crates/sqlbuild-core/src/identifier_text/tests/test_types.rs b/crates/sqlbuild-core/src/identifier_text/tests/test_types.rs new file mode 100644 index 0000000000..8b57aa418f --- /dev/null +++ b/crates/sqlbuild-core/src/identifier_text/tests/test_types.rs @@ -0,0 +1,5 @@ +pub(crate) struct LowercaseIdentifierTestCase { + pub(crate) description: &'static str, + pub(crate) value: &'static str, + pub(crate) expected_valid: bool, +} diff --git a/crates/sqlbuild-core/src/lib.rs b/crates/sqlbuild-core/src/lib.rs index 1fcd518fa3..99383f712f 100644 --- a/crates/sqlbuild-core/src/lib.rs +++ b/crates/sqlbuild-core/src/lib.rs @@ -30,6 +30,7 @@ pub mod dlt_destination; pub mod event_export; pub mod format_command; pub mod freshness_text; +pub mod identifier_text; pub mod ingestr_command; pub mod init_command; pub mod integration_results; diff --git a/crates/sqlbuild-core/src/posix_path/constants.rs b/crates/sqlbuild-core/src/posix_path/constants.rs index 9216305dc5..e58ea77ccb 100644 --- a/crates/sqlbuild-core/src/posix_path/constants.rs +++ b/crates/sqlbuild-core/src/posix_path/constants.rs @@ -5,4 +5,5 @@ pub(crate) const ROOT: &str = "/"; /// POSIX keeps exactly two leading slashes as their own root. pub(crate) const DOUBLE_ROOT: &str = "//"; pub(crate) const CURRENT: &str = "."; +pub(crate) const PARENT: &str = ".."; pub(crate) const SUFFIX_MARK: char = '.'; diff --git a/crates/sqlbuild-core/src/posix_path/main/mod.rs b/crates/sqlbuild-core/src/posix_path/main/mod.rs index e4ab185549..cbd3f33313 100644 --- a/crates/sqlbuild-core/src/posix_path/main/mod.rs +++ b/crates/sqlbuild-core/src/posix_path/main/mod.rs @@ -1,3 +1,4 @@ +pub mod posix_escapes_root; pub mod posix_join; pub mod posix_name; pub mod posix_parent; diff --git a/crates/sqlbuild-core/src/posix_path/main/posix_escapes_root.rs b/crates/sqlbuild-core/src/posix_path/main/posix_escapes_root.rs new file mode 100644 index 0000000000..dec3d18b8b --- /dev/null +++ b/crates/sqlbuild-core/src/posix_path/main/posix_escapes_root.rs @@ -0,0 +1,12 @@ +//! Whether a `PurePosixPath` leaves the directory it is relative to. + +use crate::posix_path::constants::{DOUBLE_ROOT, PARENT, ROOT}; +use crate::posix_path::main::posix_parts::posix_parts; + +/// `PurePosixPath(path).is_absolute() or ".." in PurePosixPath(path).parts`. +#[must_use] +pub fn posix_escapes_root(path: &str) -> bool { + posix_parts(path) + .iter() + .any(|part| *part == ROOT || *part == DOUBLE_ROOT || *part == PARENT) +} diff --git a/crates/sqlbuild-core/src/posix_path/tests/test_posix_path.rs b/crates/sqlbuild-core/src/posix_path/tests/test_posix_path.rs index 4722f0737b..2390da7919 100644 --- a/crates/sqlbuild-core/src/posix_path/tests/test_posix_path.rs +++ b/crates/sqlbuild-core/src/posix_path/tests/test_posix_path.rs @@ -1,9 +1,10 @@ +use crate::posix_path::main::posix_escapes_root::posix_escapes_root; use crate::posix_path::main::posix_join::posix_join; use crate::posix_path::main::posix_name::posix_name; use crate::posix_path::main::posix_parent::posix_parent; use crate::posix_path::main::posix_parts::posix_parts; use crate::posix_path::main::posix_suffix::posix_suffix; -use crate::posix_path::tests::test_types::PosixPathTestCase; +use crate::posix_path::tests::test_types::{EscapesRootTestCase, PosixPathTestCase}; #[test] fn given_paths_when_reading_pure_posix_parts_then_they_match_pathlib() { @@ -140,3 +141,67 @@ fn given_paths_when_reading_pure_posix_parts_then_they_match_pathlib() { ); } } + +#[test] +fn given_paths_when_checking_for_escapes_then_absolute_and_parent_paths_escape() { + let test_cases = [ + EscapesRootTestCase { + description: "a relative file", + path: "models/a.sql", + expected_escapes: false, + }, + EscapesRootTestCase { + description: "current and empty parts", + path: "./a//b/", + expected_escapes: false, + }, + EscapesRootTestCase { + description: "an empty path", + path: "", + expected_escapes: false, + }, + EscapesRootTestCase { + description: "the current directory", + path: ".", + expected_escapes: false, + }, + EscapesRootTestCase { + description: "an absolute path", + path: "/abs/x.yml", + expected_escapes: true, + }, + EscapesRootTestCase { + description: "a double-slash root", + path: "//x", + expected_escapes: true, + }, + EscapesRootTestCase { + description: "a leading parent", + path: "../x.sql", + expected_escapes: true, + }, + EscapesRootTestCase { + description: "an inner parent", + path: "a/../x.sql", + expected_escapes: true, + }, + EscapesRootTestCase { + description: "only a parent", + path: "..", + expected_escapes: true, + }, + EscapesRootTestCase { + description: "dots inside names", + path: "a..b/..c", + expected_escapes: false, + }, + ]; + for test_case in test_cases { + assert_eq!( + posix_escapes_root(test_case.path), + test_case.expected_escapes, + "{}", + test_case.description + ); + } +} diff --git a/crates/sqlbuild-core/src/posix_path/tests/test_types.rs b/crates/sqlbuild-core/src/posix_path/tests/test_types.rs index 126c6e31d9..da99d81c93 100644 --- a/crates/sqlbuild-core/src/posix_path/tests/test_types.rs +++ b/crates/sqlbuild-core/src/posix_path/tests/test_types.rs @@ -7,3 +7,9 @@ pub(super) struct PosixPathTestCase { /// `suffix` under Python 3.12 and under Python 3.14. pub(super) expected_suffix: &'static str, } + +pub(super) struct EscapesRootTestCase { + pub(super) description: &'static str, + pub(super) path: &'static str, + pub(super) expected_escapes: bool, +} diff --git a/crates/sqlbuild-pysource/src/rule_evidence/constants.rs b/crates/sqlbuild-pysource/src/rule_evidence/constants.rs index 0b0a3ded5d..3f0c19ea5a 100644 --- a/crates/sqlbuild-pysource/src/rule_evidence/constants.rs +++ b/crates/sqlbuild-pysource/src/rule_evidence/constants.rs @@ -22,3 +22,37 @@ pub(crate) const TEST_FILE_PREFIX: &str = "test_"; pub(crate) const TEST_FILE_SUFFIX: &str = "_test.py"; pub(crate) const TEST_CLASS_PREFIX: &str = "Test"; pub(crate) const FLOAT_ZERO: f64 = 0.0_f64; +/// The modules, and their submodules, custom-rule code may import. +pub(crate) const CUSTOM_RULE_IMPORT_ROOTS: [&str; 29] = [ + "__future__", + "abc", + "bisect", + "collections", + "copy", + "dataclasses", + "decimal", + "difflib", + "enum", + "fnmatch", + "fractions", + "functools", + "graphlib", + "hashlib", + "heapq", + "itertools", + "json", + "math", + "operator", + "pathlib", + "re", + "rules", + "sqlbuild.rules", + "statistics", + "string", + "textwrap", + "types", + "typing", + "unicodedata", +]; +/// The separator between a module and its submodules. +pub(crate) const MODULE_SEPARATOR: char = '.'; diff --git a/crates/sqlbuild-pysource/src/rule_evidence/main/allowed_custom_rule_import.rs b/crates/sqlbuild-pysource/src/rule_evidence/main/allowed_custom_rule_import.rs new file mode 100644 index 0000000000..4c6c8cb625 --- /dev/null +++ b/crates/sqlbuild-pysource/src/rule_evidence/main/allowed_custom_rule_import.rs @@ -0,0 +1,13 @@ +//! The deterministic import allowlist of custom-rule code. + +use crate::rule_evidence::constants::{CUSTOM_RULE_IMPORT_ROOTS, MODULE_SEPARATOR}; + +/// Whether custom-rule code may import the absolute module `module`. +#[must_use] +pub fn allowed_custom_rule_import(module: &str) -> bool { + CUSTOM_RULE_IMPORT_ROOTS.iter().any(|root| { + module + .strip_prefix(root) + .is_some_and(|rest| rest.is_empty() || rest.starts_with(MODULE_SEPARATOR)) + }) +} diff --git a/crates/sqlbuild-pysource/src/rule_evidence/main/disallowed_import.rs b/crates/sqlbuild-pysource/src/rule_evidence/main/disallowed_import.rs new file mode 100644 index 0000000000..6f0e972915 --- /dev/null +++ b/crates/sqlbuild-pysource/src/rule_evidence/main/disallowed_import.rs @@ -0,0 +1,49 @@ +//! The first import a custom-rule source makes outside the allowlist, in `ast.walk` order. + +use ruff_python_ast::Stmt; +use ruff_text_size::Ranged; + +use crate::cpython_tree::main::parse_cpython_module::parse_cpython_module; +use crate::cpython_tree::main::source_position::source_position; +use crate::cpython_tree::main::walk_breadth_first::walk_breadth_first; +use crate::cpython_tree::models::{CpythonNode, ParsedSource}; +use crate::rule_evidence::main::allowed_custom_rule_import::allowed_custom_rule_import; +use crate::rule_evidence::models::{DisallowedImport, UnparsedSource}; + +/// The first disallowed absolute import in breadth-first order, or `UnparsedSource`. +pub fn disallowed_import( + source: &str, + python_version: (u8, u8), +) -> Result, UnparsedSource> { + let parsed: ParsedSource = + parse_cpython_module(source, python_version).ok_or(UnparsedSource)?; + let roots: Vec> = parsed.module.body.iter().map(CpythonNode::Stmt).collect(); + for node in walk_breadth_first(roots) { + let (statement, modules): (&Stmt, Vec<&str>) = match node { + CpythonNode::Stmt(statement @ Stmt::Import(import)) => ( + statement, + import + .names + .iter() + .map(|alias| alias.name.as_str()) + .collect(), + ), + CpythonNode::Stmt(statement @ Stmt::ImportFrom(import)) if import.level == 0 => ( + statement, + vec![import.module.as_ref().map_or("", |module| module.as_str())], + ), + _ => continue, + }; + if let Some(module) = modules + .into_iter() + .find(|module| !allowed_custom_rule_import(module)) + { + let (line, _) = source_position(&parsed.lines, statement.start()); + return Ok(Some(DisallowedImport { + line, + module: module.to_owned(), + })); + } + } + Ok(None) +} diff --git a/crates/sqlbuild-pysource/src/rule_evidence/main/mod.rs b/crates/sqlbuild-pysource/src/rule_evidence/main/mod.rs index e8b3ce047f..449b4c6551 100644 --- a/crates/sqlbuild-pysource/src/rule_evidence/main/mod.rs +++ b/crates/sqlbuild-pysource/src/rule_evidence/main/mod.rs @@ -1,3 +1,5 @@ +pub mod allowed_custom_rule_import; +pub mod disallowed_import; pub mod implementation_fingerprint; pub mod import_closure; pub mod test_evidence; diff --git a/crates/sqlbuild-pysource/src/rule_evidence/models.rs b/crates/sqlbuild-pysource/src/rule_evidence/models.rs index 68acd03cdb..f18c81f752 100644 --- a/crates/sqlbuild-pysource/src/rule_evidence/models.rs +++ b/crates/sqlbuild-pysource/src/rule_evidence/models.rs @@ -15,3 +15,14 @@ pub(crate) struct RuleTarget<'a> { pub(crate) modules: Vec, pub(crate) check_name: &'a str, } + +/// An import outside the custom-rule allowlist: its statement's line and the module. +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct DisallowedImport { + pub line: usize, + pub module: String, +} + +/// A source CPython's parser would reject. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub struct UnparsedSource; diff --git a/crates/sqlbuild-pysource/src/rule_evidence/tests.rs b/crates/sqlbuild-pysource/src/rule_evidence/tests.rs index 52b71770bf..1d5ac43a78 100644 --- a/crates/sqlbuild-pysource/src/rule_evidence/tests.rs +++ b/crates/sqlbuild-pysource/src/rule_evidence/tests.rs @@ -1,5 +1,7 @@ #[path = "tests/helpers.rs"] mod helpers; +#[path = "tests/test_hermeticity.rs"] +mod hermeticity; #[path = "tests/test_rule_evidence.rs"] mod rule_evidence; #[path = "tests/test_types.rs"] diff --git a/crates/sqlbuild-pysource/src/rule_evidence/tests/test_hermeticity.rs b/crates/sqlbuild-pysource/src/rule_evidence/tests/test_hermeticity.rs new file mode 100644 index 0000000000..536347293e --- /dev/null +++ b/crates/sqlbuild-pysource/src/rule_evidence/tests/test_hermeticity.rs @@ -0,0 +1,121 @@ +use crate::rule_evidence::main::allowed_custom_rule_import::allowed_custom_rule_import; +use crate::rule_evidence::main::disallowed_import::disallowed_import; +use crate::rule_evidence::models::{DisallowedImport, UnparsedSource}; +use crate::rule_evidence::tests::test_types::{AllowedImportTestCase, DisallowedImportTestCase}; + +const PYTHON: (u8, u8) = (3, 12); + +#[test] +fn given_module_names_when_checked_then_only_allowlisted_roots_and_submodules_pass() { + let test_cases = [ + AllowedImportTestCase { + description: "an allowed root", + module: "json", + expected_allowed: true, + }, + AllowedImportTestCase { + description: "an allowed submodule", + module: "json.decoder", + expected_allowed: true, + }, + AllowedImportTestCase { + description: "an allowed dotted root", + module: "sqlbuild.rules", + expected_allowed: true, + }, + AllowedImportTestCase { + description: "a submodule of an allowed package", + module: "collections.abc", + expected_allowed: true, + }, + AllowedImportTestCase { + description: "a longer name sharing a prefix", + module: "jsonx", + expected_allowed: false, + }, + AllowedImportTestCase { + description: "the parent of an allowed dotted root", + module: "sqlbuild", + expected_allowed: false, + }, + AllowedImportTestCase { + description: "an empty module", + module: "", + expected_allowed: false, + }, + AllowedImportTestCase { + description: "a trailing dot", + module: "json.", + expected_allowed: true, + }, + AllowedImportTestCase { + description: "a leading dot", + module: ".json", + expected_allowed: false, + }, + ]; + for test_case in test_cases { + assert_eq!( + allowed_custom_rule_import(test_case.module), + test_case.expected_allowed, + "{}", + test_case.description + ); + } +} + +#[test] +fn given_rule_sources_when_scanned_then_the_first_disallowed_import_in_walk_order_is_found() { + let test_cases = [ + DisallowedImportTestCase { + description: "only allowed and relative imports", + source: "import json\nfrom . import helpers\nfrom re import x\n", + expected_import: Ok(None), + }, + DisallowedImportTestCase { + description: "the first disallowed top-level import", + source: "import json\nimport os\nimport subprocess\n", + expected_import: Ok(Some(DisallowedImport { + line: 2, + module: "os".to_owned(), + })), + }, + DisallowedImportTestCase { + description: "top-level statements are visited before nested ones", + source: "def f():\n import socket\n\nfrom urllib import request\n", + expected_import: Ok(Some(DisallowedImport { + line: 4, + module: "urllib".to_owned(), + })), + }, + DisallowedImportTestCase { + description: "each alias of one statement in order", + source: "import json, os.path as p, re\n", + expected_import: Ok(Some(DisallowedImport { + line: 1, + module: "os.path".to_owned(), + })), + }, + DisallowedImportTestCase { + description: "CRLF lines and continuations", + source: "import json\r\n\r\nimport \\\n os\r\n", + expected_import: Ok(Some(DisallowedImport { + line: 3, + module: "os".to_owned(), + })), + }, + DisallowedImportTestCase { + description: "a syntax error", + source: "def broken(:\n", + expected_import: Err(UnparsedSource), + }, + ]; + for test_case in test_cases { + assert_eq!( + disallowed_import(test_case.source, PYTHON), + test_case.expected_import, + "{}", + test_case.description + ); + } +} diff --git a/crates/sqlbuild-pysource/src/rule_evidence/tests/test_types.rs b/crates/sqlbuild-pysource/src/rule_evidence/tests/test_types.rs index 6fc509556a..fe7802a5cc 100644 --- a/crates/sqlbuild-pysource/src/rule_evidence/tests/test_types.rs +++ b/crates/sqlbuild-pysource/src/rule_evidence/tests/test_types.rs @@ -5,3 +5,18 @@ pub(crate) struct EvidenceTestCase { pub(crate) expected_closure: Vec<&'static str>, pub(crate) expected_cases: Vec<(usize, usize, &'static str)>, } + +pub(crate) struct AllowedImportTestCase { + pub(crate) description: &'static str, + pub(crate) module: &'static str, + pub(crate) expected_allowed: bool, +} + +pub(crate) struct DisallowedImportTestCase { + pub(crate) description: &'static str, + pub(crate) source: &'static str, + pub(crate) expected_import: Result< + Option, + crate::rule_evidence::models::UnparsedSource, + >, +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/authored_values/mod.rs b/crates/sqlbuild-python/src/bindings/_helpers/authored_values/mod.rs index 8e07eaa020..dca87c66e6 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/authored_values/mod.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/authored_values/mod.rs @@ -1,3 +1,4 @@ mod fields; pub(crate) mod project_vars; +mod python_nodes; pub(crate) mod registration; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/authored_values/python_nodes.rs b/crates/sqlbuild-python/src/bindings/_helpers/authored_values/python_nodes.rs new file mode 100644 index 0000000000..1172de6137 --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/authored_values/python_nodes.rs @@ -0,0 +1,209 @@ +//! Python-node authoring checks: retry policies, audit cases and hook reads, in Python's operators. + +use pyo3::exceptions::{PyBaseException, PyTypeError}; +use pyo3::prelude::{ + Bound, PyAny, PyAnyMethods, PyErr, PyModule, PyModuleMethods, PyResult, Python, +}; +use pyo3::types::{ + PyDict, PyDictMethods, PyInt, PyMemoryView, PyString, PyStringMethods, PyTuple, PyType, +}; +use pyo3::types::{PyTupleMethods, PyTypeMethods}; +use pyo3::{pyfunction, wrap_pyfunction}; +use sqlbuild_core::identifier_text::main::is_lowercase_identifier::is_lowercase_identifier; + +use crate::bindings::_helpers::boundary::keyword_request::Request; +use crate::bindings::_helpers::compile_driver::python_calls::imported; + +const ERRORS_MODULE: &str = "sqlbuild.errors.contracts.exceptions"; +const SHARED_INPUT_ERROR: &str = "SharedInputError"; +const MODELS_MODULE: &str = "sqlbuild.python_nodes.models"; +const RESOURCE_REF_CLASS: &str = "SqlResourceRef"; +const RANDOM_MODULE: &str = "random"; +const MAX_ATTEMPTS_TOO_LOW: &str = "RetryPolicy max_attempts must be at least 1"; +const INITIAL_DELAY_NEGATIVE: &str = "RetryPolicy initial_delay_seconds must be non-negative"; +const BACKOFF_TOO_LOW: &str = "RetryPolicy backoff_multiplier must be at least 1"; +const MAX_DELAY_NEGATIVE: &str = "RetryPolicy max_delay_seconds must be non-negative"; +const MAX_ELAPSED_NEGATIVE: &str = "RetryPolicy max_elapsed_seconds must be non-negative"; +const RETRY_ON_NOT_EXCEPTIONS: &str = "RetryPolicy retry_on must contain exception classes"; +const AUDIT_CASE_NAME_INVALID: &str = "AuditCase name must be a non-empty lowercase identifier \ + using letters, numbers, and underscores"; +const AUDIT_CASE_DEFINITION_EMPTY: &str = "AuditCase definition must be non-empty"; +const AUDIT_CASE_MINIMUM_SAMPLES: &str = "AuditCase minimum_samples must be a non-negative integer"; +const AUDIT_CASE_EVIDENCE_LIMIT: &str = "AuditCase evidence_limit must be a non-negative integer"; +const BYTES_NAME_ERROR: &str = "cannot use a string pattern on a bytes-like object"; +const HOOK_READS_INVALID: &str = + "@hook(reads=...) accepts model(...), source(...), and seed(...) references only; got "; + +/// `SharedInputError(message)`. +fn shared_input_error(py: Python<'_>, message: &str) -> PyErr { + match imported(py, ERRORS_MODULE, SHARED_INPUT_ERROR).and_then(|class| class.call1((message,))) + { + Ok(error) => PyErr::from_value(error), + Err(error) => error, + } +} + +/// `value < bound` under Python's comparison. +fn below(value: &Bound<'_, PyAny>, bound: i64) -> PyResult { + value.lt(bound) +} + +/// Whether `value` is an exception class. +fn exception_class(value: &Bound<'_, PyAny>) -> PyResult { + match value.downcast::() { + Ok(class) => class.is_subclass_of::(), + Err(_) => Ok(false), + } +} + +/// The validated `retry_on` exception classes of a retry policy. +#[pyfunction] +#[pyo3(signature = (**request))] +fn retry_policy_exceptions<'py>( + request: Option<&Bound<'py, PyDict>>, +) -> PyResult> { + let request: Request<'_, 'py> = Request::new("retry_policy_exceptions", request)?; + let py: Python<'py> = request.py(); + let max_delay_seconds: Bound<'py, PyAny> = request.get("max_delay_seconds")?; + let max_elapsed_seconds: Bound<'py, PyAny> = request.get("max_elapsed_seconds")?; + if below(&request.get("max_attempts")?, 1)? { + return Err(shared_input_error(py, MAX_ATTEMPTS_TOO_LOW)); + } + if below(&request.get("initial_delay_seconds")?, 0)? { + return Err(shared_input_error(py, INITIAL_DELAY_NEGATIVE)); + } + if below(&request.get("backoff_multiplier")?, 1)? { + return Err(shared_input_error(py, BACKOFF_TOO_LOW)); + } + if !max_delay_seconds.is_none() && below(&max_delay_seconds, 0)? { + return Err(shared_input_error(py, MAX_DELAY_NEGATIVE)); + } + if !max_elapsed_seconds.is_none() && below(&max_elapsed_seconds, 0)? { + return Err(shared_input_error(py, MAX_ELAPSED_NEGATIVE)); + } + let retry_on: Bound<'py, PyAny> = request.get("retry_on")?; + if exception_class(&retry_on)? { + return PyTuple::new(py, [retry_on]); + } + let classes: Bound<'py, PyTuple> = PyTuple::new( + py, + retry_on + .try_iter()? + .collect::>>>()?, + )?; + for class in classes.iter() { + if !exception_class(&class)? { + return Err(shared_input_error(py, RETRY_ON_NOT_EXCEPTIONS)); + } + } + Ok(classes) +} + +/// The delay before retry `retry_index`: exponential backoff, capped, and jittered by `random`. +#[pyfunction] +#[pyo3(signature = (*, retry_policy, retry_index))] +fn retry_delay_seconds<'py>( + retry_policy: &Bound<'py, PyAny>, + retry_index: &Bound<'py, PyAny>, +) -> PyResult> { + let initial: Bound<'py, PyAny> = retry_policy.getattr("initial_delay_seconds")?; + let growth: Bound<'py, PyAny> = retry_policy + .getattr("backoff_multiplier")? + .pow(retry_index, retry_policy.py().None())?; + let mut delay: Bound<'py, PyAny> = initial.mul(growth)?; + let maximum: Bound<'py, PyAny> = retry_policy.getattr("max_delay_seconds")?; + if !maximum.is_none() && maximum.lt(&delay)? { + delay = maximum; + } + if retry_policy.getattr("jitter")?.is_truthy()? { + return imported(retry_policy.py(), RANDOM_MODULE, "uniform")?.call1((0, delay)); + } + Ok(delay) +} + +/// The `re.fullmatch` failure for a name that is not a string. +fn name_type_error(name: &Bound<'_, PyAny>) -> PyResult { + match PyMemoryView::from(name) { + Ok(_) => return Ok(PyTypeError::new_err(BYTES_NAME_ERROR)), + Err(error) if error.is_instance_of::(name.py()) => {} + Err(error) => return Err(error), + } + Ok(PyTypeError::new_err(format!( + "expected string or bytes-like object, got '{}'", + name.get_type().name()? + ))) +} + +/// Whether an optional count is absent or a non-negative integer other than a boolean. +fn valid_count(value: &Bound<'_, PyAny>) -> PyResult { + if value.is_none() { + return Ok(true); + } + if !value.is_instance_of::() || value.is_instance_of::() { + return Ok(false); + } + Ok(!below(value, 0)?) +} + +/// Reject an audit case without a lowercase name, a definition, or valid optional counts. +#[pyfunction] +#[pyo3(signature = (*, name, definition, minimum_samples, evidence_limit))] +fn validate_audit_case( + name: &Bound<'_, PyAny>, + definition: &Bound<'_, PyAny>, + minimum_samples: &Bound<'_, PyAny>, + evidence_limit: &Bound<'_, PyAny>, +) -> Result<(), PyErr> { + let py: Python<'_> = name.py(); + let Ok(text) = name.downcast::() else { + return Err(name_type_error(name)?); + }; + if !is_lowercase_identifier(&text.to_string_lossy()) { + return Err(shared_input_error(py, AUDIT_CASE_NAME_INVALID)); + } + if !definition.call_method0("strip")?.is_truthy()? { + return Err(shared_input_error(py, AUDIT_CASE_DEFINITION_EMPTY)); + } + if !valid_count(minimum_samples)? { + return Err(shared_input_error(py, AUDIT_CASE_MINIMUM_SAMPLES)); + } + if !valid_count(evidence_limit)? { + return Err(shared_input_error(py, AUDIT_CASE_EVIDENCE_LIMIT)); + } + Ok(()) +} + +/// `@hook(reads=...)` as distinct resource references in first-seen order. +#[pyfunction] +fn normalize_hook_reads<'py>(value: &Bound<'py, PyAny>) -> PyResult> { + let py: Python<'py> = value.py(); + let reference: Bound<'py, PyAny> = imported(py, MODELS_MODULE, RESOURCE_REF_CLASS)?; + let reads: Vec> = if value.is_instance(&reference)? { + vec![value.clone()] + } else { + value.try_iter()?.collect::>()? + }; + for item in &reads { + if !item.is_instance(&reference)? { + let message: String = format!("{HOOK_READS_INVALID}{}", item.get_type().name()?); + return Err(shared_input_error(py, &message)); + } + } + let distinct: Bound<'py, PyDict> = PyDict::new(py); + let mut ordered: Vec> = Vec::new(); + for item in reads { + if !distinct.contains(&item)? { + distinct.set_item(&item, py.None())?; + ordered.push(item); + } + } + PyTuple::new(py, ordered) +} + +pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { + module.add_function(wrap_pyfunction!(retry_policy_exceptions, module)?)?; + module.add_function(wrap_pyfunction!(retry_delay_seconds, module)?)?; + module.add_function(wrap_pyfunction!(validate_audit_case, module)?)?; + module.add_function(wrap_pyfunction!(normalize_hook_reads, module)?)?; + Ok(()) +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/authored_values/registration.rs b/crates/sqlbuild-python/src/bindings/_helpers/authored_values/registration.rs index d33a3a7ff8..c1c11ce4c5 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/authored_values/registration.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/authored_values/registration.rs @@ -2,5 +2,6 @@ use pyo3::prelude::{Bound, PyModule, PyResult}; pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { crate::bindings::_helpers::authored_values::fields::register(module)?; - crate::bindings::_helpers::authored_values::project_vars::register(module) + crate::bindings::_helpers::authored_values::project_vars::register(module)?; + crate::bindings::_helpers::authored_values::python_nodes::register(module) } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compile_command/mod.rs b/crates/sqlbuild-python/src/bindings/_helpers/compile_command/mod.rs index c6f2c8f654..a10d21a019 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compile_command/mod.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compile_command/mod.rs @@ -7,6 +7,7 @@ pub(crate) mod report; pub(crate) mod reuse_attempt; pub(crate) mod reuse_recording; pub(crate) mod runtime_identity; +pub(crate) mod startup_inputs; pub(crate) mod surrogates; pub(crate) mod target_writer; pub(crate) mod test_artifact_cache; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compile_command/registration.rs b/crates/sqlbuild-python/src/bindings/_helpers/compile_command/registration.rs index add7a0460c..6b1d20cd31 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compile_command/registration.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compile_command/registration.rs @@ -5,7 +5,8 @@ use pyo3::prelude::{Bound, PyModule, PyModuleMethods, PyResult}; use crate::bindings::_helpers::compile_command::models::CompileReuseAttempt; use crate::bindings::_helpers::compile_command::{ - prepared_artifacts, report, reuse_attempt, reuse_recording, runtime_identity, target_writer, + prepared_artifacts, report, reuse_attempt, reuse_recording, runtime_identity, startup_inputs, + target_writer, }; pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { @@ -14,6 +15,7 @@ pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { reuse_attempt::register(module)?; reuse_recording::register(module)?; runtime_identity::register(module)?; + startup_inputs::register(module)?; target_writer::register(module)?; prepared_artifacts::register(module) } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compile_command/startup_inputs.rs b/crates/sqlbuild-python/src/bindings/_helpers/compile_command/startup_inputs.rs new file mode 100644 index 0000000000..3be85bc159 --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/compile_command/startup_inputs.rs @@ -0,0 +1,69 @@ +//! Stamps of the interpreter and site-path inputs Python startup depends on. + +use std::path::PathBuf; + +use pyo3::exceptions::PyOSError; +use pyo3::prelude::{ + Bound, PyAny, PyAnyMethods, PyErr, PyModule, PyModuleMethods, PyResult, Python, +}; +use pyo3::types::PyList; +use pyo3::{IntoPyObject, pyfunction, wrap_pyfunction}; +use sqlbuild_cache::startup_inputs::main::startup_stamps::startup_stamps; +use sqlbuild_cache::startup_inputs::models::{StartupLocations, StartupStamp}; + +use crate::bindings::_helpers::compile_driver::python_calls::imported; + +const OS_MODULE: &str = "os"; + +fn paths(values: &Bound<'_, PyAny>) -> PyResult> { + values.try_iter()?.map(|value| value?.extract()).collect() +} + +/// `OSError(errno, os.strerror(errno), path)`, which Python narrows to its subclass. +fn stat_error(py: Python<'_>, path: PathBuf, error: &std::io::Error) -> PyErr { + let Some(errno) = error.raw_os_error() else { + return PyOSError::new_err(error.to_string()); + }; + let message: String = match imported(py, OS_MODULE, "strerror") + .and_then(|strerror| strerror.call1((errno,))) + .and_then(|text| text.extract()) + { + Ok(message) => message, + Err(failure) => return failure, + }; + let Ok(path) = path.into_os_string().into_pyobject(py); + PyOSError::new_err((errno, message, path.unbind())) +} + +/// `[(path, st_mtime_ns, st_size), ...]` for every startup input path. +#[pyfunction] +#[pyo3(signature = (*, directories, files, prefixes))] +fn startup_input_stamps<'py>( + directories: &Bound<'py, PyAny>, + files: &Bound<'py, PyAny>, + prefixes: &Bound<'py, PyAny>, +) -> PyResult> { + let py: Python<'py> = directories.py(); + let locations = StartupLocations { + directories: paths(directories)?, + files: paths(files)?, + prefixes: paths(prefixes)?, + }; + let cwd: PathBuf = imported(py, OS_MODULE, "getcwd")?.call0()?.extract()?; + let stamps: Vec = + startup_stamps(&locations, &cwd).map_err(|(path, error)| stat_error(py, path, &error))?; + let rows: Vec> = stamps + .into_iter() + .map(|stamp| { + Ok((stamp.path.into_os_string(), stamp.modified_ns, stamp.size) + .into_pyobject(py)? + .into_any()) + }) + .collect::>()?; + PyList::new(py, rows) +} + +pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { + module.add_function(wrap_pyfunction!(startup_input_stamps, module)?)?; + Ok(()) +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/dagster_invocation/mod.rs b/crates/sqlbuild-python/src/bindings/_helpers/dagster_invocation/mod.rs index 9d1391a8de..6ad661843c 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/dagster_invocation/mod.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/dagster_invocation/mod.rs @@ -3,3 +3,4 @@ pub(crate) mod arguments; pub(crate) mod failures; pub(crate) mod registration; pub(crate) mod results; +pub(crate) mod translator; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/dagster_invocation/registration.rs b/crates/sqlbuild-python/src/bindings/_helpers/dagster_invocation/registration.rs index 58ff98354a..0700489b8f 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/dagster_invocation/registration.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/dagster_invocation/registration.rs @@ -3,7 +3,7 @@ use pyo3::prelude::{Bound, PyModule, PyModuleMethods, PyResult}; use pyo3::wrap_pyfunction; -use crate::bindings::_helpers::dagster_invocation::{arguments, failures, results}; +use crate::bindings::_helpers::dagster_invocation::{arguments, failures, results, translator}; pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { module.add_function(wrap_pyfunction!(arguments::dagster_cli_arguments, module)?)?; @@ -25,5 +25,30 @@ pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { module )?)?; module.add_function(wrap_pyfunction!(failures::dagster_failure, module)?)?; - module.add_function(wrap_pyfunction!(failures::dagster_live_failure, module)?) + module.add_function(wrap_pyfunction!(failures::dagster_live_failure, module)?)?; + module.add_function(wrap_pyfunction!( + translator::dagster_default_asset_key_path, + module + )?)?; + module.add_function(wrap_pyfunction!( + translator::dagster_default_group_name, + module + )?)?; + module.add_function(wrap_pyfunction!(translator::dagster_default_tags, module)?)?; + module.add_function(wrap_pyfunction!( + translator::dagster_default_metadata, + module + )?)?; + module.add_function(wrap_pyfunction!( + translator::dagster_default_description, + module + )?)?; + module.add_function(wrap_pyfunction!( + translator::dagster_default_check_name, + module + )?)?; + module.add_function(wrap_pyfunction!( + translator::dagster_default_check_metadata, + module + )?) } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/dagster_invocation/translator.rs b/crates/sqlbuild-python/src/bindings/_helpers/dagster_invocation/translator.rs new file mode 100644 index 0000000000..671f34c8e0 --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/dagster_invocation/translator.rs @@ -0,0 +1,238 @@ +//! The default `SqlBuildDagsterTranslator` mappings its overridable methods call. + +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult, Python}; +use pyo3::pyfunction; +use pyo3::types::{PyBytes, PyDict, PyDictMethods, PyString, PyStringMethods, PyTuple}; +use sqlbuild_core::dagster_invocation::constants::{DEFAULT_ASSET_KIND, DEFAULT_ASSET_NAME}; +use sqlbuild_core::dagster_invocation::main::asset_description::asset_description; +use sqlbuild_core::dagster_invocation::main::dagster_name_segment::dagster_name_segment; +use sqlbuild_core::dagster_invocation::main::dagster_tag_key::dagster_tag_key; +use sqlbuild_core::dagster_invocation::main::fallback_asset_description::fallback_asset_description; +use sqlbuild_core::text::main::python_strip::python_strip; + +use crate::bindings::_helpers::compile_driver::python_calls::imported; +use crate::bindings::_helpers::dagster_invocation::access::{CONSTANTS_MODULE, get, value_or}; + +const ABC_MODULE: &str = "collections.abc"; +const BUILTINS_MODULE: &str = "builtins"; +const KIND_TAG: &str = "sqlbuild/kind"; +const DEFAULT_GROUP: &str = "sqlbuild"; +const DEFAULT_CHECK_PART: &str = "check"; +const CHECK_PART_SEPARATOR: &str = "__"; +/// Node fields copied into asset metadata when present, in order. +const METADATA_KEYS: [&str; 12] = [ + "path", + "target", + "description", + "sql", + "columns", + "column_lineage", + "group", + "language", + "materialization_type", + "return_kind", + "loader", + "meta", +]; + +/// `str(value)`. +fn text(value: &Bound<'_, PyAny>) -> PyResult { + Ok(value.str()?.to_string_lossy().into_owned()) +} + +/// `get(key) or default` read lazily: `default` is not consulted when the value is truthy. +fn text_or_default<'py>(mapping: &Bound<'py, PyAny>, key: &str, default: &str) -> PyResult { + let value: Bound<'py, PyAny> = get(mapping, key)?; + if value.is_truthy()? { + return text(&value); + } + Ok(default.to_owned()) +} + +fn abc_instance(value: &Bound<'_, PyAny>, class: &str) -> PyResult { + value.is_instance(&imported(value.py(), ABC_MODULE, class)?) +} + +/// A source node's non-empty, non-text `meta.dagster.asset_key` sequence. +fn authored_asset_key<'py>(node: &Bound<'py, PyAny>) -> PyResult>> { + let source_kind: String = + imported(node.py(), CONSTANTS_MODULE, "SOURCE_NODE_KIND")?.extract()?; + if text(&get(node, "kind")?)? != source_kind { + return Ok(None); + } + let meta: Bound<'py, PyAny> = get(node, "meta")?; + if !abc_instance(&meta, "Mapping")? { + return Ok(None); + } + let dagster: Bound<'py, PyAny> = get(&meta, "dagster")?; + if !abc_instance(&dagster, "Mapping")? { + return Ok(None); + } + let asset_key: Bound<'py, PyAny> = get(&dagster, "asset_key")?; + if !abc_instance(&asset_key, "Sequence")? + || asset_key.is_instance_of::() + || asset_key.is_instance_of::() + || !asset_key.is_truthy()? + { + return Ok(None); + } + Ok(Some(asset_key)) +} + +/// The asset key path of a node: its authored Dagster key, or its SQLBuild asset key. +#[pyfunction] +pub(crate) fn dagster_default_asset_key_path(node: &Bound<'_, PyAny>) -> PyResult> { + let parts: Bound<'_, PyAny> = match authored_asset_key(node)? { + Some(parts) => parts, + None => node.get_item("asset_key")?, + }; + parts.try_iter()?.map(|part| text(&part?)).collect() +} + +/// The node's group, project name or `sqlbuild`, with `-` read as `_`. +#[pyfunction] +pub(crate) fn dagster_default_group_name(node: &Bound<'_, PyAny>) -> PyResult { + let group: Bound<'_, PyAny> = get(node, "group")?; + let chosen: String = if group.is_truthy()? { + text(&group)? + } else { + text_or_default(node, "project_name", DEFAULT_GROUP)? + }; + Ok(chosen.replace('-', "_")) +} + +/// The node kind tag and one empty-valued tag per SQLBuild tag with a usable key. +#[pyfunction] +pub(crate) fn dagster_default_tags<'py>(node: &Bound<'py, PyAny>) -> PyResult> { + let py: Python<'py> = node.py(); + let tags: Bound<'py, PyDict> = PyDict::new(py); + tags.set_item( + KIND_TAG, + text(&value_or(node, "kind", PyString::new(py, "").into_any())?)?, + )?; + for tag in value_or(node, "tags", PyTuple::empty(py).into_any())?.try_iter()? { + let key: String = dagster_tag_key(&text(&tag?)?); + if !key.is_empty() { + tags.set_item(key, "")?; + } + } + Ok(tags) +} + +/// The SQLBuild identity fields and every present node field Dagster shows as metadata. +#[pyfunction] +pub(crate) fn dagster_default_metadata<'py>( + node: &Bound<'py, PyAny>, +) -> PyResult> { + let metadata: Bound<'py, PyDict> = PyDict::new(node.py()); + metadata.set_item("sqlbuild_id", get(node, "id")?)?; + metadata.set_item("sqlbuild_name", get(node, "name")?)?; + metadata.set_item("sqlbuild_kind", get(node, "kind")?)?; + metadata.set_item("sqlbuild_selector", get(node, "name")?)?; + for key in METADATA_KEYS { + if node.contains(key)? { + metadata.set_item(key, node.get_item(key)?)?; + } + } + Ok(metadata) +} + +/// The generated description of a node without one. +fn fallback_description(node: &Bound<'_, PyAny>) -> PyResult { + let kind: String = text_or_default(node, "kind", DEFAULT_ASSET_KIND)?; + let name: String = text_or_default(node, "name", DEFAULT_ASSET_NAME)?; + let target: Bound<'_, PyAny> = get(node, "target")?; + let qualified_name: String = if abc_instance(&target, "Mapping")? { + text_or_default(&target, "qualified_name", "")? + } else { + String::new() + }; + let materialization: Bound<'_, PyAny> = get(node, "materialization_type")?; + let materialization: Option = if materialization.is_none() { + None + } else { + Some( + imported(node.py(), BUILTINS_MODULE, "format")? + .call1((materialization,))? + .str()? + .to_string_lossy() + .into_owned(), + ) + }; + Ok(fallback_asset_description( + &kind, + &name, + &qualified_name, + materialization.as_deref(), + )) +} + +/// A string field, or `None` when the field is not text. +fn text_field(node: &Bound<'_, PyAny>, key: &str) -> PyResult> { + let value: Bound<'_, PyAny> = get(node, key)?; + match value.downcast::() { + Ok(value) => Ok(Some(value.to_string_lossy().into_owned())), + Err(_) => Ok(None), + } +} + +/// The node's description, or a generated one, followed by its SQL or source file. +#[pyfunction] +pub(crate) fn dagster_default_description(node: &Bound<'_, PyAny>) -> PyResult { + let description: Bound<'_, PyAny> = get(node, "description")?; + let description: String = if description.is_truthy()? { + text(&description)? + } else { + fallback_description(node)? + }; + let sql: Option = text_field(node, "sql")?; + let path: Option = if sql + .as_deref() + .is_some_and(|sql| !python_strip(sql).is_empty()) + { + None + } else { + text_field(node, "path")? + }; + Ok(asset_description( + &description, + sql.as_deref(), + path.as_deref(), + )) +} + +/// `kind__name[__column or target]`, each part made Dagster-safe. +#[pyfunction] +pub(crate) fn dagster_default_check_name(check: &Bound<'_, PyAny>) -> PyResult { + let py: Python<'_> = check.py(); + let default = || PyString::new(py, DEFAULT_CHECK_PART).into_any(); + let mut parts: Vec = vec![ + text(&value_or(check, "kind", default())?)?, + text(&value_or(check, "name", default())?)?, + ]; + for attached in ["attached_column_name", "attached_target_name"] { + if !get(check, attached)?.is_none() { + parts.push(text(&check.get_item(attached)?)?); + break; + } + } + Ok(parts + .iter() + .filter(|part| !part.is_empty()) + .map(|part| dagster_name_segment(part)) + .collect::>() + .join(CHECK_PART_SEPARATOR)) +} + +/// The SQLBuild identity fields of one check. +#[pyfunction] +pub(crate) fn dagster_default_check_metadata<'py>( + check: &Bound<'py, PyAny>, +) -> PyResult> { + let metadata: Bound<'py, PyDict> = PyDict::new(check.py()); + metadata.set_item("sqlbuild_check_id", get(check, "id")?)?; + metadata.set_item("sqlbuild_check_kind", get(check, "kind")?)?; + metadata.set_item("sqlbuild_check_name", get(check, "name")?)?; + metadata.set_item("sqlbuild_check_selector", get(check, "id")?)?; + Ok(metadata) +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/rules/catalogue.rs b/crates/sqlbuild-python/src/bindings/_helpers/rules/catalogue.rs index 7b74700c6e..d872186f32 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/rules/catalogue.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/rules/catalogue.rs @@ -15,11 +15,11 @@ use sqlbuild_rules::rule_catalogue::models::{CatalogueRule, OptionKind, OptionSp use crate::bindings::_helpers::boundary::panics::compiler_guard; use crate::bindings::_helpers::compile_driver::python_calls::{call_with, imported, keywords}; use crate::bindings::_helpers::rules::builtin_catalogue::builtin_rules; +use crate::bindings::_helpers::rules::custom_rule_evidence::verify_custom_rules as verify_rule_sources; use crate::bindings::_helpers::rules::selection::{rules_error, selected_rule_codes}; const CUSTOM_CATALOGUE_MODULE: &str = "sqlbuild.rule_engine._helpers.engine._custom_catalogue"; const CUSTOM_PAYLOADS_MODULE: &str = "sqlbuild.rule_engine._helpers.engine.custom_payloads"; -const HERMETICITY_MODULE: &str = "sqlbuild.rule_engine._helpers.engine.hermeticity"; const PLANNER_SCOPE_MODULE: &str = "sqlbuild._native"; const DATACLASSES_MODULE: &str = "dataclasses"; const JSON_MODULE: &str = "json"; @@ -264,13 +264,7 @@ pub(crate) fn verify_custom_rules<'py>( project_dir: &Bound<'py, PyAny>, ) -> PyResult<()> { let py: Python<'py> = project_dir.py(); - let _ = call_with( - &imported(py, HERMETICITY_MODULE, "verify_custom_rules")?, - &[ - ("rules", PyTuple::new(py, rules)?.into_any()), - ("project_dir", project_dir.clone()), - ], - )?; + verify_rule_sources(PyTuple::new(py, rules)?.as_any(), project_dir)?; Ok(()) } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/rules/custom_rule_evidence.rs b/crates/sqlbuild-python/src/bindings/_helpers/rules/custom_rule_evidence.rs index fe1ab554a6..3c61e9b37e 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/rules/custom_rule_evidence.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/rules/custom_rule_evidence.rs @@ -1,20 +1,30 @@ //! Custom-rule import closures, implementation fingerprints and statically found test cases. -use std::path::PathBuf; +use std::path::{Path, PathBuf}; -use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyModule, PyModuleMethods, PyResult, Python}; -use pyo3::types::PyTuple; +use pyo3::prelude::{ + Bound, PyAny, PyAnyMethods, PyErr, PyModule, PyModuleMethods, PyResult, Python, +}; +use pyo3::types::{PyDict, PyDictMethods, PySet, PySetMethods, PyString, PyTuple, PyTupleMethods}; use pyo3::{pyfunction, wrap_pyfunction}; +use sqlbuild_pysource::rule_evidence::main::allowed_custom_rule_import::allowed_custom_rule_import; +use sqlbuild_pysource::rule_evidence::main::disallowed_import::disallowed_import; use sqlbuild_pysource::rule_evidence::main::implementation_fingerprint::implementation_fingerprint; use sqlbuild_pysource::rule_evidence::main::import_closure::import_closure as import_closure_of; use sqlbuild_pysource::rule_evidence::main::test_evidence::test_evidence; -use sqlbuild_pysource::rule_evidence::models::RuleTestEvidence; +use sqlbuild_pysource::rule_evidence::models::{ + DisallowedImport, RuleTestEvidence, UnparsedSource, +}; use crate::bindings::_helpers::compile_driver::python_calls::imported; const PATHLIB_MODULE: &str = "pathlib"; const INSPECT_MODULE: &str = "inspect"; const HASHLIB_MODULE: &str = "hashlib"; +const AST_MODULE: &str = "ast"; +const NATIVE_MODULE: &str = "sqlbuild._native"; +const RULES_ERROR: &str = "RulesError"; +const NON_HERMETIC_RULE_ERROR: &str = "NonHermeticRuleError"; fn python_version(py: Python<'_>) -> (u8, u8) { let version = py.version_info(); @@ -120,7 +130,108 @@ fn custom_rule_test_evidence<'py>( .collect()) } +/// `error_class(message)` from the native module. +fn rules_error(py: Python<'_>, error_class: &str, message: String) -> PyErr { + match imported(py, NATIVE_MODULE, error_class).and_then(|class| class.call1((message,))) { + Ok(error) => PyErr::from_value(error), + Err(error) => error, + } +} + +/// Whether custom-rule code may import one absolute module name. +#[pyfunction] +#[pyo3(name = "allowed_custom_rule_import")] +fn allowed_custom_rule_import_binding(module: &str) -> bool { + allowed_custom_rule_import(module) +} + +/// Reject a repository-owned rule file that imports outside the allowlist. +fn verify_source(path: &Bound<'_, PyAny>, root: &Path) -> PyResult<()> { + let py: Python<'_> = path.py(); + if !native_path(path)?.starts_with(root) { + return Err(rules_error( + py, + RULES_ERROR, + format!( + "custom rule source must be repository-owned: {}", + path.str()? + ), + )); + } + let source: Bound<'_, PyAny> = path.call_method("read_text", (), Some(&utf8_encoding(py)?))?; + let text: String = source.extract()?; + match disallowed_import(&text, python_version(py)) { + Ok(None) => Ok(()), + Ok(Some(DisallowedImport { line, module })) => Err(rules_error( + py, + NON_HERMETIC_RULE_ERROR, + format!( + "non-hermetic custom rule at {}:{line}: import {} is not allowed", + path.str()?, + PyString::new(py, &module).repr()? + ), + )), + Err(UnparsedSource) => { + let keywords: Bound<'_, PyDict> = PyDict::new(py); + keywords.set_item("filename", path.str()?)?; + imported(py, AST_MODULE, "parse")?.call((source,), Some(&keywords))?; + Err(rules_error( + py, + RULES_ERROR, + format!( + "custom rule source could not be checked for imports: {}", + path.str()? + ), + )) + } + } +} + +fn utf8_encoding(py: Python<'_>) -> PyResult> { + let keywords: Bound<'_, PyDict> = PyDict::new(py); + keywords.set_item("encoding", "utf-8")?; + Ok(keywords) +} + +/// Reject imports outside the deterministic allowlist in every fingerprinted rule file. +#[pyfunction] +#[pyo3(signature = (*, rules, project_dir))] +pub(crate) fn verify_custom_rules( + rules: &Bound<'_, PyAny>, + project_dir: &Bound<'_, PyAny>, +) -> PyResult<()> { + let py: Python<'_> = rules.py(); + let root_path: Bound<'_, PyAny> = project_dir.call_method0("resolve")?; + let root: PathBuf = native_path(&root_path)?; + let checked: Bound<'_, PySet> = PySet::empty(py)?; + for rule in rules.try_iter()? { + let rule: Bound<'_, PyAny> = rule?; + if !rule.getattr("custom")?.is_truthy()? { + continue; + } + let Some(source) = resolved_source(&rule)? else { + continue; + }; + let closure: Vec = + import_closure_of(&native_path(&source)?, &root, python_version(py)); + let paths: Bound<'_, PyTuple> = closure_paths(py, closure)?; + for path in std::iter::once(source).chain(paths.iter()) { + if checked.contains(&path)? { + continue; + } + checked.add(&path)?; + verify_source(&path, &root)?; + } + } + Ok(()) +} + pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { + module.add_function(wrap_pyfunction!( + allowed_custom_rule_import_binding, + module + )?)?; + module.add_function(wrap_pyfunction!(verify_custom_rules, module)?)?; module.add_function(wrap_pyfunction!(custom_rule_import_closure, module)?)?; module.add_function(wrap_pyfunction!( custom_rule_implementation_fingerprint, diff --git a/crates/sqlbuild-python/src/bindings/_helpers/rules/mod.rs b/crates/sqlbuild-python/src/bindings/_helpers/rules/mod.rs index 4b10fcadae..fade28f354 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/rules/mod.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/rules/mod.rs @@ -27,6 +27,7 @@ pub(crate) mod registration; pub(crate) mod relation_keys; pub(crate) mod results; pub(crate) mod rows; +pub(crate) mod rule_api; pub(crate) mod rule_cache; pub(crate) mod rule_dispatch; pub(crate) mod rule_plan; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/rules/registration.rs b/crates/sqlbuild-python/src/bindings/_helpers/rules/registration.rs index efb60e8500..38120db142 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/rules/registration.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/rules/registration.rs @@ -7,11 +7,12 @@ use pyo3::{types::PyModuleMethods, wrap_pyfunction}; use crate::bindings::_helpers::rules::{ builtin_catalogue, catalogue, custom_rule_cache, custom_rule_evidence, dsl_headers, early_sql_lint, evaluation, guidance, lint, lint_expansion, lint_sql_bodies, lint_stop, - lint_text, results, rule_cache, rule_plan, rule_run, rules_config, run_lint, selection, - sql_rules, + lint_text, results, rule_api, rule_cache, rule_plan, rule_run, rules_config, run_lint, + selection, sql_rules, }; pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { + rule_api::register(module)?; module.add_function(wrap_pyfunction!( crate::bindings::_helpers::rules::lint_headers::lint_file_role, module diff --git a/crates/sqlbuild-python/src/bindings/_helpers/rules/rule_api.rs b/crates/sqlbuild-python/src/bindings/_helpers/rules/rule_api.rs new file mode 100644 index 0000000000..2ccf4ce6fc --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/rules/rule_api.rs @@ -0,0 +1,226 @@ +//! Custom-rule API decisions: finding positions, harness option TOML and project-tree paths. + +use std::path::PathBuf; + +use pyo3::exceptions::{PyAttributeError, PyOSError, PyValueError}; +use pyo3::prelude::{ + Bound, PyAny, PyAnyMethods, PyErr, PyModule, PyModuleMethods, PyResult, Python, +}; +use pyo3::types::{ + PyBool, PyBoolMethods, PyDict, PyDictMethods, PyInt, PyList, PyString, PyStringMethods, PyTuple, +}; +use pyo3::{IntoPyObject, pyfunction, wrap_pyfunction}; +use sqlbuild_config::toml::main::toml_literal::toml_literal; +use sqlbuild_config::toml::models::TomlLiteral; +use sqlbuild_core::posix_path::main::posix_escapes_root::posix_escapes_root; +use sqlbuild_rules::project_tree::errors::ProjectTreeFileError; +use sqlbuild_rules::project_tree::main::project_tree_entries::project_tree_entries; +use sqlbuild_rules::project_tree::main::project_tree_file::project_tree_file; +use sqlbuild_rules::project_tree::main::tracked_text_suffixes::tracked_text_suffixes; +use sqlbuild_rules::project_tree::models::ProjectTreeEntry; + +use crate::bindings::_helpers::compile_driver::python_calls::imported; + +const NATIVE_MODULE: &str = "sqlbuild._native"; +const RULE_USAGE_ERROR: &str = "RuleUsageError"; +const PROJECT_PATH_CLASS: &str = "ProjectPath"; +const BUILTINS_MODULE: &str = "builtins"; +const DEFAULT_POSITION: i64 = 1; +const OS_MODULE: &str = "os"; +/// The error `os.lstat` raises for a path holding a NUL byte. +const EMBEDDED_NUL_MESSAGE: &str = "lstat: embedded null character in path"; + +/// `RuleUsageError(message)`. +fn usage_error(py: Python<'_>, message: String) -> PyErr { + match imported(py, NATIVE_MODULE, RULE_USAGE_ERROR).and_then(|class| class.call1((message,))) { + Ok(error) => PyErr::from_value(error), + Err(error) => error, + } +} + +/// `RuleUsageError(prefix + str(value))`, keeping the value's text exactly as Python spells it. +fn usage_error_for(prefix: &str, value: &Bound<'_, PyAny>) -> PyErr { + let py: Python<'_> = value.py(); + let message = value + .str() + .and_then(|text| PyString::new(py, prefix).as_any().add(text)); + match message.and_then(|message| { + imported(py, NATIVE_MODULE, RULE_USAGE_ERROR).and_then(|class| class.call1((message,))) + }) { + Ok(error) => PyErr::from_value(error), + Err(error) => error, + } +} + +/// An explicit position, or `int(getattr(node, attribute, 1) or 1)`. +fn position<'py>( + explicit: &Bound<'py, PyAny>, + node: &Bound<'py, PyAny>, + attribute: &str, +) -> PyResult> { + if !explicit.is_none() { + return Ok(explicit.clone()); + } + let py: Python<'py> = node.py(); + let value: Bound<'py, PyAny> = match node.getattr(attribute) { + Ok(value) => value, + Err(error) if error.is_instance_of::(py) => { + DEFAULT_POSITION.into_pyobject(py)?.into_any() + } + Err(error) => return Err(error), + }; + let value: Bound<'py, PyAny> = if value.is_truthy()? { + value + } else { + DEFAULT_POSITION.into_pyobject(py)?.into_any() + }; + imported(py, BUILTINS_MODULE, "int")?.call1((value,)) +} + +/// The `(line, column)` of a finding at the project-relative `path`, from `node` when unset. +#[pyfunction] +#[pyo3(signature = (*, path, line, column, node))] +fn finding_position<'py>( + path: &Bound<'py, PyAny>, + line: &Bound<'py, PyAny>, + column: &Bound<'py, PyAny>, + node: &Bound<'py, PyAny>, +) -> PyResult<(Bound<'py, PyAny>, Bound<'py, PyAny>)> { + let py: Python<'py> = path.py(); + if posix_escapes_root(&path.str()?.to_string_lossy()) { + return Err(usage_error_for( + "finding path must be project-relative: ", + path, + )); + } + let line: Bound<'py, PyAny> = position(line, node, "line")?; + let column: Bound<'py, PyAny> = position(column, node, "column")?; + if line.lt(DEFAULT_POSITION)? || column.lt(DEFAULT_POSITION)? { + return Err(usage_error( + py, + "finding line and column must be positive".to_owned(), + )); + } + Ok((line, column)) +} + +/// One rule-option value as a TOML literal, or the harness's usage error. +fn option_literal(value: &Bound<'_, PyAny>) -> PyResult { + if let Ok(flag) = value.downcast::() { + return Ok(TomlLiteral::Bool(flag.is_true())); + } + if value.is_instance_of::() { + return Ok(TomlLiteral::Integer(value.str()?.to_string())); + } + if let Ok(text) = value.downcast::() { + return Ok(TomlLiteral::Text(text.to_str()?.to_owned())); + } + if value.is_instance_of::() || value.is_instance_of::() { + return Ok(TomlLiteral::Array( + value + .try_iter()? + .map(|item| option_literal(&item?)) + .collect::>()?, + )); + } + Err(usage_error( + value.py(), + format!( + "RuleCase config value {} cannot be represented as a rule option", + value.repr()? + ), + )) +} + +/// The TOML text the rule harness writes for one `RuleCase` option value. +#[pyfunction] +fn rule_option_toml_value(value: &Bound<'_, PyAny>) -> PyResult { + Ok(toml_literal(&option_literal(value)?)) +} + +/// Reject a project path that is absolute or climbs out of the project. +#[pyfunction] +fn validate_project_relative(path: &Bound<'_, PyAny>) -> Result<(), PyErr> { + if posix_escapes_root(&path.str()?.to_string_lossy()) { + return Err(usage_error_for("project path escapes the project: ", path)); + } + Ok(()) +} + +/// Every supported path under the resolved project `root`, as sorted `ProjectPath`s. +#[pyfunction] +fn project_tree_paths<'py>(root: &Bound<'py, PyAny>) -> PyResult> { + let py: Python<'py> = root.py(); + let root: PathBuf = root.extract()?; + let class: Bound<'py, PyAny> = imported(py, NATIVE_MODULE, PROJECT_PATH_CLASS)?; + let mut paths: Vec<(Bound<'py, PyAny>, bool)> = project_tree_entries(&root) + .into_iter() + .map(|ProjectTreeEntry { relative, is_file }| { + Ok(( + relative.into_os_string().into_pyobject(py)?.into_any(), + is_file, + )) + }) + .collect::>()?; + let mut failure: Option = None; + paths.sort_by(|left, right| { + left.0.compare(&right.0).unwrap_or_else(|error| { + failure.get_or_insert(error); + std::cmp::Ordering::Equal + }) + }); + if let Some(error) = failure { + return Err(error); + } + let nodes: Vec> = paths + .into_iter() + .map(|(value, is_file)| { + let keywords: Bound<'py, PyDict> = PyDict::new(py); + keywords.set_item("value", value)?; + keywords.set_item("is_file", is_file)?; + keywords.set_item("is_dir", !is_file)?; + class.call((), Some(&keywords)) + }) + .collect::>()?; + PyTuple::new(py, nodes) +} + +/// The real file a rule may read at the project-relative `path` under the resolved `root`. +#[pyfunction] +fn project_tree_file_path<'py>( + root: &Bound<'py, PyAny>, + path: &Bound<'py, PyAny>, +) -> PyResult> { + let py: Python<'py> = root.py(); + let relative: PathBuf = path.str()?.extract()?; + let prefix: String = match project_tree_file(&root.extract::()?, &relative) { + Ok(file) => return Ok(file.into_os_string().into_pyobject(py)?.into_any()), + Err(ProjectTreeFileError::Escapes) => "project path escapes the project: ".to_owned(), + Err(ProjectTreeFileError::EmbeddedNul) => { + return Err(PyValueError::new_err(EMBEDDED_NUL_MESSAGE)); + } + Err(ProjectTreeFileError::Missing) => "project file does not exist: ".to_owned(), + Err(ProjectTreeFileError::Untracked) => format!( + "compiler rules input must be a tracked file type ({}): ", + tracked_text_suffixes().join(", ") + ), + Err(ProjectTreeFileError::Symlink) => "project file must not be a symlink: ".to_owned(), + Err(ProjectTreeFileError::Unreadable { path, errno }) => { + let message: String = imported(py, OS_MODULE, "strerror")? + .call1((errno,))? + .extract()?; + let Ok(path) = path.into_os_string().into_pyobject(py); + return Err(PyOSError::new_err((errno, message, path.unbind()))); + } + }; + Err(usage_error_for(&prefix, path)) +} + +pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { + module.add_function(wrap_pyfunction!(finding_position, module)?)?; + module.add_function(wrap_pyfunction!(rule_option_toml_value, module)?)?; + module.add_function(wrap_pyfunction!(validate_project_relative, module)?)?; + module.add_function(wrap_pyfunction!(project_tree_paths, module)?)?; + module.add_function(wrap_pyfunction!(project_tree_file_path, module)?)?; + Ok(()) +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/rules/rule_dispatch.rs b/crates/sqlbuild-python/src/bindings/_helpers/rules/rule_dispatch.rs index 3dbea99f83..9ef1718718 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/rules/rule_dispatch.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/rules/rule_dispatch.rs @@ -23,13 +23,13 @@ use crate::bindings::_helpers::rules::custom_evaluation::{ CustomRun, evaluate_custom_rules_cached, outcome, }; use crate::bindings::_helpers::rules::custom_hosts::HostCancel; +use crate::bindings::_helpers::rules::custom_rule_evidence::verify_custom_rules; const NATIVE_RECORDS_MODULE: &str = "sqlbuild._native"; const NATIVE_MODULE: &str = "sqlbuild._native"; const RULES_MODELS_MODULE: &str = "sqlbuild._native"; const CUSTOM_PAYLOADS_MODULE: &str = "sqlbuild.rule_engine._helpers.engine.custom_payloads"; -const HERMETICITY_MODULE: &str = "sqlbuild.rule_engine._helpers.engine.hermeticity"; const CUSTOM_THREAD: &str = "sqlbuild-custom-rules"; const GENERIC_DIALECT: &str = "generic"; const PROJECT_RULES_OPTIONS: (&[&str], &[&str]) = ( @@ -164,10 +164,7 @@ fn evaluate_project_rules<'py>( ("project_dir", project_dir.clone()), ], )?; - call_with( - &imported(py, HERMETICITY_MODULE, "verify_custom_rules")?, - &[("rules", rules), ("project_dir", project_dir.clone())], - )?; + verify_custom_rules(&rules, &project_dir)?; } let run: NativeRun<'py> = NativeRun { project, diff --git a/crates/sqlbuild-python/src/bindings/_helpers/type_system/mod.rs b/crates/sqlbuild-python/src/bindings/_helpers/type_system/mod.rs index 30a05c9b4f..7070281c01 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/type_system/mod.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/type_system/mod.rs @@ -1,3 +1,4 @@ pub(crate) mod equality; pub(crate) mod normalization; +pub(crate) mod nullability; pub(crate) mod registration; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/type_system/nullability.rs b/crates/sqlbuild-python/src/bindings/_helpers/type_system/nullability.rs new file mode 100644 index 0000000000..5ca8f2ccff --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/type_system/nullability.rs @@ -0,0 +1,75 @@ +//! Adapter nullability rules over `InferredNullability` arguments, decided by the native rules. + +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyModule, PyModuleMethods, PyResult}; +use pyo3::types::PyString; +use pyo3::{pyfunction, wrap_pyfunction}; +use sqlbuild_analysis::nullability_rules::main::conditional_result_nullability::conditional_result_nullability; +use sqlbuild_analysis::nullability_rules::main::safe_cast_nullability::safe_cast_nullability; + +use crate::bindings::_helpers::compile_driver::python_calls::imported; + +const LINEAGE_TYPES_MODULE: &str = "sqlbuild.compiler.lineage.types"; +const NULLABILITY_CLASS: &str = "InferredNullability"; +const UNKNOWN_MEMBER: &str = "UNKNOWN"; + +/// The `InferredNullability` member whose value is `value`. +fn member<'py>(args: &Bound<'py, PyAny>, value: &str) -> PyResult> { + imported(args.py(), LINEAGE_TYPES_MODULE, NULLABILITY_CLASS)?.call1((value,)) +} + +/// The text of one argument; an argument that is not a string matches no state. +fn argument_text(value: &Bound<'_, PyAny>) -> String { + match value.downcast::() { + Ok(text) => text.to_string(), + Err(_) => String::new(), + } +} + +/// The text of the first `count` arguments, read by index. +fn argument_values(args: &Bound<'_, PyAny>, count: usize) -> PyResult> { + (0..count) + .map(|index| Ok(argument_text(&args.get_item(index)?))) + .collect() +} + +/// The first argument itself, or `InferredNullability.UNKNOWN` without arguments. +#[pyfunction] +fn infer_first_arg_nullability<'py>(args: &Bound<'py, PyAny>) -> PyResult> { + if !args.is_truthy()? { + return imported(args.py(), LINEAGE_TYPES_MODULE, NULLABILITY_CLASS)? + .getattr(UNKNOWN_MEMBER); + } + args.get_item(0) +} + +/// IF/IFF-style nullability from the true and false result arguments. +#[pyfunction] +fn infer_conditional_result_nullability<'py>( + args: &Bound<'py, PyAny>, +) -> PyResult> { + let values: Vec = argument_values(args, args.len()?)?; + let values: Vec<&str> = values.iter().map(String::as_str).collect(); + member(args, conditional_result_nullability(&values)) +} + +/// SAFE_CAST/TRY_CAST nullability without assuming the conversion succeeds. +#[pyfunction] +fn infer_safe_cast_nullability<'py>(args: &Bound<'py, PyAny>) -> PyResult> { + let values: Vec = if args.is_truthy()? { + argument_values(args, 1)? + } else { + Vec::new() + }; + let values: Vec<&str> = values.iter().map(String::as_str).collect(); + member(args, safe_cast_nullability(&values)) +} + +pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { + module.add_function(wrap_pyfunction!(infer_first_arg_nullability, module)?)?; + module.add_function(wrap_pyfunction!( + infer_conditional_result_nullability, + module + )?)?; + module.add_function(wrap_pyfunction!(infer_safe_cast_nullability, module)?)?; + Ok(()) +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/type_system/registration.rs b/crates/sqlbuild-python/src/bindings/_helpers/type_system/registration.rs index b1afbad36f..1077f61865 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/type_system/registration.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/type_system/registration.rs @@ -2,9 +2,10 @@ use pyo3::prelude::{Bound, PyModule, PyResult}; -use crate::bindings::_helpers::type_system::normalization; +use crate::bindings::_helpers::type_system::{normalization, nullability}; pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { normalization::register(module)?; + nullability::register(module)?; Ok(()) } diff --git a/crates/sqlbuild-rules/src/lib.rs b/crates/sqlbuild-rules/src/lib.rs index 7c4bd3a4cb..685595fba4 100644 --- a/crates/sqlbuild-rules/src/lib.rs +++ b/crates/sqlbuild-rules/src/lib.rs @@ -12,6 +12,7 @@ pub mod errors; pub mod fix_verification; pub mod lint_text; pub mod models; +pub mod project_tree; pub mod rule_cache; pub mod rule_catalogue; pub mod rule_results; diff --git a/crates/sqlbuild-rules/src/project_tree/_helpers/entries.rs b/crates/sqlbuild-rules/src/project_tree/_helpers/entries.rs new file mode 100644 index 0000000000..c66261b5d2 --- /dev/null +++ b/crates/sqlbuild-rules/src/project_tree/_helpers/entries.rs @@ -0,0 +1,28 @@ +//! How the project walk classifies and filters directory entries. + +use std::fs::DirEntry; +use std::os::unix::ffi::OsStrExt; + +use crate::project_tree::constants::{EXCLUDED_ROOTS, HIDDEN_PREFIX}; +use crate::project_tree::models::EntryKind; + +/// How the walk treats one directory entry, without following links. +pub(crate) fn kind(entry: &DirEntry) -> EntryKind { + match entry.file_type() { + Ok(file_type) if file_type.is_symlink() => EntryKind::Link, + Ok(file_type) if file_type.is_dir() => EntryKind::Directory, + _ => EntryKind::Other, + } +} + +/// Whether the walk lists and enters a directory entry. +pub(crate) fn listed_directory(entry: &DirEntry, at_root: bool) -> bool { + let name = entry.file_name(); + if name.as_bytes().first() == Some(&HIDDEN_PREFIX) { + return false; + } + !(at_root + && EXCLUDED_ROOTS + .iter() + .any(|excluded| name.as_bytes() == excluded.as_bytes())) +} diff --git a/crates/sqlbuild-rules/src/project_tree/_helpers/files.rs b/crates/sqlbuild-rules/src/project_tree/_helpers/files.rs new file mode 100644 index 0000000000..a0a2ae5f23 --- /dev/null +++ b/crates/sqlbuild-rules/src/project_tree/_helpers/files.rs @@ -0,0 +1,64 @@ +//! Resolution of one project-relative path to a real, readable project file. + +use std::ffi::OsStr; +use std::os::unix::ffi::OsStrExt; +use std::path::{Path, PathBuf}; + +use sqlbuild_core::posix_path::main::posix_suffix::posix_suffix; + +use crate::project_tree::constants::{MISSING_FILE_ERRNOS, TRACKED_TEXT_SUFFIXES}; +use crate::project_tree::errors::ProjectTreeFileError; + +/// `root` joined with the raw path components. +pub(crate) fn joined(root: &Path, parts: &[&[u8]]) -> PathBuf { + let mut path: PathBuf = root.to_path_buf(); + path.extend(parts.iter().map(|part| OsStr::from_bytes(part))); + path +} + +/// `Path.is_file()`: false for the failures `pathlib` ignores, the failure otherwise. +pub(crate) fn points_to_file(path: &Path) -> Result { + match std::fs::metadata(path) { + Ok(metadata) => Ok(metadata.is_file()), + Err(error) => match error.raw_os_error() { + Some(errno) if !MISSING_FILE_ERRNOS.contains(&errno) => { + Err(ProjectTreeFileError::Unreadable { + path: path.to_path_buf(), + errno, + }) + } + _ => Ok(false), + }, + } +} + +/// A regular, non-link file under the project's resolved parent directory of the path. +pub(crate) fn regular_file(root: &Path, parts: &[&[u8]]) -> Option { + let (name, parent) = parts.split_last()?; + let directory: PathBuf = match std::fs::canonicalize(joined(root, parent)) { + Ok(directory) => directory, + Err(_unresolved) => return None, + }; + if !directory.starts_with(root) { + return None; + } + let candidate: PathBuf = directory.join(OsStr::from_bytes(name)); + match std::fs::symlink_metadata(&candidate) { + Ok(metadata) => metadata.file_type().is_file().then_some(candidate), + Err(_unreadable) => None, + } +} + +/// `candidate` when its lowercased suffix is a tracked text type. +pub(crate) fn tracked(candidate: PathBuf) -> Result { + let name: String = candidate + .file_name() + .map(|name| name.to_string_lossy().into_owned()) + .unwrap_or_default(); + let suffix: String = posix_suffix(&name).to_lowercase(); + if TRACKED_TEXT_SUFFIXES.contains(&suffix.as_str()) { + Ok(candidate) + } else { + Err(ProjectTreeFileError::Untracked) + } +} diff --git a/crates/sqlbuild-rules/src/project_tree/_helpers/mod.rs b/crates/sqlbuild-rules/src/project_tree/_helpers/mod.rs new file mode 100644 index 0000000000..ad40bd0d59 --- /dev/null +++ b/crates/sqlbuild-rules/src/project_tree/_helpers/mod.rs @@ -0,0 +1,3 @@ +pub(crate) mod entries; +pub(crate) mod files; +pub(crate) mod relative_paths; diff --git a/crates/sqlbuild-rules/src/project_tree/_helpers/relative_paths.rs b/crates/sqlbuild-rules/src/project_tree/_helpers/relative_paths.rs new file mode 100644 index 0000000000..5cb7efab91 --- /dev/null +++ b/crates/sqlbuild-rules/src/project_tree/_helpers/relative_paths.rs @@ -0,0 +1,20 @@ +//! `PurePosixPath` reads of a project-relative path over its raw bytes. + +use std::os::unix::ffi::OsStrExt; +use std::path::Path; + +use crate::project_tree::constants::{CURRENT, PARENT, SEPARATOR}; + +/// The components `PurePosixPath(path).parts` keeps after any root. +pub(crate) fn parts(path: &Path) -> Vec<&[u8]> { + path.as_os_str() + .as_bytes() + .split(|byte| *byte == SEPARATOR) + .filter(|part| !part.is_empty() && *part != CURRENT) + .collect() +} + +/// `PurePosixPath(path).is_absolute() or ".." in PurePosixPath(path).parts`. +pub(crate) fn escapes(path: &Path) -> bool { + path.as_os_str().as_bytes().first() == Some(&SEPARATOR) || parts(path).contains(&PARENT) +} diff --git a/crates/sqlbuild-rules/src/project_tree/constants.rs b/crates/sqlbuild-rules/src/project_tree/constants.rs new file mode 100644 index 0000000000..76e93fe745 --- /dev/null +++ b/crates/sqlbuild-rules/src/project_tree/constants.rs @@ -0,0 +1,16 @@ +//! Directories the project tree skips and the file types rules may read. + +/// Top-level directories that are tooling output or environments, never project sources. +pub(crate) const EXCLUDED_ROOTS: [&str; 5] = [".git", ".venv", "logs", "target", "venv"]; +/// The lowercase suffixes of the text files rules may read, sorted. +pub(crate) const TRACKED_TEXT_SUFFIXES: [&str; 5] = [".py", ".sql", ".toml", ".yaml", ".yml"]; +/// The leading byte of a hidden directory name. +pub(crate) const HIDDEN_PREFIX: u8 = b'.'; +/// `errno` values `pathlib` treats as a missing file when it tests `is_file()`. +#[cfg(target_os = "macos")] +pub(crate) const MISSING_FILE_ERRNOS: [i32; 4] = [2, 20, 9, 62]; +#[cfg(not(target_os = "macos"))] +pub(crate) const MISSING_FILE_ERRNOS: [i32; 4] = [2, 20, 9, 40]; +pub(crate) const SEPARATOR: u8 = b'/'; +pub(crate) const CURRENT: &[u8] = b"."; +pub(crate) const PARENT: &[u8] = b".."; diff --git a/crates/sqlbuild-rules/src/project_tree/errors.rs b/crates/sqlbuild-rules/src/project_tree/errors.rs new file mode 100644 index 0000000000..155a903fa1 --- /dev/null +++ b/crates/sqlbuild-rules/src/project_tree/errors.rs @@ -0,0 +1,20 @@ +//! Why a project-relative path is not a file rules may read. + +use std::path::PathBuf; + +/// The reason a project path cannot be read through the project tree. +#[derive(Clone, Debug, PartialEq, Eq)] +pub enum ProjectTreeFileError { + /// The path is absolute or climbs out with `..`. + Escapes, + /// The path holds a NUL byte, which no file name can. + EmbeddedNul, + /// No regular file exists at the path inside the project. + Missing, + /// The file is not one of the tracked text types. + Untracked, + /// The path resolves through a symbolic link. + Symlink, + /// Testing the resolved path failed with this `errno`. + Unreadable { path: PathBuf, errno: i32 }, +} diff --git a/crates/sqlbuild-rules/src/project_tree/main/mod.rs b/crates/sqlbuild-rules/src/project_tree/main/mod.rs new file mode 100644 index 0000000000..fa8c2bcca2 --- /dev/null +++ b/crates/sqlbuild-rules/src/project_tree/main/mod.rs @@ -0,0 +1,3 @@ +pub mod project_tree_entries; +pub mod project_tree_file; +pub mod tracked_text_suffixes; diff --git a/crates/sqlbuild-rules/src/project_tree/main/project_tree_entries.rs b/crates/sqlbuild-rules/src/project_tree/main/project_tree_entries.rs new file mode 100644 index 0000000000..ed57e4aa4e --- /dev/null +++ b/crates/sqlbuild-rules/src/project_tree/main/project_tree_entries.rs @@ -0,0 +1,38 @@ +//! Every file and directory under a project, as `os.walk` finds them without following links. + +use std::path::{Path, PathBuf}; + +use crate::project_tree::_helpers::entries::{kind, listed_directory}; +use crate::project_tree::models::{EntryKind, ProjectTreeEntry}; + +/// Files and directories under `root`, without links, hidden or top-level tooling directories. +#[must_use] +pub fn project_tree_entries(root: &Path) -> Vec { + let mut entries: Vec = Vec::new(); + let mut pending: Vec = vec![PathBuf::new()]; + while let Some(relative) = pending.pop() { + let Ok(listing) = std::fs::read_dir(root.join(&relative)) else { + continue; + }; + for entry in listing.flatten() { + let child: PathBuf = relative.join(entry.file_name()); + match kind(&entry) { + EntryKind::Link => {} + EntryKind::Directory => { + if listed_directory(&entry, relative.as_os_str().is_empty()) { + entries.push(ProjectTreeEntry { + relative: child.clone(), + is_file: false, + }); + pending.push(child); + } + } + EntryKind::Other => entries.push(ProjectTreeEntry { + relative: child, + is_file: true, + }), + } + } + } + entries +} diff --git a/crates/sqlbuild-rules/src/project_tree/main/project_tree_file.rs b/crates/sqlbuild-rules/src/project_tree/main/project_tree_file.rs new file mode 100644 index 0000000000..10c97ad871 --- /dev/null +++ b/crates/sqlbuild-rules/src/project_tree/main/project_tree_file.rs @@ -0,0 +1,32 @@ +//! The real file a project-relative path names, when rules may read it. + +use std::os::unix::ffi::OsStrExt; +use std::path::{Path, PathBuf}; + +use crate::project_tree::_helpers::files::{joined, points_to_file, regular_file, tracked}; +use crate::project_tree::_helpers::relative_paths::{escapes, parts}; +use crate::project_tree::errors::ProjectTreeFileError; + +/// The tracked file inside the resolved `root` that the project-relative `path` names. +pub fn project_tree_file(root: &Path, path: &Path) -> Result { + if escapes(path) { + return Err(ProjectTreeFileError::Escapes); + } + if path.as_os_str().as_bytes().contains(&0) { + return Err(ProjectTreeFileError::EmbeddedNul); + } + let parts: Vec<&[u8]> = parts(path); + if let Some(regular) = regular_file(root, &parts) { + return tracked(regular); + } + let lexical: PathBuf = joined(root, &parts); + let candidate: PathBuf = std::fs::canonicalize(&lexical).unwrap_or(lexical); + if !candidate.starts_with(root) || !points_to_file(&candidate)? { + return Err(ProjectTreeFileError::Missing); + } + let candidate: PathBuf = tracked(candidate)?; + if candidate.is_symlink() { + return Err(ProjectTreeFileError::Symlink); + } + Ok(candidate) +} diff --git a/crates/sqlbuild-rules/src/project_tree/main/tracked_text_suffixes.rs b/crates/sqlbuild-rules/src/project_tree/main/tracked_text_suffixes.rs new file mode 100644 index 0000000000..1219bdb01f --- /dev/null +++ b/crates/sqlbuild-rules/src/project_tree/main/tracked_text_suffixes.rs @@ -0,0 +1,9 @@ +//! The file types rules may read, as listed in errors. + +use crate::project_tree::constants::TRACKED_TEXT_SUFFIXES; + +/// The tracked suffixes in sorted order. +#[must_use] +pub fn tracked_text_suffixes() -> &'static [&'static str] { + &TRACKED_TEXT_SUFFIXES +} diff --git a/crates/sqlbuild-rules/src/project_tree/mod.rs b/crates/sqlbuild-rules/src/project_tree/mod.rs new file mode 100644 index 0000000000..67a8c068b8 --- /dev/null +++ b/crates/sqlbuild-rules/src/project_tree/mod.rs @@ -0,0 +1,10 @@ +//! The deterministic project-tree view custom rules read: discovered paths and readable files. + +pub(crate) mod _helpers; +pub(crate) mod constants; +pub mod errors; +pub mod main; +pub mod models; + +#[cfg(test)] +mod tests; diff --git a/crates/sqlbuild-rules/src/project_tree/models.rs b/crates/sqlbuild-rules/src/project_tree/models.rs new file mode 100644 index 0000000000..7a5e0f5a13 --- /dev/null +++ b/crates/sqlbuild-rules/src/project_tree/models.rs @@ -0,0 +1,17 @@ +//! One discovered project path. + +use std::path::PathBuf; + +/// A project-relative file or directory the tree lists. +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct ProjectTreeEntry { + pub relative: PathBuf, + pub is_file: bool, +} + +/// How the walk treats one directory entry. +pub(crate) enum EntryKind { + Link, + Directory, + Other, +} diff --git a/crates/sqlbuild-rules/src/project_tree/tests.rs b/crates/sqlbuild-rules/src/project_tree/tests.rs new file mode 100644 index 0000000000..6993fa3916 --- /dev/null +++ b/crates/sqlbuild-rules/src/project_tree/tests.rs @@ -0,0 +1,6 @@ +#[path = "tests/helpers.rs"] +mod helpers; +#[path = "tests/test_project_tree.rs"] +mod project_tree; +#[path = "tests/test_types.rs"] +mod test_types; diff --git a/crates/sqlbuild-rules/src/project_tree/tests/helpers.rs b/crates/sqlbuild-rules/src/project_tree/tests/helpers.rs new file mode 100644 index 0000000000..75da32afaa --- /dev/null +++ b/crates/sqlbuild-rules/src/project_tree/tests/helpers.rs @@ -0,0 +1,24 @@ +use std::os::unix::fs::symlink; +use std::path::PathBuf; + +/// A project with nested, hidden, tooling and linked paths, and its resolved root. +pub(crate) fn linked_project() -> (tempfile::TempDir, PathBuf) { + let directory = tempfile::tempdir().expect("temporary project"); + let root: PathBuf = std::fs::canonicalize(directory.path()).expect("resolved root"); + for folder in ["models/marts", "models/.hidden", "logs", "sub/logs"] { + std::fs::create_dir_all(root.join(folder)).expect("folder"); + } + for file in [ + "models/marts/orders.sql", + "models/.dot.sql", + "models/.hidden/x.sql", + ] { + std::fs::write(root.join(file), "select 1\n").expect("file"); + } + std::fs::write(root.join("readme.md"), "# r\n").expect("readme"); + std::fs::write(root.join("sub/logs/kept.sql"), "k").expect("kept"); + symlink("marts/orders.sql", root.join("models/link.sql")).expect("link"); + symlink("marts", root.join("models/linkdir")).expect("dir link"); + symlink("../readme.md", root.join("models/readme.sql")).expect("readme link"); + (directory, root) +} diff --git a/crates/sqlbuild-rules/src/project_tree/tests/test_project_tree.rs b/crates/sqlbuild-rules/src/project_tree/tests/test_project_tree.rs new file mode 100644 index 0000000000..7ec13537db --- /dev/null +++ b/crates/sqlbuild-rules/src/project_tree/tests/test_project_tree.rs @@ -0,0 +1,113 @@ +use std::path::Path; + +use crate::project_tree::errors::ProjectTreeFileError; +use crate::project_tree::main::project_tree_entries::project_tree_entries; +use crate::project_tree::main::project_tree_file::project_tree_file; +use crate::project_tree::tests::helpers::linked_project; +use crate::project_tree::tests::test_types::{ProjectTreeEntriesTestCase, ProjectTreeFileTestCase}; + +#[test] +fn given_project_when_walked_then_links_hidden_and_root_tooling_directories_are_skipped() { + let test_cases = [ProjectTreeEntriesTestCase { + description: "links, hidden directories and top-level logs are left out", + expected_entries: vec![ + ("models", false), + ("models/.dot.sql", true), + ("models/marts", false), + ("models/marts/orders.sql", true), + ("readme.md", true), + ("sub", false), + ("sub/logs", false), + ("sub/logs/kept.sql", true), + ], + }]; + for test_case in test_cases { + let (_directory, root) = linked_project(); + let mut entries: Vec<(String, bool)> = project_tree_entries(&root) + .into_iter() + .map(|entry| (entry.relative.to_string_lossy().into_owned(), entry.is_file)) + .collect(); + entries.sort(); + let entries: Vec<(&str, bool)> = entries + .iter() + .map(|(path, is_file)| (path.as_str(), *is_file)) + .collect(); + assert_eq!( + entries, test_case.expected_entries, + "{}", + test_case.description + ); + } +} + +#[test] +fn given_project_paths_when_resolved_then_only_tracked_files_inside_the_project_are_readable() { + let test_cases = [ + ProjectTreeFileTestCase { + description: "a regular file", + path: "models/marts/orders.sql", + expected_file: Ok("models/marts/orders.sql"), + }, + ProjectTreeFileTestCase { + description: "current, empty and trailing components", + path: "./models//marts/orders.sql/", + expected_file: Ok("models/marts/orders.sql"), + }, + ProjectTreeFileTestCase { + description: "a linked file resolves to its target", + path: "models/link.sql", + expected_file: Ok("models/marts/orders.sql"), + }, + ProjectTreeFileTestCase { + description: "a file under a linked directory", + path: "models/linkdir/orders.sql", + expected_file: Ok("models/marts/orders.sql"), + }, + ProjectTreeFileTestCase { + description: "a link to an untracked file", + path: "models/readme.sql", + expected_file: Err(ProjectTreeFileError::Untracked), + }, + ProjectTreeFileTestCase { + description: "an untracked file type", + path: "readme.md", + expected_file: Err(ProjectTreeFileError::Untracked), + }, + ProjectTreeFileTestCase { + description: "a missing file", + path: "missing.sql", + expected_file: Err(ProjectTreeFileError::Missing), + }, + ProjectTreeFileTestCase { + description: "a directory", + path: "models", + expected_file: Err(ProjectTreeFileError::Missing), + }, + ProjectTreeFileTestCase { + description: "a path above the project", + path: "../outside.sql", + expected_file: Err(ProjectTreeFileError::Escapes), + }, + ProjectTreeFileTestCase { + description: "a NUL byte", + path: "a\0b.sql", + expected_file: Err(ProjectTreeFileError::EmbeddedNul), + }, + ]; + for test_case in test_cases { + let (_directory, root) = linked_project(); + let resolved: Result = + project_tree_file(&root, Path::new(test_case.path)).map(|file| { + file.strip_prefix(&root) + .unwrap_or(&file) + .to_string_lossy() + .into_owned() + }); + assert_eq!( + resolved.as_ref().map(String::as_str).map_err(Clone::clone), + test_case.expected_file, + "{}", + test_case.description + ); + } +} diff --git a/crates/sqlbuild-rules/src/project_tree/tests/test_types.rs b/crates/sqlbuild-rules/src/project_tree/tests/test_types.rs new file mode 100644 index 0000000000..f37eccfcac --- /dev/null +++ b/crates/sqlbuild-rules/src/project_tree/tests/test_types.rs @@ -0,0 +1,13 @@ +use crate::project_tree::errors::ProjectTreeFileError; + +pub(crate) struct ProjectTreeEntriesTestCase { + pub(crate) description: &'static str, + pub(crate) expected_entries: Vec<(&'static str, bool)>, +} + +pub(crate) struct ProjectTreeFileTestCase { + pub(crate) description: &'static str, + pub(crate) path: &'static str, + /// The project-relative file read, or why none may be. + pub(crate) expected_file: Result<&'static str, ProjectTreeFileError>, +} diff --git a/crates/sqlbuild-rules/src/rules/_helpers/evaluation.rs b/crates/sqlbuild-rules/src/rules/_helpers/evaluation.rs index 50729ded5e..339a58906f 100644 --- a/crates/sqlbuild-rules/src/rules/_helpers/evaluation.rs +++ b/crates/sqlbuild-rules/src/rules/_helpers/evaluation.rs @@ -15,6 +15,7 @@ use crate::rules::models::{ FaultCollector, ModelEvaluationRequest, ProjectEvaluationRequest, ResolvedThresholdOverride, }; use globset::{Glob, GlobSetBuilder}; +use sqlbuild_core::identifier_text::main::is_lowercase_identifier::is_lowercase_identifier; use sqlbuild_sqltext::sql_scan::main::matching_paren::matching_paren; use sqlbuild_sqltext::sql_scan::main::non_code_end::non_code_end; use sqlbuild_sqltext::sql_scan::main::quote_policy::quote_policy as rules_quote_policy; @@ -972,12 +973,12 @@ pub(super) fn parse_name(name: &str) -> Option { "mart", "mart_v", ]; - if !valid_lower_name_token(parts[0]) + if !is_lowercase_identifier(parts[0]) || !layers.contains(&parts[1]) - || !valid_lower_name_token(parts[2]) + || !is_lowercase_identifier(parts[2]) || parts .get(3) - .is_some_and(|value| !valid_lower_name_token(value)) + .is_some_and(|value| !is_lowercase_identifier(value)) { return None; } @@ -990,12 +991,6 @@ pub(super) fn parse_name(name: &str) -> Option { }) } -fn valid_lower_name_token(value: &str) -> bool { - let mut bytes = value.bytes(); - bytes.next().is_some_and(|byte| byte.is_ascii_lowercase()) - && bytes.all(|byte| byte.is_ascii_lowercase() || byte.is_ascii_digit() || byte == b'_') -} - fn view_marker(parsed: &ParsedModel<'_>, rule: &RuleMetadata, faults: &FaultCollector) { let Some(parts) = parse_name(&parsed.model.name) else { return; diff --git a/python-boundary.toml b/python-boundary.toml index eb97c109a2..7acec6e0b6 100644 --- a/python-boundary.toml +++ b/python-boundary.toml @@ -35,7 +35,7 @@ note = "Relation inspection wrappers adapters call, with per-connection caches; [[stays]] path = "adapter/type_system" rule = 1 -note = "Type normalisation adapters import; already a thin native wrapper." +note = "Type normalisation and the public nullability rule callables adapters import and register (profile_inputs.rs recognises them by path); normalisation and the first_arg, conditional_result and safe_cast rules are native." [[stays]] path = "provider" @@ -45,7 +45,7 @@ note = "Provider base class users subclass, and injection into user functions." [[stays]] path = "python_nodes" rule = 3 -note = "Decorators that run when user modules are imported." +note = "Public decorators and authoring shapes re-exported by assets.py, tasks.py, loaders.py, hooks.py and retries.py, run when user modules are imported; RetryPolicy and AuditCase validation, retry delays and hook-read normalization are native (validate_retry_policy, retry_delay_seconds, validate_audit_case, normalize_hook_reads)." [[stays]] path = "microbatches" @@ -115,22 +115,17 @@ note = "Imports the project's custom rule modules and collects their @rule funct [[stays]] path = "rule_engine/classes/rule_context.py" rule = 3 -note = "The context object user rules receive." +note = "The context object user rules receive; finding path and position validation is native (finding_position)." [[stays]] path = "rule_engine/_helpers/engine/definition.py" rule = 2 note = "The @rule decorator implementation; inspects user rule functions and their signatures." -[[stays]] -path = "rule_engine/_helpers/engine/hermeticity.py" -rule = 2 -note = "Statically checks the imports of user rule source files." - [[stays]] path = "rule_engine/_helpers/engine/rule_harness.py" rule = 2 -note = "Public custom-rule test harness that runs user rules over a real compiled project." +note = "Public custom-rule test harness that writes a temporary project and runs user rules over it; rule-option TOML literals are native (rule_option_toml_value)." [[stays]] path = "rule_engine/_helpers/engine/fact_replay.py" @@ -190,7 +185,7 @@ note = "Fact view user rules read through RuleContext; the Python API over nativ [[stays]] path = "rule_engine/classes/project_tree.py" rule = 3 -note = "Project-tree fact view user rules read through RuleContext." +note = "Project-tree fact view user rules read through RuleContext; the walk, path validation and file resolution are native (project_tree_paths, project_tree_file_path, validate_project_relative) and only the file read stays here." [[stays]] path = "rule_engine/classes/sql_document.py" @@ -370,7 +365,7 @@ note = "Rule 3 shapes: the command-output sink parameter name and the installed [[stays]] path = "integrations/dagster" rule = 4 -note = "Documented orchestrator integration and rule 4 Dagster glue: translator invocation, context reads, subprocess streaming and Dagster event construction; CLI arguments, selector logs, payload and failure parsing, asset/check/selection projection of the DAG artifact are native." +note = "Documented orchestrator integration and rule 4 Dagster glue: overridable translator methods, translator invocation, context reads, subprocess streaming and Dagster event construction; CLI arguments, selector logs, payload and failure parsing, asset/check/selection projection of the DAG artifact and the default translator mappings (dagster_default_*) are native." [[stays]] path = "integrations/rivers" @@ -415,7 +410,7 @@ note = "Inspects user provider settings classes for the environment and files th [[stays]] path = "cli/compile_reuse/_helpers/startup_inputs.py" rule = 2 -note = "Interpreter and site-path inputs user modules import from." +note = "Reads the live interpreter's sys.path, site-packages, executable and prefixes that user modules import from; absolute paths, .pth discovery, ordering and stamps are native (startup_input_stamps)." [[stays]] path = "cli/entry/main/entry.py" @@ -715,7 +710,7 @@ note = "Shape rule: data enums and aliases only (CostCapability, CostStatus), no [[stays]] path = "compiler/lineage/types.py" rule = 1 -note = "Shape rule: data enums and aliases only (InferredNullability), no logic; consumed by adapter/contract/types.py, adapter/type_system/_helpers/inference_rules.py, adapter/type_system/main/conditional_result_nullability.py and 2 more." +note = "Shape rule: data enums and aliases only (InferredNullability), no logic; consumed by adapter/contract/types.py, adapter/type_system/main/conditional_result_nullability.py, adapter/type_system/main/first_arg_nullability.py and adapter/type_system/main/safe_cast_nullability.py, and read natively to build nullability results." [[stays]] path = "compiler/macro_bridge/models.py" @@ -765,7 +760,7 @@ note = "Shape rule: data protocol and aliases only (RuleContext, RuleCheck, Rule [[stays]] path = "rule_engine/constants.py" rule = 2 -note = "Shape rule: custom-rule host and fact-name constants (CUSTOM_HOST_MAX_TRACKED_READS, CUSTOM_RULE_IMPORT_ROOTS, FACT_PROJECT_MODELS, ...) used by the Python custom-rule host, no logic; versions, launch module and shared keys are native (sqlbuild._native)." +note = "Shape rule: custom-rule host and fact-name constants (CUSTOM_HOST_MAX_TRACKED_READS, CUSTOM_HOST_STARTUP_ENVIRONMENT, FACT_PROJECT_MODELS, ...) used by the Python custom-rule host, no logic; versions, launch module, shared keys and the import allowlist are native (sqlbuild._native)." [[stays]] path = "compiler/compile/exceptions.py" diff --git a/src/sqlbuild/_native.pyi b/src/sqlbuild/_native.pyi index 138bac52ba..7d4a7a7ce3 100644 --- a/src/sqlbuild/_native.pyi +++ b/src/sqlbuild/_native.pyi @@ -5632,6 +5632,15 @@ def digest_files(paths: list[str]) -> list[str | None]: ... def normalize_type( type_sql: str, dialect: str ) -> tuple[tuple[str, str, str | None, str | None, str | None], str | None]: ... +def infer_first_arg_nullability( + args: Sequence[InferredNullability | str], +) -> InferredNullability: ... +def infer_conditional_result_nullability( + args: Sequence[InferredNullability | str], +) -> InferredNullability: ... +def infer_safe_cast_nullability( + args: Sequence[InferredNullability | str], +) -> InferredNullability: ... # Native analysis: model analysis session. type _AnalysisDiagnosticRow = tuple[str, str, int | None, int | None, int | None, int | None, str] @@ -5780,6 +5789,22 @@ def parse_audit_instances( error_class: type[Exception], null_as_empty: bool, ) -> tuple[SchemaAuditInstance, ...]: ... +def retry_policy_exceptions( + *, + max_attempts: int, + retry_on: type[BaseException] | Iterable[type[BaseException]], + initial_delay_seconds: float, + backoff_multiplier: float, + max_delay_seconds: float | None, + max_elapsed_seconds: float | None, +) -> tuple[type[BaseException], ...]: ... +def retry_delay_seconds(*, retry_policy: RetryPolicy, retry_index: int) -> float: ... +def validate_audit_case( + *, name: str, definition: str, minimum_samples: int | None, evidence_limit: int | None +) -> None: ... +def normalize_hook_reads( + value: SqlResourceRef | tuple[SqlResourceRef, ...] | list[SqlResourceRef], +) -> tuple[SqlResourceRef, ...]: ... def optional_named_string( *, raw_value: object | None, @@ -6280,6 +6305,13 @@ def dagster_failure( def dagster_live_failure( *, asset: IntegrationAssetResult, envelope: IntegrationResultEnvelope, command: list[str] ) -> tuple[list[str], dict[str, Any]]: ... +def dagster_default_asset_key_path(node: Mapping[str, Any]) -> list[str]: ... +def dagster_default_group_name(node: Mapping[str, Any]) -> str: ... +def dagster_default_tags(node: Mapping[str, Any]) -> dict[str, str]: ... +def dagster_default_metadata(node: Mapping[str, Any]) -> dict[str, Any]: ... +def dagster_default_description(node: Mapping[str, Any]) -> str: ... +def dagster_default_check_name(check: Mapping[str, Any]) -> str: ... +def dagster_default_check_metadata(check: Mapping[str, Any]) -> dict[str, Any]: ... def build_janitor_plan( *, project: CompiledProjectView, @@ -7006,6 +7038,18 @@ def get_config_string_tuple( ) -> tuple[str, ...] | None: ... def get_config_cursor_bound(*, values: Mapping[str, object], key: str) -> str | None: ... def custom_rule_import_closure(*, rule: Rule, project_dir: Path) -> tuple[Path, ...]: ... +def allowed_custom_rule_import(module: str) -> bool: ... +def verify_custom_rules(*, rules: Iterable[Rule], project_dir: Path) -> None: ... +def finding_position( + *, path: Path, line: int | None, column: int | None, node: object +) -> tuple[int, int]: ... +def rule_option_toml_value(value: object) -> str: ... +def validate_project_relative(path: str) -> None: ... +def project_tree_paths(root: Path) -> tuple[ProjectPath, ...]: ... +def project_tree_file_path(root: Path, path: str) -> str: ... +def startup_input_stamps( + *, directories: Iterable[str], files: Iterable[str], prefixes: Iterable[str] +) -> list[tuple[str, int | None, int | None]]: ... def custom_rule_implementation_fingerprint( *, rule: Rule, project_dir: Path, import_closure: tuple[Path, ...] | None = None ) -> str: ... diff --git a/src/sqlbuild/adapter/type_system/_helpers/inference_rules.py b/src/sqlbuild/adapter/type_system/_helpers/inference_rules.py deleted file mode 100644 index e7d2b22a3b..0000000000 --- a/src/sqlbuild/adapter/type_system/_helpers/inference_rules.py +++ /dev/null @@ -1,41 +0,0 @@ -"""Public expression inference capability for adapters.""" - -from __future__ import annotations - -from sqlbuild.compiler.lineage.types import InferredNullability - - -def first_arg_nullability( - args: tuple[InferredNullability, ...], -) -> InferredNullability: - """Return the first argument's nullability for null-propagating unary functions.""" - - if not args: - return InferredNullability.UNKNOWN - return args[0] - - -def conditional_result_nullability( - args: tuple[InferredNullability, ...], -) -> InferredNullability: - """Infer IF/IFF-style nullability from true/false result arguments.""" - - source_target_and_default_argument_count: int = 3 - if len(args) < source_target_and_default_argument_count: - return InferredNullability.UNKNOWN - result_args: tuple[InferredNullability, InferredNullability] = (args[1], args[2]) - if all(value == InferredNullability.NON_NULL for value in result_args): - return InferredNullability.NON_NULL - if any(value == InferredNullability.NULLABLE for value in result_args): - return InferredNullability.NULLABLE - return InferredNullability.UNKNOWN - - -def safe_cast_nullability( - args: tuple[InferredNullability, ...], -) -> InferredNullability: - """Infer SAFE_CAST/TRY_CAST nullability without assuming conversion succeeds.""" - - if args and args[0] == InferredNullability.NULLABLE: - return InferredNullability.NULLABLE - return InferredNullability.UNKNOWN diff --git a/src/sqlbuild/adapter/type_system/main/conditional_result_nullability.py b/src/sqlbuild/adapter/type_system/main/conditional_result_nullability.py index 46a0128295..5e18a0780e 100644 --- a/src/sqlbuild/adapter/type_system/main/conditional_result_nullability.py +++ b/src/sqlbuild/adapter/type_system/main/conditional_result_nullability.py @@ -1,8 +1,6 @@ """Public adapter conditional nullability inference operation.""" -from sqlbuild.adapter.type_system._helpers.inference_rules import ( - conditional_result_nullability as _conditional_result_nullability, -) +from sqlbuild._native import infer_conditional_result_nullability from sqlbuild.compiler.lineage.types import InferredNullability @@ -11,4 +9,4 @@ def conditional_result_nullability( ) -> InferredNullability: """Infer IF/IFF-style nullability from true and false result arguments.""" - return _conditional_result_nullability(args) + return infer_conditional_result_nullability(args) diff --git a/src/sqlbuild/adapter/type_system/main/first_arg_nullability.py b/src/sqlbuild/adapter/type_system/main/first_arg_nullability.py index 3117238ff8..1e4e38ceee 100644 --- a/src/sqlbuild/adapter/type_system/main/first_arg_nullability.py +++ b/src/sqlbuild/adapter/type_system/main/first_arg_nullability.py @@ -1,12 +1,10 @@ """Public adapter unary nullability inference operation.""" -from sqlbuild.adapter.type_system._helpers.inference_rules import ( - first_arg_nullability as _first_arg_nullability, -) +from sqlbuild._native import infer_first_arg_nullability from sqlbuild.compiler.lineage.types import InferredNullability def first_arg_nullability(args: tuple[InferredNullability, ...]) -> InferredNullability: """Return the first argument's nullability for null-propagating unary functions.""" - return _first_arg_nullability(args) + return infer_first_arg_nullability(args) diff --git a/src/sqlbuild/adapter/type_system/main/safe_cast_nullability.py b/src/sqlbuild/adapter/type_system/main/safe_cast_nullability.py index 5dddd886e9..03829b08e0 100644 --- a/src/sqlbuild/adapter/type_system/main/safe_cast_nullability.py +++ b/src/sqlbuild/adapter/type_system/main/safe_cast_nullability.py @@ -1,12 +1,10 @@ """Public adapter safe-cast nullability inference operation.""" -from sqlbuild.adapter.type_system._helpers.inference_rules import ( - safe_cast_nullability as _safe_cast_nullability, -) +from sqlbuild._native import infer_safe_cast_nullability from sqlbuild.compiler.lineage.types import InferredNullability def safe_cast_nullability(args: tuple[InferredNullability, ...]) -> InferredNullability: """Infer safe-cast nullability without assuming conversion succeeds.""" - return _safe_cast_nullability(args) + return infer_safe_cast_nullability(args) diff --git a/src/sqlbuild/cli/compile_reuse/_helpers/startup_inputs.py b/src/sqlbuild/cli/compile_reuse/_helpers/startup_inputs.py index 0cc52f976d..79cc5c8b2e 100644 --- a/src/sqlbuild/cli/compile_reuse/_helpers/startup_inputs.py +++ b/src/sqlbuild/cli/compile_reuse/_helpers/startup_inputs.py @@ -2,31 +2,15 @@ import site import sys -from os import stat_result -from pathlib import Path + +from sqlbuild._native import startup_input_stamps def startup_inputs() -> list[tuple[str, int | None, int | None]]: """Stamp startup directories, path files, interpreter and virtual-environment configuration.""" - directories: set[Path] = { - Path(entry).absolute() - for entry in (*sys.path, *site.getsitepackages(), site.getusersitepackages()) - } - paths: set[Path] = { - *directories, - Path(sys.executable), - Path(sys.prefix) / "pyvenv.cfg", - Path(sys.base_prefix) / "pyvenv.cfg", - } - for directory in directories: - paths.update(directory.glob("*.pth")) - stamps: list[tuple[str, int | None, int | None]] = [] - for path in sorted(paths): - try: - status: stat_result = path.stat() - except FileNotFoundError: - stamps.append((str(path), None, None)) - else: - stamps.append((str(path), status.st_mtime_ns, status.st_size)) - return stamps + return startup_input_stamps( + directories=(*sys.path, *site.getsitepackages(), site.getusersitepackages()), + files=(sys.executable,), + prefixes=(sys.prefix, sys.base_prefix), + ) diff --git a/src/sqlbuild/integrations/dagster/classes/sqlbuild_dagster_translator.py b/src/sqlbuild/integrations/dagster/classes/sqlbuild_dagster_translator.py index 249462beb6..4cff960df9 100644 --- a/src/sqlbuild/integrations/dagster/classes/sqlbuild_dagster_translator.py +++ b/src/sqlbuild/integrations/dagster/classes/sqlbuild_dagster_translator.py @@ -2,12 +2,19 @@ from __future__ import annotations -import re -from collections.abc import Mapping, Sequence +from collections.abc import Mapping from typing import Any +from sqlbuild._native import ( + dagster_default_asset_key_path, + dagster_default_check_metadata, + dagster_default_check_name, + dagster_default_description, + dagster_default_group_name, + dagster_default_metadata, + dagster_default_tags, +) from sqlbuild.integrations.dagster._helpers.imports import load_dagster -from sqlbuild.integrations.dagster.constants import SOURCE_NODE_KIND class SqlBuildDagsterTranslator: @@ -15,10 +22,7 @@ class SqlBuildDagsterTranslator: def get_asset_key(self, node: Mapping[str, Any]) -> Any: dg: Any = load_dagster() - authored_asset_key: Sequence[object] | None = _authored_dagster_asset_key(node) - if authored_asset_key is not None: - return dg.AssetKey([str(part) for part in authored_asset_key]) - return dg.AssetKey([str(part) for part in node["asset_key"]]) + return dg.AssetKey(dagster_default_asset_key_path(node)) def is_asset_node(self, node: Mapping[str, Any]) -> bool: """Return whether SQLBuild owns this node as a Dagster asset.""" @@ -31,112 +35,19 @@ def is_asset_check(self, check: Mapping[str, Any]) -> bool: return True def get_group_name(self, node: Mapping[str, Any]) -> str | None: - group_name: str = str(node.get("group") or node.get("project_name") or "sqlbuild") - return group_name.replace("-", "_") + return dagster_default_group_name(node) def get_tags(self, node: Mapping[str, Any]) -> Mapping[str, str]: - tags: dict[str, str] = {"sqlbuild/kind": str(node.get("kind", ""))} - for tag in node.get("tags", ()): - tag_key: str = _normalize_tag_key(str(tag)) - if tag_key: - tags[tag_key] = "" - return tags + return dagster_default_tags(node) def get_metadata(self, node: Mapping[str, Any]) -> Mapping[str, Any]: - metadata: dict[str, Any] = { - "sqlbuild_id": node.get("id"), - "sqlbuild_name": node.get("name"), - "sqlbuild_kind": node.get("kind"), - "sqlbuild_selector": node.get("name"), - } - for key in ( - "path", - "target", - "description", - "sql", - "columns", - "column_lineage", - "group", - "language", - "materialization_type", - "return_kind", - "loader", - "meta", - ): - if key in node: - metadata[key] = node[key] - return metadata + return dagster_default_metadata(node) def get_description(self, node: Mapping[str, Any]) -> str | None: - description: str = str(node.get("description") or _fallback_description(node)) - sql: object = node.get("sql") - if isinstance(sql, str) and sql.strip(): - return f"{description}\n\n**SQLBuild SQL:**\n```sql\n{sql.strip()}\n```" - path: object = node.get("path") - if isinstance(path, str) and path: - return f"{description}\n\n**Source file:** {_markdown_code_span(path)}" - return description + return dagster_default_description(node) def get_check_name(self, check: Mapping[str, Any]) -> str: - parts: list[str] = [str(check.get("kind", "check")), str(check.get("name", "check"))] - if check.get("attached_column_name") is not None: - parts.append(str(check["attached_column_name"])) - elif check.get("attached_target_name") is not None: - parts.append(str(check["attached_target_name"])) - return "__".join(_normalize_name(part) for part in parts if part) + return dagster_default_check_name(check) def get_check_metadata(self, check: Mapping[str, Any]) -> Mapping[str, Any]: - return { - "sqlbuild_check_id": check.get("id"), - "sqlbuild_check_kind": check.get("kind"), - "sqlbuild_check_name": check.get("name"), - "sqlbuild_check_selector": check.get("id"), - } - - -def _normalize_tag_key(value: str) -> str: - normalized: str = re.sub(r"[^A-Za-z0-9_.\-/]+", "_", value).strip("_") - return normalized - - -def _authored_dagster_asset_key(node: Mapping[str, Any]) -> Sequence[object] | None: - if str(node.get("kind")) != SOURCE_NODE_KIND: - return None - meta: object = node.get("meta") - if not isinstance(meta, Mapping): - return None - dagster_meta: object = meta.get("dagster") - if not isinstance(dagster_meta, Mapping): - return None - asset_key: object = dagster_meta.get("asset_key") - if not isinstance(asset_key, Sequence) or isinstance(asset_key, (str, bytes)) or not asset_key: - return None - return asset_key - - -def _normalize_name(value: str) -> str: - normalized: str = re.sub(r"[^A-Za-z0-9_]+", "_", value).strip("_") - return normalized or "check" - - -def _markdown_code_span(value: str) -> str: - longest_backtick_run: int = max( - (len(match.group()) for match in re.finditer(r"`+", value)), default=0 - ) - delimiter: str = "`" * (longest_backtick_run + 1) - padding: str = " " if value.startswith(("`", " ")) or value.endswith(("`", " ")) else "" - return f"{delimiter}{padding}{value}{padding}{delimiter}" - - -def _fallback_description(node: Mapping[str, Any]) -> str: - kind: str = str(node.get("kind") or "asset").replace("_", " ") - name: str = str(node.get("name") or "unknown") - target: object = node.get("target") - qualified_name: str = "" - if isinstance(target, Mapping): - qualified_name = str(target.get("qualified_name") or "") - relation_suffix: str = f" at `{qualified_name}`" if qualified_name else "" - materialization: object = node.get("materialization_type") - if materialization is not None: - return f"SQLBuild {kind} `{name}` materialized as `{materialization}`{relation_suffix}." - return f"SQLBuild {kind} `{name}`{relation_suffix}." + return dagster_default_check_metadata(check) diff --git a/src/sqlbuild/python_nodes/_helpers/dependency_normalization.py b/src/sqlbuild/python_nodes/_helpers/dependency_normalization.py index 6af73f223e..ec6adc9837 100644 --- a/src/sqlbuild/python_nodes/_helpers/dependency_normalization.py +++ b/src/sqlbuild/python_nodes/_helpers/dependency_normalization.py @@ -2,7 +2,6 @@ from collections.abc import Callable -from sqlbuild.errors.contracts.exceptions import SharedInputError from sqlbuild.python_nodes.models import SqlResourceRef @@ -21,18 +20,3 @@ def normalize_loader_dependencies( value: tuple[Callable[..., object], ...] | list[Callable[..., object]], ) -> tuple[Callable[..., object], ...]: return tuple(value) - - -def normalize_hook_reads( - value: SqlResourceRef | tuple[SqlResourceRef, ...] | list[SqlResourceRef], -) -> tuple[SqlResourceRef, ...]: - reads: tuple[object, ...] = (value,) if isinstance(value, SqlResourceRef) else tuple(value) - invalid: object | None = next( - (item for item in reads if not isinstance(item, SqlResourceRef)), None - ) - if invalid is not None: - raise SharedInputError( - "@hook(reads=...) accepts model(...), source(...), and seed(...) references only; " - f"got {type(invalid).__name__}" - ) - return tuple(dict.fromkeys(item for item in reads if isinstance(item, SqlResourceRef))) diff --git a/src/sqlbuild/python_nodes/main/apply_hook.py b/src/sqlbuild/python_nodes/main/apply_hook.py index c74abe3473..99dd9459ea 100644 --- a/src/sqlbuild/python_nodes/main/apply_hook.py +++ b/src/sqlbuild/python_nodes/main/apply_hook.py @@ -5,8 +5,8 @@ from collections.abc import Callable from typing import Any, cast +from sqlbuild._native import normalize_hook_reads from sqlbuild.python_nodes._helpers.attachment import attach_definition -from sqlbuild.python_nodes._helpers.dependency_normalization import normalize_hook_reads from sqlbuild.python_nodes._helpers.description_resolution import resolve_description from sqlbuild.python_nodes.models import HookDefinition, SqlResourceRef diff --git a/src/sqlbuild/python_nodes/main/calculate_retry_delay.py b/src/sqlbuild/python_nodes/main/calculate_retry_delay.py index ebe10fa53d..3e7b10876f 100644 --- a/src/sqlbuild/python_nodes/main/calculate_retry_delay.py +++ b/src/sqlbuild/python_nodes/main/calculate_retry_delay.py @@ -1,18 +1,10 @@ """Calculate retry delays for Python-node execution.""" -import random - +from sqlbuild._native import retry_delay_seconds from sqlbuild.python_nodes.models import RetryPolicy def calculate_retry_delay(*, retry_policy: RetryPolicy, retry_index: int) -> float: """Calculate the delay before a retry attempt.""" - delay_seconds: float = retry_policy.initial_delay_seconds * ( - retry_policy.backoff_multiplier**retry_index - ) - if retry_policy.max_delay_seconds is not None: - delay_seconds = min(delay_seconds, retry_policy.max_delay_seconds) - if retry_policy.jitter: - return random.uniform(0, delay_seconds) - return delay_seconds + return retry_delay_seconds(retry_policy=retry_policy, retry_index=retry_index) diff --git a/src/sqlbuild/python_nodes/models.py b/src/sqlbuild/python_nodes/models.py index ac50102945..2f5c61e697 100644 --- a/src/sqlbuild/python_nodes/models.py +++ b/src/sqlbuild/python_nodes/models.py @@ -2,13 +2,17 @@ from __future__ import annotations -import re from collections.abc import Callable, Mapping from dataclasses import dataclass, field -from sqlbuild._native import MeasurementThresholds, SourceColumnEntry, SourceWriteStrategy +from sqlbuild._native import ( + MeasurementThresholds, + SourceColumnEntry, + SourceWriteStrategy, + retry_policy_exceptions, + validate_audit_case, +) from sqlbuild.compiler.auditing.types import AuditSeverity -from sqlbuild.errors.contracts.exceptions import SharedInputError from sqlbuild.python_nodes.constants import RELATION_RENDERER from sqlbuild.python_nodes.types import PythonCheckSeverity, SqlResourceRefKind @@ -62,27 +66,14 @@ def __init__( max_elapsed_seconds: float | None = None, jitter: bool = True, ) -> None: - if max_attempts < 1: - raise SharedInputError("RetryPolicy max_attempts must be at least 1") - if initial_delay_seconds < 0: - raise SharedInputError("RetryPolicy initial_delay_seconds must be non-negative") - if backoff_multiplier < 1: - raise SharedInputError("RetryPolicy backoff_multiplier must be at least 1") - if max_delay_seconds is not None and max_delay_seconds < 0: - raise SharedInputError("RetryPolicy max_delay_seconds must be non-negative") - if max_elapsed_seconds is not None and max_elapsed_seconds < 0: - raise SharedInputError("RetryPolicy max_elapsed_seconds must be non-negative") - normalized_retry_on: tuple[type[BaseException], ...] - if isinstance(retry_on, type) and issubclass(retry_on, BaseException): - normalized_retry_on = (retry_on,) - else: - normalized_retry_on = tuple(retry_on) - exception_type: type[BaseException] - for exception_type in normalized_retry_on: - if not isinstance(exception_type, type) or not issubclass( - exception_type, BaseException - ): - raise SharedInputError("RetryPolicy retry_on must contain exception classes") + normalized_retry_on: tuple[type[BaseException], ...] = retry_policy_exceptions( + max_attempts=max_attempts, + retry_on=retry_on, + initial_delay_seconds=initial_delay_seconds, + backoff_multiplier=backoff_multiplier, + max_delay_seconds=max_delay_seconds, + max_elapsed_seconds=max_elapsed_seconds, + ) object.__setattr__(self, "max_attempts", max_attempts) object.__setattr__(self, "retry_on", normalized_retry_on) object.__setattr__(self, "initial_delay_seconds", initial_delay_seconds) @@ -187,22 +178,9 @@ class AuditCase: evidence_limit: int | None = None def __post_init__(self) -> None: - if not re.fullmatch(r"^[a-z][a-z0-9_]*$", self.name): - raise SharedInputError( - "AuditCase name must be a non-empty lowercase identifier using letters, " - "numbers, and underscores" - ) - if not self.definition.strip(): - raise SharedInputError("AuditCase definition must be non-empty") - if self.minimum_samples is not None and ( - not isinstance(self.minimum_samples, int) - or isinstance(self.minimum_samples, bool) - or self.minimum_samples < 0 - ): - raise SharedInputError("AuditCase minimum_samples must be a non-negative integer") - if self.evidence_limit is not None and ( - not isinstance(self.evidence_limit, int) - or isinstance(self.evidence_limit, bool) - or self.evidence_limit < 0 - ): - raise SharedInputError("AuditCase evidence_limit must be a non-negative integer") + validate_audit_case( + name=self.name, + definition=self.definition, + minimum_samples=self.minimum_samples, + evidence_limit=self.evidence_limit, + ) diff --git a/src/sqlbuild/rule_engine/_helpers/engine/hermeticity.py b/src/sqlbuild/rule_engine/_helpers/engine/hermeticity.py deleted file mode 100644 index 343c66867d..0000000000 --- a/src/sqlbuild/rule_engine/_helpers/engine/hermeticity.py +++ /dev/null @@ -1,52 +0,0 @@ -"""Static import verification for the fingerprinted custom-rule source closure.""" - -from __future__ import annotations - -import ast -from pathlib import Path - -from sqlbuild._native import NonHermeticRuleError, Rule, RulesError, custom_rule_import_closure -from sqlbuild.rule_engine.constants import CUSTOM_RULE_IMPORT_ROOTS - - -def verify_custom_rules(*, rules: tuple[Rule, ...], project_dir: Path) -> None: - """Reject imports outside the deterministic allowlist in every fingerprinted rule file.""" - - root: Path = project_dir.resolve() - checked: set[Path] = set() - for rule in rules: - if not rule.custom or rule.source is None: - continue - source_path: Path = Path(rule.source).resolve() - closure: tuple[Path, ...] = custom_rule_import_closure(rule=rule, project_dir=root) - for path in (source_path, *closure): - if path in checked: - continue - checked.add(path) - _verify_source(path=path, project_dir=root) - - -def allowed_custom_rule_import(module: str) -> bool: - """Return whether custom-rule code may import one absolute module name.""" - - return any(module == root or module.startswith(f"{root}.") for root in CUSTOM_RULE_IMPORT_ROOTS) - - -def _verify_source(*, path: Path, project_dir: Path) -> None: - if not path.is_relative_to(project_dir): - raise RulesError(f"custom rule source must be repository-owned: {path}") - source: str = path.read_text(encoding="utf-8") - tree: ast.Module = ast.parse(source, filename=str(path)) - for node in ast.walk(tree): - if isinstance(node, ast.Import): - for alias in node.names: - _verify_import(path=path, line=node.lineno, module=alias.name) - elif isinstance(node, ast.ImportFrom) and node.level == 0: - _verify_import(path=path, line=node.lineno, module=node.module or "") - - -def _verify_import(*, path: Path, line: int, module: str) -> None: - if not allowed_custom_rule_import(module): - raise NonHermeticRuleError( - f"non-hermetic custom rule at {path}:{line}: import {module!r} is not allowed" - ) diff --git a/src/sqlbuild/rule_engine/_helpers/engine/rule_harness.py b/src/sqlbuild/rule_engine/_helpers/engine/rule_harness.py index e1662febb2..4acfa16047 100644 --- a/src/sqlbuild/rule_engine/_helpers/engine/rule_harness.py +++ b/src/sqlbuild/rule_engine/_helpers/engine/rule_harness.py @@ -20,6 +20,7 @@ discover_project_inputs, evaluate_project_rules, load_rules_config, + rule_option_toml_value, ) from sqlbuild.adapters.duckdb.classes.duckdb_adapter import DuckDbAdapter from sqlbuild.compiler.compile.types import CompiledProjectView @@ -75,7 +76,7 @@ def _write_rule_project(*, root: Path, rule: Rule, test_case: RuleCase) -> Path: if test_case.config: config_lines.append(f"[rules.rule_options.{rule.code}]") for name, value in sorted(test_case.config.items()): - config_lines.append(f"{name} = {_toml_value(value=value)}") + config_lines.append(f"{name} = {rule_option_toml_value(value)}") config_lines.append("") project_config: str = "\n".join(config_lines) (root / "sqlbuild_project.toml").write_text(project_config, encoding="utf-8") @@ -95,19 +96,6 @@ def _write_rule_project(*, root: Path, rule: Rule, test_case: RuleCase) -> Path: return rule_path -def _toml_value(*, value: object) -> str: - if isinstance(value, bool): - return str(value).lower() - if isinstance(value, int): - return str(value) - if isinstance(value, str): - escaped: str = value.replace("\\", "\\\\").replace('"', '\\"') - return f'"{escaped}"' - if isinstance(value, (list, tuple)): - return "[" + ", ".join(_toml_value(value=item) for item in value) + "]" - raise RuleUsageError(f"RuleCase config value {value!r} cannot be represented as a rule option") - - def _compile_rule_project(*, root: Path) -> CompiledProjectView: discovered: DiscoveredProjectInputs = discover_project_inputs(project_dir=root) adapter: DuckDbAdapter = DuckDbAdapter() diff --git a/src/sqlbuild/rule_engine/classes/project_tree.py b/src/sqlbuild/rule_engine/classes/project_tree.py index 99b34707a1..ba020703ac 100644 --- a/src/sqlbuild/rule_engine/classes/project_tree.py +++ b/src/sqlbuild/rule_engine/classes/project_tree.py @@ -3,17 +3,18 @@ from __future__ import annotations import fnmatch -import os -import stat from pathlib import Path import sqlbuild._native as _native -from sqlbuild._native import Model, ProjectPath, RuleUsageError +from sqlbuild._native import ( + Model, + ProjectPath, + RuleUsageError, + project_tree_file_path, + project_tree_paths, + validate_project_relative, +) from sqlbuild.compiler.compile.types import CompiledProjectView -from sqlbuild.rule_engine.constants import PARENT_DIRECTORY_TOKEN - -_EXCLUDED_ROOTS: frozenset[str] = frozenset({".git", ".venv", "logs", "target", "venv"}) -_TRACKED_TEXT_SUFFIXES: frozenset[str] = frozenset({".py", ".sql", ".toml", ".yaml", ".yml"}) class ProjectTree: @@ -28,7 +29,6 @@ def __init__( ) -> None: self._root: Path = project_dir.resolve() self._discovered: tuple[ProjectPath, ...] | None = None - self._directories: dict[Path, Path | None] = {} model_views: tuple[Model, ...] = ( tuple(public_model(item) for item in project.models) if models is None else models ) @@ -39,7 +39,7 @@ def __init__( @property def _nodes(self) -> tuple[ProjectPath, ...]: if self._discovered is None: - self._discovered = self._discover() + self._discovered = project_tree_paths(self._root) return self._discovered def paths(self) -> tuple[ProjectPath, ...]: @@ -54,7 +54,7 @@ def descendants(self, path: str = "") -> tuple[ProjectPath, ...]: return tuple(node for node in self._nodes if node.value.startswith(prefix)) def glob(self, pattern: str) -> tuple[ProjectPath, ...]: - self._validate_relative(pattern) + validate_project_relative(pattern) return tuple(node for node in self._nodes if fnmatch.fnmatchcase(node.value, pattern)) def relative_parts(self, *, path: Path | str, under: str = "") -> tuple[str, ...]: @@ -80,75 +80,12 @@ def resources_under(self, path: str) -> tuple[Model, ...]: ) def read_text(self, path: str) -> str: - self._validate_relative(path) - regular: Path | None = self._regular_file(path) - candidate: Path = (self._root / path).resolve() if regular is None else regular - if regular is None and ( - not candidate.is_relative_to(self._root) or not candidate.is_file() - ): - raise RuleUsageError(f"project file does not exist: {path}") - if candidate.suffix.lower() not in _TRACKED_TEXT_SUFFIXES: - supported: str = ", ".join(sorted(_TRACKED_TEXT_SUFFIXES)) - raise RuleUsageError( - f"compiler rules input must be a tracked file type ({supported}): {path}" - ) - if regular is None and candidate.is_symlink(): - raise RuleUsageError(f"project file must not be a symlink: {path}") + file: str = project_tree_file_path(self._root, path) try: - return candidate.read_text(encoding="utf-8") + return Path(file).read_text(encoding="utf-8") except (OSError, UnicodeError) as error: raise RuleUsageError(f"could not read project file {path}: {error}") from error - def _regular_file(self, path: str) -> Path | None: - """Resolve a regular, non-symlink file under the project through its cached directory.""" - - relative: Path = Path(path) - if not relative.name: - return None - directory: Path | None - if relative.parent in self._directories: - directory = self._directories[relative.parent] - else: - resolved: Path = (self._root / relative.parent).resolve() - directory = resolved if resolved.is_relative_to(self._root) else None - self._directories[relative.parent] = directory - if directory is None: - return None - candidate: Path = directory / relative.name - try: - regular: bool = stat.S_ISREG(os.lstat(candidate).st_mode) - except OSError: - return None - return candidate if regular else None - - def _discover(self) -> tuple[ProjectPath, ...]: - nodes: list[ProjectPath] = [] - for root, directories, filenames in os.walk(self._root, followlinks=False): - root_path: Path = Path(root) - directories[:] = sorted( - name - for name in directories - if not name.startswith(".") - and not (root_path == self._root and name in _EXCLUDED_ROOTS) - and not (root_path / name).is_symlink() - ) - for name in directories: - relative: str = (root_path / name).relative_to(self._root).as_posix() - nodes.append(ProjectPath(value=relative, is_file=False, is_dir=True)) - for name in sorted(filenames): - file_path: Path = root_path / name - if file_path.is_symlink(): - continue - relative = file_path.relative_to(self._root).as_posix() - nodes.append(ProjectPath(value=relative, is_file=True, is_dir=False)) - return tuple(sorted(nodes)) - - @staticmethod - def _validate_relative(path: str) -> None: - candidate: Path = Path(path) - if candidate.is_absolute() or PARENT_DIRECTORY_TOKEN in candidate.parts: - raise RuleUsageError(f"project path escapes the project: {path}") - def public_model(model: object) -> Model: from sqlbuild._native import CompiledModel diff --git a/src/sqlbuild/rule_engine/classes/rule_context.py b/src/sqlbuild/rule_engine/classes/rule_context.py index 0c20fc0a8e..555d7db93e 100644 --- a/src/sqlbuild/rule_engine/classes/rule_context.py +++ b/src/sqlbuild/rule_engine/classes/rule_context.py @@ -15,6 +15,7 @@ RulesConfig, RuleUsageError, SqlNode, + finding_position, ) from sqlbuild.compiler.compile.types import CompiledModelView, CompiledProjectView from sqlbuild.rule_engine.classes.audit_facts import AuditFacts @@ -26,7 +27,6 @@ from sqlbuild.rule_engine.classes.model_test_facts import TestFacts from sqlbuild.rule_engine.classes.project_facts import ProjectFacts from sqlbuild.rule_engine.classes.sql_facts import SqlFacts -from sqlbuild.rule_engine.constants import PARENT_DIRECTORY_TOKEN def build_rule_fact_views( @@ -114,14 +114,9 @@ def finding( path = Path(subject.value) else: path = Path(subject) - if path.is_absolute() or PARENT_DIRECTORY_TOKEN in path.parts: - raise RuleUsageError(f"finding path must be project-relative: {path}") - resolved_line: int = line if line is not None else int(getattr(node, "line", 1) or 1) - resolved_column: int = ( - column if column is not None else int(getattr(node, "column", 1) or 1) + resolved_line, resolved_column = finding_position( + path=path, line=line, column=column, node=node ) - if resolved_line < 1 or resolved_column < 1: - raise RuleUsageError("finding line and column must be positive") return Finding( code=self._rule.code, path=path, diff --git a/src/sqlbuild/rule_engine/classes/runtime_guard.py b/src/sqlbuild/rule_engine/classes/runtime_guard.py index d54026716f..03fc03ba1e 100644 --- a/src/sqlbuild/rule_engine/classes/runtime_guard.py +++ b/src/sqlbuild/rule_engine/classes/runtime_guard.py @@ -12,8 +12,7 @@ from pathlib import Path from types import CodeType, FrameType, FunctionType, ModuleType -from sqlbuild._native import NonHermeticRuleError -from sqlbuild.rule_engine._helpers.engine.hermeticity import allowed_custom_rule_import +from sqlbuild._native import NonHermeticRuleError, allowed_custom_rule_import from sqlbuild.rule_engine.classes.project_tree import ProjectTree _IMPORT_EVENT: str = "import" diff --git a/src/sqlbuild/rule_engine/constants.py b/src/sqlbuild/rule_engine/constants.py index ee72f74f0d..042f1b718e 100644 --- a/src/sqlbuild/rule_engine/constants.py +++ b/src/sqlbuild/rule_engine/constants.py @@ -19,42 +19,7 @@ "PYTHONUSERBASE", "SYSTEMROOT", ) -CUSTOM_RULE_IMPORT_ROOTS: frozenset[str] = frozenset( - { - "__future__", - "abc", - "bisect", - "collections", - "copy", - "dataclasses", - "decimal", - "difflib", - "enum", - "fnmatch", - "fractions", - "functools", - "graphlib", - "hashlib", - "heapq", - "itertools", - "json", - "math", - "operator", - "pathlib", - "re", - "rules", - "sqlbuild.rules", - "statistics", - "string", - "textwrap", - "types", - "typing", - "unicodedata", - } -) - RULE_DECORATOR_TOKEN: str = "@rule" -PARENT_DIRECTORY_TOKEN: str = ".." FACT_PROJECT_MODELS: str = "project.models" FACT_PROJECT_SOURCES: str = "project.sources" diff --git a/tests/unit/src/sqlbuild/adapter/type_system/_test_types.py b/tests/unit/src/sqlbuild/adapter/type_system/_test_types.py index ce055965dc..9f110cd9d6 100644 --- a/tests/unit/src/sqlbuild/adapter/type_system/_test_types.py +++ b/tests/unit/src/sqlbuild/adapter/type_system/_test_types.py @@ -1,6 +1,7 @@ from dataclasses import dataclass from sqlbuild.adapter.contract.models import NormalizedType +from sqlbuild.compiler.lineage.types import InferredNullability @dataclass(frozen=True) @@ -26,3 +27,19 @@ class NumericFamilyTestCase: dialect: str | None raw_type: str expected_family: str | None + + +@dataclass(frozen=True) +class NullabilityRuleTestCase: + description: str + args: tuple[InferredNullability, ...] + expected_first_arg: InferredNullability + expected_conditional: InferredNullability + expected_safe_cast: InferredNullability + + +@dataclass(frozen=True) +class FirstArgIdentityTestCase: + description: str + argument: object + expected_same_object: bool diff --git a/tests/unit/src/sqlbuild/adapter/type_system/test_nullability_rules.py b/tests/unit/src/sqlbuild/adapter/type_system/test_nullability_rules.py new file mode 100644 index 0000000000..17c57e0ce8 --- /dev/null +++ b/tests/unit/src/sqlbuild/adapter/type_system/test_nullability_rules.py @@ -0,0 +1,92 @@ +import pytest + +from sqlbuild.adapter.type_system.main.conditional_result_nullability import ( + conditional_result_nullability, +) +from sqlbuild.adapter.type_system.main.first_arg_nullability import first_arg_nullability +from sqlbuild.adapter.type_system.main.safe_cast_nullability import safe_cast_nullability +from sqlbuild.compiler.lineage.types import InferredNullability +from tests.unit.src.sqlbuild.adapter.type_system._test_types import ( + FirstArgIdentityTestCase, + NullabilityRuleTestCase, +) + + +@pytest.mark.parametrize( + "test_case", + [ + NullabilityRuleTestCase( + description="no arguments", + args=(), + expected_first_arg=InferredNullability.UNKNOWN, + expected_conditional=InferredNullability.UNKNOWN, + expected_safe_cast=InferredNullability.UNKNOWN, + ), + NullabilityRuleTestCase( + description="two non-null results", + args=( + InferredNullability.UNKNOWN, + InferredNullability.NON_NULL, + InferredNullability.NON_NULL, + ), + expected_first_arg=InferredNullability.UNKNOWN, + expected_conditional=InferredNullability.NON_NULL, + expected_safe_cast=InferredNullability.UNKNOWN, + ), + NullabilityRuleTestCase( + description="one nullable result", + args=( + InferredNullability.NULLABLE, + InferredNullability.NON_NULL, + InferredNullability.NULLABLE, + ), + expected_first_arg=InferredNullability.NULLABLE, + expected_conditional=InferredNullability.NULLABLE, + expected_safe_cast=InferredNullability.NULLABLE, + ), + NullabilityRuleTestCase( + description="an unproven result", + args=( + InferredNullability.NON_NULL, + InferredNullability.NON_NULL, + InferredNullability.UNKNOWN, + ), + expected_first_arg=InferredNullability.NON_NULL, + expected_conditional=InferredNullability.UNKNOWN, + expected_safe_cast=InferredNullability.UNKNOWN, + ), + ], + ids=lambda case: case.description, +) +def test_given_argument_nullabilities_when_inferred_then_each_public_rule_returns_its_member( + test_case: NullabilityRuleTestCase, +) -> None: + assert first_arg_nullability(test_case.args) is test_case.expected_first_arg + assert conditional_result_nullability(test_case.args) is test_case.expected_conditional + assert safe_cast_nullability(test_case.args) is test_case.expected_safe_cast + + +@pytest.mark.parametrize( + "test_case", + [ + FirstArgIdentityTestCase( + description="plain text", argument="nullable", expected_same_object=True + ), + FirstArgIdentityTestCase( + description="an enum member", + argument=InferredNullability.NON_NULL, + expected_same_object=True, + ), + ], + ids=lambda case: case.description, +) +def test_given_first_argument_when_first_arg_inferred_then_the_argument_itself_returns( + test_case: FirstArgIdentityTestCase, +) -> None: + returned: object = first_arg_nullability((test_case.argument,)) # ty: ignore[invalid-argument-type] + + assert (returned is test_case.argument) is test_case.expected_same_object + + +if __name__ == "__main__": + pytest.main([__file__, "-vv"]) diff --git a/tests/unit/src/sqlbuild/cli/compile_reuse/_helpers/_test_types.py b/tests/unit/src/sqlbuild/cli/compile_reuse/_helpers/_test_types.py index 33442e434c..e01af2c24b 100644 --- a/tests/unit/src/sqlbuild/cli/compile_reuse/_helpers/_test_types.py +++ b/tests/unit/src/sqlbuild/cli/compile_reuse/_helpers/_test_types.py @@ -17,3 +17,25 @@ class ProviderSettingsInputsTestCase: expected_env_file_names: tuple[str, ...] = () expected_secrets_dir_names: tuple[str, ...] = () expected_unsupported: bool = False + + +@dataclass(frozen=True) +class StartupInputsTestCase: + """Import-path entries under a temporary root and the stamps they produce there.""" + + description: str + folders: tuple[str, ...] + files: tuple[str, ...] + path_entries: tuple[str, ...] + expected_stamps: tuple[tuple[str, bool], ...] + + +@dataclass(frozen=True) +class StartupInputsErrorTestCase: + """An import-path entry whose stat fails with an error other than a missing file.""" + + description: str + files: tuple[str, ...] + path_entry: str + expected_error: type[OSError] + expected_message: str diff --git a/tests/unit/src/sqlbuild/cli/compile_reuse/_helpers/test_startup_inputs.py b/tests/unit/src/sqlbuild/cli/compile_reuse/_helpers/test_startup_inputs.py new file mode 100644 index 0000000000..2d4e416b6a --- /dev/null +++ b/tests/unit/src/sqlbuild/cli/compile_reuse/_helpers/test_startup_inputs.py @@ -0,0 +1,85 @@ +import site +import sys +from pathlib import Path + +import pytest + +from sqlbuild.cli.compile_reuse._helpers.startup_inputs import startup_inputs +from tests.unit.src.sqlbuild.cli.compile_reuse._helpers._test_types import ( + StartupInputsErrorTestCase, + StartupInputsTestCase, +) + + +@pytest.mark.parametrize( + "test_case", + [ + StartupInputsTestCase( + description="relative, normalized and missing entries with their path files", + folders=("a", "a-b", "a/b"), + files=("a/q.pth", "a/notes.txt", "a-b/.pth"), + path_entries=("a", "./a-b/", "a/b", "gone"), + expected_stamps=( + ("a", True), + ("a/b", True), + ("a/q.pth", True), + ("a-b", True), + ("a-b/.pth", True), + ("gone", False), + ("python", False), + ("pyvenv.cfg", False), + ), + ), + ], + ids=lambda case: case.description, +) +def test_given_import_path_directories_when_stamping_then_lists_absolute_paths_and_pth_files( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch, test_case: StartupInputsTestCase +) -> None: + for folder in test_case.folders: + (tmp_path / folder).mkdir(parents=True) + for file in test_case.files: + (tmp_path / file).write_text("x\n") + monkeypatch.chdir(tmp_path) + monkeypatch.setattr(sys, "path", list(test_case.path_entries)) + monkeypatch.setattr(sys, "executable", str(tmp_path / "python")) + monkeypatch.setattr(sys, "prefix", str(tmp_path)) + monkeypatch.setattr(sys, "base_prefix", str(tmp_path)) + monkeypatch.setattr(site, "getsitepackages", list) + monkeypatch.setattr(site, "getusersitepackages", lambda: str(tmp_path / "gone")) + + stamps: list[tuple[str, int | None, int | None]] = startup_inputs() + listed: tuple[tuple[str, bool], ...] = tuple( + (path.removeprefix(f"{tmp_path}/"), size is not None) for path, _, size in stamps + ) + + assert listed == test_case.expected_stamps + assert stamps == sorted(stamps, key=lambda row: Path(row[0])) + + +@pytest.mark.parametrize( + "test_case", + [ + StartupInputsErrorTestCase( + description="an entry below a regular file", + files=("file.txt",), + path_entry="file.txt/sub", + expected_error=NotADirectoryError, + expected_message=r"\[Errno 20\] Not a directory", + ), + ], + ids=lambda case: case.description, +) +def test_given_import_path_entry_below_a_file_when_stamping_then_raises_the_stat_error( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch, test_case: StartupInputsErrorTestCase +) -> None: + for file in test_case.files: + (tmp_path / file).write_text("x\n") + monkeypatch.setattr(sys, "path", [str(tmp_path / test_case.path_entry)]) + + with pytest.raises(test_case.expected_error, match=test_case.expected_message): + startup_inputs() + + +if __name__ == "__main__": + pytest.main([__file__, "-vv"]) diff --git a/tests/unit/src/sqlbuild/python_nodes/main/_test_types.py b/tests/unit/src/sqlbuild/python_nodes/main/_test_types.py index 841628e6d2..fb7bbf10b5 100644 --- a/tests/unit/src/sqlbuild/python_nodes/main/_test_types.py +++ b/tests/unit/src/sqlbuild/python_nodes/main/_test_types.py @@ -3,6 +3,7 @@ from dataclasses import dataclass from sqlbuild.compiler.auditing.types import ThresholdOperator +from sqlbuild.python_nodes.models import RetryPolicy @dataclass(frozen=True) @@ -25,3 +26,21 @@ class InvalidAuditCaseNameTestCase: description: str name: str expected_error_fragment: str + + +@dataclass(frozen=True) +class RetryDelayTestCase: + description: str + policy: RetryPolicy + retry_index: int + expected_delay: float + expected_type: type + + +@dataclass(frozen=True) +class JitteredRetryDelayTestCase: + description: str + policy: RetryPolicy + retry_index: int + seed: int + expected_upper_bound: float diff --git a/tests/unit/src/sqlbuild/python_nodes/main/test_calculate_retry_delay.py b/tests/unit/src/sqlbuild/python_nodes/main/test_calculate_retry_delay.py new file mode 100644 index 0000000000..040bec869d --- /dev/null +++ b/tests/unit/src/sqlbuild/python_nodes/main/test_calculate_retry_delay.py @@ -0,0 +1,93 @@ +import random + +import pytest + +from sqlbuild.python_nodes.main.calculate_retry_delay import calculate_retry_delay +from sqlbuild.python_nodes.models import RetryPolicy +from tests.unit.src.sqlbuild.python_nodes.main._test_types import ( + JitteredRetryDelayTestCase, + RetryDelayTestCase, +) + + +@pytest.mark.parametrize( + "test_case", + [ + RetryDelayTestCase( + description="first retry waits the initial delay", + policy=RetryPolicy(jitter=False), + retry_index=0, + expected_delay=1.0, + expected_type=float, + ), + RetryDelayTestCase( + description="later retries grow by the multiplier", + policy=RetryPolicy(jitter=False, initial_delay_seconds=0.5, backoff_multiplier=3.0), + retry_index=2, + expected_delay=4.5, + expected_type=float, + ), + RetryDelayTestCase( + description="the delay is capped", + policy=RetryPolicy(jitter=False, max_delay_seconds=7.5), + retry_index=10, + expected_delay=7.5, + expected_type=float, + ), + RetryDelayTestCase( + description="integer settings keep an integer delay", + policy=RetryPolicy( + jitter=False, initial_delay_seconds=1, backoff_multiplier=2, max_delay_seconds=None + ), + retry_index=3, + expected_delay=8, + expected_type=int, + ), + ], + ids=lambda case: case.description, +) +def test_given_retry_policy_when_calculating_delay_then_returns_backoff_value_and_type( + test_case: RetryDelayTestCase, +) -> None: + delay: float = calculate_retry_delay( + retry_policy=test_case.policy, retry_index=test_case.retry_index + ) + + assert delay == test_case.expected_delay + assert type(delay) is test_case.expected_type + + +@pytest.mark.parametrize( + "test_case", + [ + JitteredRetryDelayTestCase( + description="uncapped jitter", + policy=RetryPolicy(max_delay_seconds=None, initial_delay_seconds=2.0), + retry_index=1, + seed=1234, + expected_upper_bound=4.0, + ), + JitteredRetryDelayTestCase( + description="capped jitter", + policy=RetryPolicy(max_delay_seconds=3.0), + retry_index=5, + seed=7, + expected_upper_bound=3.0, + ), + ], + ids=lambda case: case.description, +) +def test_given_jittered_policy_when_calculating_delay_then_draws_from_python_random( + test_case: JitteredRetryDelayTestCase, +) -> None: + random.seed(test_case.seed) + delay: float = calculate_retry_delay( + retry_policy=test_case.policy, retry_index=test_case.retry_index + ) + random.seed(test_case.seed) + + assert delay == random.uniform(0, test_case.expected_upper_bound) + + +if __name__ == "__main__": + pytest.main([__file__, "-vv"]) diff --git a/tests/unit/src/sqlbuild/rule_engine/_helpers/engine/test_hermeticity.py b/tests/unit/src/sqlbuild/rule_engine/_helpers/engine/test_hermeticity.py index 62a2dc8658..deee74da56 100644 --- a/tests/unit/src/sqlbuild/rule_engine/_helpers/engine/test_hermeticity.py +++ b/tests/unit/src/sqlbuild/rule_engine/_helpers/engine/test_hermeticity.py @@ -4,8 +4,7 @@ import pytest -from sqlbuild._native import NonHermeticRuleError, Rule -from sqlbuild.rule_engine._helpers.engine.hermeticity import verify_custom_rules +from sqlbuild._native import NonHermeticRuleError, Rule, verify_custom_rules from tests.unit.src.sqlbuild.rule_engine._helpers.engine._test_types import ( CustomRuleImportTestCase, ) From 65c41ccd0af47255fb47599b085be404dff0735d Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Fri, 16 Oct 2026 01:22:27 +0100 Subject: [PATCH 42/48] chore: drop fensu exceptions for removed Python packages --- fensu.toml | 10 ---------- 1 file changed, 10 deletions(-) diff --git a/fensu.toml b/fensu.toml index 0093a15632..db6188ab05 100644 --- a/fensu.toml +++ b/fensu.toml @@ -73,11 +73,6 @@ rule = "FFR309" path = "src/sqlbuild/compiler/migrations/__init__.py" reason = "Native bindings own migration planning and state rows; this package only declares the migration models, decisions and table names the executor imports." -[[targets.python.rule_exceptions]] -rule = "FFR309" -path = "src/sqlbuild/compiler/source_freshness/__init__.py" -reason = "Native bindings own source freshness observation, planning, propagation and state reads and writes; this package only declares the models, types and constants they build." - [[targets.python.rule_exceptions]] rule = "FFR309" path = "src/sqlbuild/compiler/pipeline/__init__.py" @@ -163,11 +158,6 @@ rule = "FFR309" path = "src/sqlbuild/sql_values/exceptions.py" reason = "Typed SQL values are normalized and state literals rendered natively; the namespace package keeps the value models, types and the rendering error adapters raise." -[[targets.python.rule_exceptions]] -rule = "FFR309" -path = "src/sqlbuild/cli/output/__init__.py" -reason = "Execution-result output is rendered natively; the package keeps only the integration-result constants its callers and readers share." - [[targets.python.rule_exceptions]] rule = "FFR309" path = "src/sqlbuild/compiler/lineage/__init__.py" From e9c014612b25b2e58ea9016d2317fba66c841354 Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Fri, 16 Oct 2026 01:30:51 +0100 Subject: [PATCH 43/48] fix: keep tolerating None in hook reads as before --- .../_helpers/authored_values/python_nodes.rs | 7 +++ .../python_nodes/decorators/_test_types.py | 8 +++ .../decorators/test_python_decorators.py | 54 +++++++++++++++++++ 3 files changed, 69 insertions(+) diff --git a/crates/sqlbuild-python/src/bindings/_helpers/authored_values/python_nodes.rs b/crates/sqlbuild-python/src/bindings/_helpers/authored_values/python_nodes.rs index 1172de6137..dc6fb69b7e 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/authored_values/python_nodes.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/authored_values/python_nodes.rs @@ -183,8 +183,12 @@ fn normalize_hook_reads<'py>(value: &Bound<'py, PyAny>) -> PyResult>()? }; + // Only the first non-reference item is checked, and a `None` there is tolerated, as before. for item in &reads { if !item.is_instance(&reference)? { + if item.is_none() { + break; + } let message: String = format!("{HOOK_READS_INVALID}{}", item.get_type().name()?); return Err(shared_input_error(py, &message)); } @@ -192,6 +196,9 @@ fn normalize_hook_reads<'py>(value: &Bound<'py, PyAny>) -> PyResult = PyDict::new(py); let mut ordered: Vec> = Vec::new(); for item in reads { + if !item.is_instance(&reference)? { + continue; + } if !distinct.contains(&item)? { distinct.set_item(&item, py.None())?; ordered.push(item); diff --git a/tests/unit/src/sqlbuild/python_nodes/decorators/_test_types.py b/tests/unit/src/sqlbuild/python_nodes/decorators/_test_types.py index 8cc435a57e..346a42c7e6 100644 --- a/tests/unit/src/sqlbuild/python_nodes/decorators/_test_types.py +++ b/tests/unit/src/sqlbuild/python_nodes/decorators/_test_types.py @@ -54,3 +54,11 @@ class FactoryDecoratorMetadataTestCase: class FactoryDecoratorErrorTestCase: description: str expected_error_fragment: str + + +@dataclass(frozen=True) +class HookReadsTestCase: + description: str + reads: object + expected_reads: tuple[str, ...] | None + expected_error: str | None = None diff --git a/tests/unit/src/sqlbuild/python_nodes/decorators/test_python_decorators.py b/tests/unit/src/sqlbuild/python_nodes/decorators/test_python_decorators.py index 87b8d6c06e..f155354d1d 100644 --- a/tests/unit/src/sqlbuild/python_nodes/decorators/test_python_decorators.py +++ b/tests/unit/src/sqlbuild/python_nodes/decorators/test_python_decorators.py @@ -8,7 +8,9 @@ from sqlbuild.assets import asset, get_asset_definition from sqlbuild.checks import check, get_check_definition +from sqlbuild.errors.contracts.exceptions import SharedInputError from sqlbuild.factories import factory, get_factory_definition +from sqlbuild.hooks import get_hook_definition, hook from sqlbuild.python_nodes.models import ( AssetDefinition, CheckDefinition, @@ -18,12 +20,14 @@ TaskDefinition, ) from sqlbuild.python_nodes.types import PythonCheckSeverity +from sqlbuild.refs import model, seed from sqlbuild.tasks import get_task_definition, task from tests.unit.src.sqlbuild.python_nodes.decorators._test_types import ( AssetDecoratorMetadataTestCase, CheckDecoratorMetadataTestCase, FactoryDecoratorErrorTestCase, FactoryDecoratorMetadataTestCase, + HookReadsTestCase, TaskDecoratorMetadataTestCase, ) from tests.unit.src.sqlbuild.python_nodes.decorators.helpers import upstream_asset, upstream_task @@ -244,3 +248,53 @@ def test_given_factory_decorator_when_called_without_function_then_raises( factory_ref: Any = cast(Any, factory) with pytest.raises(TypeError, match=test_case.expected_error_fragment): factory_ref() + + +@pytest.mark.parametrize( + "test_case", + [ + HookReadsTestCase( + description="single reference becomes one read", + reads=model("orders"), + expected_reads=("orders",), + ), + HookReadsTestCase( + description="duplicate references keep first-seen order", + reads=[model("orders"), seed("customers"), model("orders")], + expected_reads=("orders", "customers"), + ), + HookReadsTestCase( + description="None as the first invalid item is dropped", + reads=[model("orders"), None, model("products")], + expected_reads=("orders", "products"), + ), + HookReadsTestCase( + description="items after a leading None are filtered, not rejected", + reads=[None, 5, model("orders")], + expected_reads=("orders",), + ), + HookReadsTestCase( + description="a non-reference item is rejected", + reads=[model("orders"), "customers"], + expected_reads=None, + expected_error="got str", + ), + ], + ids=lambda case: case.description, +) +def test_given_hook_reads_when_decorating_then_normalizes_like_the_documented_contract( + test_case: HookReadsTestCase, +) -> None: + def refresh_orders() -> None: + return None + + if test_case.expected_error is not None: + with pytest.raises(SharedInputError, match=test_case.expected_error): + cast(Any, hook(reads=cast(Any, test_case.reads)))(refresh_orders) + return + + decorated: Any = cast(Any, hook(reads=cast(Any, test_case.reads)))(refresh_orders) + definition = get_hook_definition(decorated) + + assert definition is not None + assert tuple(read.name for read in definition.reads) == test_case.expected_reads From 11385cc6f7afb07af4f73e56f7c7e32f7c705890 Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Fri, 16 Oct 2026 01:29:23 +0100 Subject: [PATCH 44/48] refactor: run project-tree queries and tracked-file selection natively --- .../src/bindings/_helpers/rules/mod.rs | 1 + .../_helpers/rules/project_tree_queries.rs | 180 ++++++++++++++++++ .../bindings/_helpers/rules/registration.rs | 1 + .../src/bindings/_helpers/rules/rule_api.rs | 4 +- .../src/project_tree/_helpers/files.rs | 12 +- .../project_tree/_helpers/relative_paths.rs | 70 ++++++- .../src/project_tree/constants.rs | 6 + .../sqlbuild-rules/src/project_tree/errors.rs | 11 ++ .../src/project_tree/main/mod.rs | 5 + .../main/project_relative_parts.rs | 32 ++++ .../main/project_tree_children.rs | 17 ++ .../main/project_tree_descendants.rs | 29 +++ .../project_tree/main/project_tree_glob.rs | 20 ++ .../project_tree/main/tracked_text_path.rs | 17 ++ .../sqlbuild-rules/src/project_tree/tests.rs | 2 + .../project_tree/tests/test_path_queries.rs | 176 +++++++++++++++++ .../src/project_tree/tests/test_types.rs | 24 ++- python-boundary.toml | 4 +- src/sqlbuild/_native.pyi | 13 ++ .../_helpers/engine/fact_replay.py | 8 +- .../rule_engine/classes/project_tree.py | 28 +-- .../classes/project_tree/_test_types.py | 16 ++ .../classes/project_tree/test_queries.py | 152 +++++++++++++++ 23 files changed, 788 insertions(+), 40 deletions(-) create mode 100644 crates/sqlbuild-python/src/bindings/_helpers/rules/project_tree_queries.rs create mode 100644 crates/sqlbuild-rules/src/project_tree/main/project_relative_parts.rs create mode 100644 crates/sqlbuild-rules/src/project_tree/main/project_tree_children.rs create mode 100644 crates/sqlbuild-rules/src/project_tree/main/project_tree_descendants.rs create mode 100644 crates/sqlbuild-rules/src/project_tree/main/project_tree_glob.rs create mode 100644 crates/sqlbuild-rules/src/project_tree/main/tracked_text_path.rs create mode 100644 crates/sqlbuild-rules/src/project_tree/tests/test_path_queries.rs create mode 100644 tests/unit/src/sqlbuild/rule_engine/classes/project_tree/test_queries.py diff --git a/crates/sqlbuild-python/src/bindings/_helpers/rules/mod.rs b/crates/sqlbuild-python/src/bindings/_helpers/rules/mod.rs index fade28f354..5c3ad0aa26 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/rules/mod.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/rules/mod.rs @@ -23,6 +23,7 @@ pub(crate) mod lint_preparation; pub(crate) mod lint_sql_bodies; pub(crate) mod lint_stop; pub(crate) mod lint_text; +pub(crate) mod project_tree_queries; pub(crate) mod registration; pub(crate) mod relation_keys; pub(crate) mod results; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/rules/project_tree_queries.rs b/crates/sqlbuild-python/src/bindings/_helpers/rules/project_tree_queries.rs new file mode 100644 index 0000000000..62a6f2b4dd --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/rules/project_tree_queries.rs @@ -0,0 +1,180 @@ +//! Project-tree queries custom rules run over the discovered paths. + +use std::ffi::OsString; +use std::path::PathBuf; + +use pyo3::exceptions::PyValueError; +use pyo3::prelude::{ + Bound, PyAny, PyAnyMethods, PyErr, PyModule, PyModuleMethods, PyResult, Python, +}; +use pyo3::types::{PyString, PyTuple}; +use pyo3::{IntoPyObject, pyfunction, wrap_pyfunction}; +use sqlbuild_rules::project_tree::errors::RelativePartsError; +use sqlbuild_rules::project_tree::main::project_relative_parts::project_relative_parts; +use sqlbuild_rules::project_tree::main::project_tree_children::project_tree_children; +use sqlbuild_rules::project_tree::main::project_tree_descendants::project_tree_descendants; +use sqlbuild_rules::project_tree::main::project_tree_glob::project_tree_glob; +use sqlbuild_rules::project_tree::main::tracked_text_path::tracked_text_path; + +use crate::bindings::_helpers::compile_driver::python_calls::imported; +use crate::bindings::_helpers::rules::rule_api::{usage_error_for, validate_project_relative}; + +const PATHLIB_MODULE: &str = "pathlib"; +const RULE_USAGE_ERROR: &str = "RuleUsageError"; +const NATIVE_MODULE: &str = "sqlbuild._native"; + +/// `pathlib.Path(value)`, raising exactly as `Path` does for an unsupported value. +fn python_path<'py>(value: &Bound<'py, PyAny>) -> PyResult> { + imported(value.py(), PATHLIB_MODULE, "Path")?.call1((value,)) +} + +/// The nodes and their `value`s as raw paths. +fn node_values<'py>(nodes: &Bound<'py, PyAny>) -> PyResult<(Vec>, Vec)> { + let nodes: Vec> = nodes.try_iter()?.collect::>()?; + let values: Vec = nodes + .iter() + .map(|node| node.getattr("value")?.extract()) + .collect::>()?; + Ok((nodes, values)) +} + +/// The nodes at `indexes`, in order. +fn selected<'py>( + py: Python<'py>, + nodes: &[Bound<'py, PyAny>], + indexes: Vec, +) -> PyResult> { + PyTuple::new(py, indexes.into_iter().map(|index| nodes[index].clone())) +} + +/// The nodes directly inside the project directory `path`. +#[pyfunction] +fn project_tree_child_nodes<'py>( + nodes: &Bound<'py, PyAny>, + path: &Bound<'py, PyAny>, +) -> PyResult> { + let directory: PathBuf = python_path(path)?.extract()?; + let (nodes, values) = node_values(nodes)?; + let values: Vec<&std::path::Path> = values.iter().map(PathBuf::as_path).collect(); + selected( + path.py(), + &nodes, + project_tree_children(&values, &directory), + ) +} + +/// The nodes below the project directory `path`, or every node for an empty path. +#[pyfunction] +fn project_tree_descendant_nodes<'py>( + nodes: &Bound<'py, PyAny>, + path: &Bound<'py, PyAny>, +) -> PyResult> { + let directory: Option = if path.is_truthy()? { + Some(python_path(path)?.extract()?) + } else { + None + }; + let (nodes, values) = node_values(nodes)?; + let values: Vec<&std::path::Path> = values.iter().map(PathBuf::as_path).collect(); + selected( + path.py(), + &nodes, + project_tree_descendants(&values, directory.as_deref()), + ) +} + +/// The nodes a project-relative shell-style pattern matches. +#[pyfunction] +fn project_tree_glob_nodes<'py>( + nodes: &Bound<'py, PyAny>, + pattern: &Bound<'py, PyAny>, +) -> PyResult> { + let py: Python<'py> = pattern.py(); + validate_project_relative(pattern)?; + let pattern: PathBuf = pattern.extract()?; + let (nodes, values) = node_values(nodes)?; + let values: Vec<&std::path::Path> = values.iter().map(PathBuf::as_path).collect(); + selected(py, &nodes, project_tree_glob(&values, &pattern)) +} + +/// The values of the file nodes rules may read as tracked text, in order. +#[pyfunction] +fn tracked_text_file_values<'py>(nodes: &Bound<'py, PyAny>) -> PyResult> { + let mut values: Vec> = Vec::new(); + for node in nodes.try_iter()? { + let node: Bound<'py, PyAny> = node?; + if !node.getattr("is_file")?.is_truthy()? { + continue; + } + let value: Bound<'py, PyAny> = node.getattr("value")?; + if tracked_text_path(&value.extract::()?) { + values.push(value); + } + } + PyTuple::new(nodes.py(), values) +} + +/// `RuleUsageError(f"project path {path} is not beneath {under}")`, spelled by Python. +fn not_beneath(path: &Bound<'_, PyAny>, under: &Bound<'_, PyAny>) -> PyErr { + let py: Python<'_> = path.py(); + let message = path.str().and_then(|path_text| { + let under_text = under.str()?; + PyString::new(py, "project path ") + .as_any() + .add(path_text)? + .add(" is not beneath ")? + .add(under_text) + }); + match message.and_then(|message| { + imported(py, NATIVE_MODULE, RULE_USAGE_ERROR).and_then(|class| class.call1((message,))) + }) { + Ok(error) => PyErr::from_value(error), + Err(error) => error, + } +} + +/// `path`'s parts relative to the resolved project `root`, then to `under` when given. +#[pyfunction] +#[pyo3(signature = (*, root, path, under))] +fn project_tree_relative_parts<'py>( + root: &Bound<'py, PyAny>, + path: &Bound<'py, PyAny>, + under: &Bound<'py, PyAny>, +) -> PyResult> { + let py: Python<'py> = path.py(); + let relative: Bound<'py, PyAny> = python_path(path)?; + let under_path: Option = if under.is_truthy()? { + Some(python_path(under)?.extract()?) + } else { + None + }; + let parts: Vec = match project_relative_parts( + &relative.extract::()?, + &root.extract::()?, + under_path.as_deref(), + |_| relative.call_method0("resolve")?.extract::(), + ) { + Ok(parts) => parts, + Err(RelativePartsError::Resolve(error)) if !error.is_instance_of::(py) => { + return Err(error); + } + Err(RelativePartsError::Resolve(_) | RelativePartsError::Escapes) => { + return Err(usage_error_for("project path escapes the project: ", path)); + } + Err(RelativePartsError::NotBeneath) => return Err(not_beneath(path, under)), + }; + let parts: Vec> = parts + .into_iter() + .map(|part| Ok(part.into_pyobject(py)?.into_any())) + .collect::>()?; + PyTuple::new(py, parts) +} + +pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { + module.add_function(wrap_pyfunction!(project_tree_child_nodes, module)?)?; + module.add_function(wrap_pyfunction!(project_tree_descendant_nodes, module)?)?; + module.add_function(wrap_pyfunction!(project_tree_glob_nodes, module)?)?; + module.add_function(wrap_pyfunction!(tracked_text_file_values, module)?)?; + module.add_function(wrap_pyfunction!(project_tree_relative_parts, module)?)?; + Ok(()) +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/rules/registration.rs b/crates/sqlbuild-python/src/bindings/_helpers/rules/registration.rs index 38120db142..802295076a 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/rules/registration.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/rules/registration.rs @@ -13,6 +13,7 @@ use crate::bindings::_helpers::rules::{ pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { rule_api::register(module)?; + crate::bindings::_helpers::rules::project_tree_queries::register(module)?; module.add_function(wrap_pyfunction!( crate::bindings::_helpers::rules::lint_headers::lint_file_role, module diff --git a/crates/sqlbuild-python/src/bindings/_helpers/rules/rule_api.rs b/crates/sqlbuild-python/src/bindings/_helpers/rules/rule_api.rs index 2ccf4ce6fc..5de87ba555 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/rules/rule_api.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/rules/rule_api.rs @@ -39,7 +39,7 @@ fn usage_error(py: Python<'_>, message: String) -> PyErr { } /// `RuleUsageError(prefix + str(value))`, keeping the value's text exactly as Python spells it. -fn usage_error_for(prefix: &str, value: &Bound<'_, PyAny>) -> PyErr { +pub(crate) fn usage_error_for(prefix: &str, value: &Bound<'_, PyAny>) -> PyErr { let py: Python<'_> = value.py(); let message = value .str() @@ -140,7 +140,7 @@ fn rule_option_toml_value(value: &Bound<'_, PyAny>) -> PyResult { /// Reject a project path that is absolute or climbs out of the project. #[pyfunction] -fn validate_project_relative(path: &Bound<'_, PyAny>) -> Result<(), PyErr> { +pub(crate) fn validate_project_relative(path: &Bound<'_, PyAny>) -> Result<(), PyErr> { if posix_escapes_root(&path.str()?.to_string_lossy()) { return Err(usage_error_for("project path escapes the project: ", path)); } diff --git a/crates/sqlbuild-rules/src/project_tree/_helpers/files.rs b/crates/sqlbuild-rules/src/project_tree/_helpers/files.rs index a0a2ae5f23..b036a3a5c0 100644 --- a/crates/sqlbuild-rules/src/project_tree/_helpers/files.rs +++ b/crates/sqlbuild-rules/src/project_tree/_helpers/files.rs @@ -4,10 +4,9 @@ use std::ffi::OsStr; use std::os::unix::ffi::OsStrExt; use std::path::{Path, PathBuf}; -use sqlbuild_core::posix_path::main::posix_suffix::posix_suffix; - -use crate::project_tree::constants::{MISSING_FILE_ERRNOS, TRACKED_TEXT_SUFFIXES}; +use crate::project_tree::constants::MISSING_FILE_ERRNOS; use crate::project_tree::errors::ProjectTreeFileError; +use crate::project_tree::main::tracked_text_path::tracked_text_path; /// `root` joined with the raw path components. pub(crate) fn joined(root: &Path, parts: &[&[u8]]) -> PathBuf { @@ -51,12 +50,7 @@ pub(crate) fn regular_file(root: &Path, parts: &[&[u8]]) -> Option { /// `candidate` when its lowercased suffix is a tracked text type. pub(crate) fn tracked(candidate: PathBuf) -> Result { - let name: String = candidate - .file_name() - .map(|name| name.to_string_lossy().into_owned()) - .unwrap_or_default(); - let suffix: String = posix_suffix(&name).to_lowercase(); - if TRACKED_TEXT_SUFFIXES.contains(&suffix.as_str()) { + if tracked_text_path(&candidate) { Ok(candidate) } else { Err(ProjectTreeFileError::Untracked) diff --git a/crates/sqlbuild-rules/src/project_tree/_helpers/relative_paths.rs b/crates/sqlbuild-rules/src/project_tree/_helpers/relative_paths.rs index 5cb7efab91..3fc2996153 100644 --- a/crates/sqlbuild-rules/src/project_tree/_helpers/relative_paths.rs +++ b/crates/sqlbuild-rules/src/project_tree/_helpers/relative_paths.rs @@ -3,7 +3,9 @@ use std::os::unix::ffi::OsStrExt; use std::path::Path; -use crate::project_tree::constants::{CURRENT, PARENT, SEPARATOR}; +use crate::project_tree::constants::{ + CURRENT, DOUBLE_ROOT, ESCAPED_BYTE_BASE, PARENT, ROOT, SEPARATOR, +}; /// The components `PurePosixPath(path).parts` keeps after any root. pub(crate) fn parts(path: &Path) -> Vec<&[u8]> { @@ -18,3 +20,69 @@ pub(crate) fn parts(path: &Path) -> Vec<&[u8]> { pub(crate) fn escapes(path: &Path) -> bool { path.as_os_str().as_bytes().first() == Some(&SEPARATOR) || parts(path).contains(&PARENT) } + +/// `PurePosixPath(path).parts`: its root (`/` or exactly `//`) first, then each component. +pub(crate) fn rooted_parts(path: &Path) -> Vec<&[u8]> { + let bytes: &[u8] = path.as_os_str().as_bytes(); + let mut rooted: Vec<&[u8]> = Vec::new(); + if bytes.starts_with(DOUBLE_ROOT) && bytes.get(DOUBLE_ROOT.len()) != Some(&SEPARATOR) { + rooted.push(DOUBLE_ROOT); + } else if bytes.first() == Some(&SEPARATOR) { + rooted.push(ROOT); + } + rooted.extend(parts(path)); + rooted +} + +/// Whether a part list starts with a root. +pub(crate) fn is_rooted(parts: &[&[u8]]) -> bool { + parts + .first() + .is_some_and(|first| *first == ROOT || *first == DOUBLE_ROOT) +} + +/// `str(PurePosixPath(*parts))` as bytes: the root, the components joined by `/`, or `.`. +pub(crate) fn joined_text(parts: &[&[u8]]) -> Vec { + let (root, components): (&[u8], &[&[u8]]) = match parts.split_first() { + Some((first, rest)) if *first == ROOT || *first == DOUBLE_ROOT => (first, rest), + _ => (&[], parts), + }; + if root.is_empty() && components.is_empty() { + return CURRENT.to_vec(); + } + let mut text: Vec = root.to_vec(); + text.extend(components.join(&SEPARATOR)); + text +} + +/// `PurePosixPath(*parts).parent` as parts: a bare root or `.` is its own parent. +pub(crate) fn parent_parts<'a>(parts: &[&'a [u8]]) -> Vec<&'a [u8]> { + match parts { + [] => Vec::new(), + [only] if is_rooted(&[only]) => vec![*only], + [init @ .., _] => init.to_vec(), + } +} + +/// `PurePosixPath(*parts).relative_to(PurePosixPath(*base)).parts`, or `None` when not beneath. +pub(crate) fn relative_parts_to<'a>(parts: &[&'a [u8]], base: &[&[u8]]) -> Option> { + let beneath: bool = parts.len() >= base.len() + && parts.iter().zip(base).all(|(part, prefix)| part == prefix) + && !(base.is_empty() && is_rooted(parts)); + beneath.then(|| parts[base.len()..].to_vec()) +} + +/// Path bytes as text with one character per byte UTF-8 cannot decode, as `surrogateescape` reads. +pub(crate) fn escaped_text(bytes: &[u8]) -> String { + let mut text: String = String::with_capacity(bytes.len()); + for chunk in bytes.utf8_chunks() { + text.push_str(chunk.valid()); + text.extend( + chunk + .invalid() + .iter() + .filter_map(|byte| char::from_u32(ESCAPED_BYTE_BASE + u32::from(*byte))), + ); + } + text +} diff --git a/crates/sqlbuild-rules/src/project_tree/constants.rs b/crates/sqlbuild-rules/src/project_tree/constants.rs index 76e93fe745..4e4f7e9c41 100644 --- a/crates/sqlbuild-rules/src/project_tree/constants.rs +++ b/crates/sqlbuild-rules/src/project_tree/constants.rs @@ -12,5 +12,11 @@ pub(crate) const MISSING_FILE_ERRNOS: [i32; 4] = [2, 20, 9, 62]; #[cfg(not(target_os = "macos"))] pub(crate) const MISSING_FILE_ERRNOS: [i32; 4] = [2, 20, 9, 40]; pub(crate) const SEPARATOR: u8 = b'/'; +pub(crate) const ROOT: &[u8] = b"/"; +/// POSIX keeps exactly two leading slashes as their own root. +pub(crate) const DOUBLE_ROOT: &[u8] = b"//"; +pub(crate) const DESCENDANT_SEPARATOR: &[u8] = b"/"; pub(crate) const CURRENT: &[u8] = b"."; pub(crate) const PARENT: &[u8] = b".."; +/// Private-use characters standing for undecodable path bytes, one per byte value. +pub(crate) const ESCAPED_BYTE_BASE: u32 = 0xF_0000; diff --git a/crates/sqlbuild-rules/src/project_tree/errors.rs b/crates/sqlbuild-rules/src/project_tree/errors.rs index 155a903fa1..9a49f8f65f 100644 --- a/crates/sqlbuild-rules/src/project_tree/errors.rs +++ b/crates/sqlbuild-rules/src/project_tree/errors.rs @@ -18,3 +18,14 @@ pub enum ProjectTreeFileError { /// Testing the resolved path failed with this `errno`. Unreadable { path: PathBuf, errno: i32 }, } + +/// Why a path has no parts relative to the project or the requested directory. +#[derive(Clone, Debug, PartialEq, Eq)] +pub enum RelativePartsError { + /// Resolving the absolute path failed. + Resolve(E), + /// The resolved path is outside the project. + Escapes, + /// The path is not beneath the requested directory. + NotBeneath, +} diff --git a/crates/sqlbuild-rules/src/project_tree/main/mod.rs b/crates/sqlbuild-rules/src/project_tree/main/mod.rs index fa8c2bcca2..1045d8d2b0 100644 --- a/crates/sqlbuild-rules/src/project_tree/main/mod.rs +++ b/crates/sqlbuild-rules/src/project_tree/main/mod.rs @@ -1,3 +1,8 @@ +pub mod project_relative_parts; +pub mod project_tree_children; +pub mod project_tree_descendants; pub mod project_tree_entries; pub mod project_tree_file; +pub mod project_tree_glob; +pub mod tracked_text_path; pub mod tracked_text_suffixes; diff --git a/crates/sqlbuild-rules/src/project_tree/main/project_relative_parts.rs b/crates/sqlbuild-rules/src/project_tree/main/project_relative_parts.rs new file mode 100644 index 0000000000..1d70ed58a8 --- /dev/null +++ b/crates/sqlbuild-rules/src/project_tree/main/project_relative_parts.rs @@ -0,0 +1,32 @@ +//! The parts of a path relative to the project and an optional directory inside it. + +use std::ffi::{OsStr, OsString}; +use std::os::unix::ffi::OsStrExt; +use std::path::{Path, PathBuf}; + +use crate::project_tree::_helpers::relative_paths::{is_rooted, relative_parts_to, rooted_parts}; +use crate::project_tree::errors::RelativePartsError; + +/// `path`'s parts, made project-relative through `resolve` when absolute, then relative to `under`. +pub fn project_relative_parts( + path: &Path, + root: &Path, + under: Option<&Path>, + resolve: impl FnOnce(&Path) -> Result, +) -> Result, RelativePartsError> { + let resolved: PathBuf; + let mut parts: Vec<&[u8]> = rooted_parts(path); + if is_rooted(&parts) { + resolved = resolve(path).map_err(RelativePartsError::Resolve)?; + parts = relative_parts_to(&rooted_parts(&resolved), &rooted_parts(root)) + .ok_or(RelativePartsError::Escapes)?; + } + if let Some(under) = under { + parts = relative_parts_to(&parts, &rooted_parts(under)) + .ok_or(RelativePartsError::NotBeneath)?; + } + Ok(parts + .into_iter() + .map(|part| OsStr::from_bytes(part).to_os_string()) + .collect()) +} diff --git a/crates/sqlbuild-rules/src/project_tree/main/project_tree_children.rs b/crates/sqlbuild-rules/src/project_tree/main/project_tree_children.rs new file mode 100644 index 0000000000..d8ba41b719 --- /dev/null +++ b/crates/sqlbuild-rules/src/project_tree/main/project_tree_children.rs @@ -0,0 +1,17 @@ +//! The listed paths directly inside one project directory. + +use std::path::Path; + +use crate::project_tree::_helpers::relative_paths::{joined_text, parent_parts, rooted_parts}; + +/// Indexes of the `values` whose `PurePosixPath(value).parent == PurePosixPath(directory)`. +#[must_use] +pub fn project_tree_children(values: &[&Path], directory: &Path) -> Vec { + let directory: Vec = joined_text(&rooted_parts(directory)); + values + .iter() + .enumerate() + .filter(|(_, value)| joined_text(&parent_parts(&rooted_parts(value))) == directory) + .map(|(index, _)| index) + .collect() +} diff --git a/crates/sqlbuild-rules/src/project_tree/main/project_tree_descendants.rs b/crates/sqlbuild-rules/src/project_tree/main/project_tree_descendants.rs new file mode 100644 index 0000000000..0450fad701 --- /dev/null +++ b/crates/sqlbuild-rules/src/project_tree/main/project_tree_descendants.rs @@ -0,0 +1,29 @@ +//! The listed paths below one project directory. + +use std::os::unix::ffi::OsStrExt; +use std::path::Path; + +use crate::project_tree::_helpers::relative_paths::{joined_text, rooted_parts}; +use crate::project_tree::constants::{DESCENDANT_SEPARATOR, SEPARATOR}; + +/// Indexes of the `values` below `directory` as `as_posix().rstrip("/") + "/"` prefixes them. +#[must_use] +pub fn project_tree_descendants(values: &[&Path], directory: Option<&Path>) -> Vec { + let prefix: Vec = match directory { + Some(directory) => { + let mut text: Vec = joined_text(&rooted_parts(directory)); + while text.last() == Some(&SEPARATOR) { + text.pop(); + } + text.extend_from_slice(DESCENDANT_SEPARATOR); + text + } + None => Vec::new(), + }; + values + .iter() + .enumerate() + .filter(|(_, value)| value.as_os_str().as_bytes().starts_with(&prefix)) + .map(|(index, _)| index) + .collect() +} diff --git a/crates/sqlbuild-rules/src/project_tree/main/project_tree_glob.rs b/crates/sqlbuild-rules/src/project_tree/main/project_tree_glob.rs new file mode 100644 index 0000000000..a94a230d9c --- /dev/null +++ b/crates/sqlbuild-rules/src/project_tree/main/project_tree_glob.rs @@ -0,0 +1,20 @@ +//! The listed paths a shell-style pattern matches. + +use std::os::unix::ffi::OsStrExt; +use std::path::Path; + +use sqlbuild_core::text::main::glob_match::glob_match; + +use crate::project_tree::_helpers::relative_paths::escaped_text; + +/// Indexes of the `values` that `fnmatch.fnmatchcase(value, pattern)` accepts. +#[must_use] +pub fn project_tree_glob(values: &[&Path], pattern: &Path) -> Vec { + let pattern: String = escaped_text(pattern.as_os_str().as_bytes()); + values + .iter() + .enumerate() + .filter(|(_, value)| glob_match(&escaped_text(value.as_os_str().as_bytes()), &pattern)) + .map(|(index, _)| index) + .collect() +} diff --git a/crates/sqlbuild-rules/src/project_tree/main/tracked_text_path.rs b/crates/sqlbuild-rules/src/project_tree/main/tracked_text_path.rs new file mode 100644 index 0000000000..91be1ef8be --- /dev/null +++ b/crates/sqlbuild-rules/src/project_tree/main/tracked_text_path.rs @@ -0,0 +1,17 @@ +//! Whether a project path names a text file type rules may read. + +use std::path::Path; + +use sqlbuild_core::posix_path::main::posix_suffix::posix_suffix; + +use crate::project_tree::constants::TRACKED_TEXT_SUFFIXES; + +/// Whether the lowercased suffix of the path's name is a tracked text type. +#[must_use] +pub fn tracked_text_path(path: &Path) -> bool { + let name: String = path + .file_name() + .map(|name| name.to_string_lossy().into_owned()) + .unwrap_or_default(); + TRACKED_TEXT_SUFFIXES.contains(&posix_suffix(&name).to_lowercase().as_str()) +} diff --git a/crates/sqlbuild-rules/src/project_tree/tests.rs b/crates/sqlbuild-rules/src/project_tree/tests.rs index 6993fa3916..f6bd42db5d 100644 --- a/crates/sqlbuild-rules/src/project_tree/tests.rs +++ b/crates/sqlbuild-rules/src/project_tree/tests.rs @@ -1,5 +1,7 @@ #[path = "tests/helpers.rs"] mod helpers; +#[path = "tests/test_path_queries.rs"] +mod path_queries; #[path = "tests/test_project_tree.rs"] mod project_tree; #[path = "tests/test_types.rs"] diff --git a/crates/sqlbuild-rules/src/project_tree/tests/test_path_queries.rs b/crates/sqlbuild-rules/src/project_tree/tests/test_path_queries.rs new file mode 100644 index 0000000000..b56f909415 --- /dev/null +++ b/crates/sqlbuild-rules/src/project_tree/tests/test_path_queries.rs @@ -0,0 +1,176 @@ +use std::path::{Path, PathBuf}; + +use crate::project_tree::errors::RelativePartsError; +use crate::project_tree::main::project_relative_parts::project_relative_parts; +use crate::project_tree::main::project_tree_children::project_tree_children; +use crate::project_tree::main::project_tree_descendants::project_tree_descendants; +use crate::project_tree::main::project_tree_glob::project_tree_glob; +use crate::project_tree::main::tracked_text_path::tracked_text_path; +use crate::project_tree::tests::test_types::{ + PathSelectionTestCase, RelativePartsTestCase, TrackedTextTestCase, +}; + +const VALUES: [&str; 6] = [ + "models", + "models/marts", + "models/marts/orders.sql", + "models/a.sql", + "rules", + "readme.md", +]; + +#[test] +fn given_listed_paths_when_selecting_then_children_descendants_and_globs_match_pathlib() { + let test_cases = [ + PathSelectionTestCase { + description: "the project root", + argument: None, + expected_children: vec!["models", "rules", "readme.md"], + expected_descendants: VALUES.to_vec(), + expected_glob: vec![], + }, + PathSelectionTestCase { + description: "a folder spelled with dots and slashes", + argument: Some("./models//"), + expected_children: vec!["models/marts", "models/a.sql"], + expected_descendants: vec!["models/marts", "models/marts/orders.sql", "models/a.sql"], + expected_glob: vec![], + }, + PathSelectionTestCase { + description: "a star pattern crossing folders", + argument: Some("*.sql"), + expected_children: vec![], + expected_descendants: vec![], + expected_glob: vec!["models/marts/orders.sql", "models/a.sql"], + }, + PathSelectionTestCase { + description: "a negated set", + argument: Some("[!m]*"), + expected_children: vec![], + expected_descendants: vec![], + expected_glob: vec!["rules", "readme.md"], + }, + ]; + let values: Vec<&Path> = VALUES.iter().map(Path::new).collect(); + let pick = |indexes: Vec| -> Vec<&str> { + indexes.into_iter().map(|index| VALUES[index]).collect() + }; + for test_case in test_cases { + let argument: &Path = Path::new(test_case.argument.unwrap_or("")); + assert_eq!( + ( + pick(project_tree_children(&values, argument)), + pick(project_tree_descendants( + &values, + test_case.argument.map(Path::new) + )), + pick(project_tree_glob(&values, argument)), + ), + ( + test_case.expected_children, + test_case.expected_descendants, + test_case.expected_glob + ), + "{}", + test_case.description + ); + } +} + +#[test] +fn given_paths_when_made_project_relative_then_parts_follow_relative_to() { + let test_cases = [ + RelativePartsTestCase { + description: "a relative path keeps its parts", + path: "models/../a.sql", + resolved: "", + under: None, + expected_parts: Ok(vec!["models", "..", "a.sql"]), + }, + RelativePartsTestCase { + description: "an absolute path inside the project", + path: "/project/link/a.sql", + resolved: "/project/models/a.sql", + under: Some("models"), + expected_parts: Ok(vec!["a.sql"]), + }, + RelativePartsTestCase { + description: "an absolute path outside the project", + path: "/elsewhere/a.sql", + resolved: "/elsewhere/a.sql", + under: None, + expected_parts: Err(RelativePartsError::Escapes), + }, + RelativePartsTestCase { + description: "a path outside the requested folder", + path: "models/a.sql", + resolved: "", + under: Some("rules"), + expected_parts: Err(RelativePartsError::NotBeneath), + }, + RelativePartsTestCase { + description: "a relative path is beneath the current folder", + path: "a.sql", + resolved: "", + under: Some("."), + expected_parts: Ok(vec!["a.sql"]), + }, + ]; + for test_case in test_cases { + let parts = project_relative_parts( + Path::new(test_case.path), + Path::new("/project"), + test_case.under.map(Path::new), + |_| Ok::(PathBuf::from(test_case.resolved)), + ) + .map(|parts| { + parts + .into_iter() + .map(|part| part.to_string_lossy().into_owned()) + .collect::>() + }); + assert_eq!( + parts + .as_ref() + .map(|parts| parts.iter().map(String::as_str).collect::>()) + .map_err(Clone::clone), + test_case.expected_parts, + "{}", + test_case.description + ); + } +} + +#[test] +fn given_file_names_when_checking_tracked_text_then_only_known_suffixes_count() { + let test_cases = [ + TrackedTextTestCase { + description: "a SQL model", + path: "models/a.sql", + expected_tracked: true, + }, + TrackedTextTestCase { + description: "an uppercase suffix", + path: "models/A.YML", + expected_tracked: true, + }, + TrackedTextTestCase { + description: "a hidden name without a suffix", + path: "models/.sql", + expected_tracked: false, + }, + TrackedTextTestCase { + description: "a markdown file", + path: "readme.md", + expected_tracked: false, + }, + ]; + for test_case in test_cases { + assert_eq!( + tracked_text_path(Path::new(test_case.path)), + test_case.expected_tracked, + "{}", + test_case.description + ); + } +} diff --git a/crates/sqlbuild-rules/src/project_tree/tests/test_types.rs b/crates/sqlbuild-rules/src/project_tree/tests/test_types.rs index f37eccfcac..3e05f20299 100644 --- a/crates/sqlbuild-rules/src/project_tree/tests/test_types.rs +++ b/crates/sqlbuild-rules/src/project_tree/tests/test_types.rs @@ -1,4 +1,4 @@ -use crate::project_tree::errors::ProjectTreeFileError; +use crate::project_tree::errors::{ProjectTreeFileError, RelativePartsError}; pub(crate) struct ProjectTreeEntriesTestCase { pub(crate) description: &'static str, @@ -11,3 +11,25 @@ pub(crate) struct ProjectTreeFileTestCase { /// The project-relative file read, or why none may be. pub(crate) expected_file: Result<&'static str, ProjectTreeFileError>, } + +pub(crate) struct PathSelectionTestCase { + pub(crate) description: &'static str, + pub(crate) argument: Option<&'static str>, + pub(crate) expected_children: Vec<&'static str>, + pub(crate) expected_descendants: Vec<&'static str>, + pub(crate) expected_glob: Vec<&'static str>, +} + +pub(crate) struct RelativePartsTestCase { + pub(crate) description: &'static str, + pub(crate) path: &'static str, + pub(crate) resolved: &'static str, + pub(crate) under: Option<&'static str>, + pub(crate) expected_parts: Result, RelativePartsError<()>>, +} + +pub(crate) struct TrackedTextTestCase { + pub(crate) description: &'static str, + pub(crate) path: &'static str, + pub(crate) expected_tracked: bool, +} diff --git a/python-boundary.toml b/python-boundary.toml index 7acec6e0b6..4b1df2ee41 100644 --- a/python-boundary.toml +++ b/python-boundary.toml @@ -130,7 +130,7 @@ note = "Public custom-rule test harness that writes a temporary project and runs [[stays]] path = "rule_engine/_helpers/engine/fact_replay.py" rule = 2 -note = "Rule 2: the fact digest scheme shared with the rule host (tracked_facts.py, custom_host.py); recomputes values through the user-facing fact views and encodes live Python objects with orjson." +note = "Rule 2: the fact digest scheme shared with the rule host (tracked_facts.py, custom_host.py); recomputes values through the user-facing fact views and encodes live Python objects with orjson. The readable-file selection is native (tracked_text_file_values)." [[stays]] path = "rule_engine/main/define.py" @@ -185,7 +185,7 @@ note = "Fact view user rules read through RuleContext; the Python API over nativ [[stays]] path = "rule_engine/classes/project_tree.py" rule = 3 -note = "Project-tree fact view user rules read through RuleContext; the walk, path validation and file resolution are native (project_tree_paths, project_tree_file_path, validate_project_relative) and only the file read stays here." +note = "Project-tree fact view user rules read through RuleContext; the walk, path validation, file resolution and the children, descendants, glob and relative_parts queries are native (project_tree_*), and only the file read and the model lookup stay here." [[stays]] path = "rule_engine/classes/sql_document.py" diff --git a/src/sqlbuild/_native.pyi b/src/sqlbuild/_native.pyi index 7d4a7a7ce3..5bfbae85ff 100644 --- a/src/sqlbuild/_native.pyi +++ b/src/sqlbuild/_native.pyi @@ -7047,6 +7047,19 @@ def rule_option_toml_value(value: object) -> str: ... def validate_project_relative(path: str) -> None: ... def project_tree_paths(root: Path) -> tuple[ProjectPath, ...]: ... def project_tree_file_path(root: Path, path: str) -> str: ... +def project_tree_child_nodes( + nodes: Iterable[ProjectPath], path: str | PurePath +) -> tuple[ProjectPath, ...]: ... +def project_tree_descendant_nodes( + nodes: Iterable[ProjectPath], path: str | PurePath +) -> tuple[ProjectPath, ...]: ... +def project_tree_glob_nodes( + nodes: Iterable[ProjectPath], pattern: str +) -> tuple[ProjectPath, ...]: ... +def tracked_text_file_values(nodes: Iterable[ProjectPath]) -> tuple[str, ...]: ... +def project_tree_relative_parts( + *, root: Path, path: str | PurePath, under: str | PurePath +) -> tuple[str, ...]: ... def startup_input_stamps( *, directories: Iterable[str], files: Iterable[str], prefixes: Iterable[str] ) -> list[tuple[str, int | None, int | None]]: ... diff --git a/src/sqlbuild/rule_engine/_helpers/engine/fact_replay.py b/src/sqlbuild/rule_engine/_helpers/engine/fact_replay.py index d3868e3a87..dc149b703e 100644 --- a/src/sqlbuild/rule_engine/_helpers/engine/fact_replay.py +++ b/src/sqlbuild/rule_engine/_helpers/engine/fact_replay.py @@ -9,7 +9,7 @@ import orjson -from sqlbuild._native import FactDigestError, Model, RuleFactViews +from sqlbuild._native import FactDigestError, Model, RuleFactViews, tracked_text_file_values from sqlbuild.rule_engine.classes.shared_fact_encoder import SharedFactEncoder from sqlbuild.rule_engine.classes.sql_document import SqlDocument from sqlbuild.rule_engine.constants import ( @@ -45,7 +45,6 @@ ) from sqlbuild.rule_engine.types import FactKey -_TRACKED_TEXT_SUFFIXES: frozenset[str] = frozenset({".py", ".sql", ".toml", ".yaml", ".yml"}) _ORJSON_OPTIONS: int = orjson.OPT_SORT_KEYS | orjson.OPT_NON_STR_KEYS _GLOBAL_FACTS: dict[str, Callable[[RuleFactViews], object]] = { @@ -117,9 +116,8 @@ def full_fact_keys(views: RuleFactViews) -> tuple[FactKey, ...]: keys.extend((fact, path) for fact in _MODEL_PATH_FACTS) keys.extend((fact, model.name) for fact in _MODEL_NAME_FACTS) keys.extend( - (FACT_TREE_READ_TEXT, node.value) - for node in views.project.tree.paths() - if node.is_file and Path(node.value).suffix.lower() in _TRACKED_TEXT_SUFFIXES + (FACT_TREE_READ_TEXT, value) + for value in tracked_text_file_values(views.project.tree.paths()) ) return tuple(keys) diff --git a/src/sqlbuild/rule_engine/classes/project_tree.py b/src/sqlbuild/rule_engine/classes/project_tree.py index ba020703ac..b009d556b0 100644 --- a/src/sqlbuild/rule_engine/classes/project_tree.py +++ b/src/sqlbuild/rule_engine/classes/project_tree.py @@ -2,7 +2,6 @@ from __future__ import annotations -import fnmatch from pathlib import Path import sqlbuild._native as _native @@ -10,9 +9,12 @@ Model, ProjectPath, RuleUsageError, + project_tree_child_nodes, + project_tree_descendant_nodes, project_tree_file_path, + project_tree_glob_nodes, project_tree_paths, - validate_project_relative, + project_tree_relative_parts, ) from sqlbuild.compiler.compile.types import CompiledProjectView @@ -46,30 +48,16 @@ def paths(self) -> tuple[ProjectPath, ...]: return self._nodes def children(self, path: str = "") -> tuple[ProjectPath, ...]: - parent: Path = Path(path) - return tuple(node for node in self._nodes if Path(node.value).parent == parent) + return project_tree_child_nodes(self._nodes, path) def descendants(self, path: str = "") -> tuple[ProjectPath, ...]: - prefix: str = f"{Path(path).as_posix().rstrip('/')}/" if path else "" - return tuple(node for node in self._nodes if node.value.startswith(prefix)) + return project_tree_descendant_nodes(self._nodes, path) def glob(self, pattern: str) -> tuple[ProjectPath, ...]: - validate_project_relative(pattern) - return tuple(node for node in self._nodes if fnmatch.fnmatchcase(node.value, pattern)) + return project_tree_glob_nodes(self._nodes, pattern) def relative_parts(self, *, path: Path | str, under: str = "") -> tuple[str, ...]: - relative: Path = Path(path) - if relative.is_absolute(): - try: - relative = relative.resolve().relative_to(self._root) - except ValueError as error: - raise RuleUsageError(f"project path escapes the project: {path}") from error - if under: - try: - relative = relative.relative_to(Path(under)) - except ValueError as error: - raise RuleUsageError(f"project path {path} is not beneath {under}") from error - return relative.parts + return project_tree_relative_parts(root=self._root, path=path, under=under) def resources_under(self, path: str) -> tuple[Model, ...]: prefix: str = f"{Path(path).as_posix().rstrip('/')}/" diff --git a/tests/unit/src/sqlbuild/rule_engine/classes/project_tree/_test_types.py b/tests/unit/src/sqlbuild/rule_engine/classes/project_tree/_test_types.py index 99914f928d..16a26c7e9f 100644 --- a/tests/unit/src/sqlbuild/rule_engine/classes/project_tree/_test_types.py +++ b/tests/unit/src/sqlbuild/rule_engine/classes/project_tree/_test_types.py @@ -7,3 +7,19 @@ class ReadTextTestCase: path: str expected_text: str | None = None expected_error: str = "" + + +@dataclass(frozen=True) +class TreeQueryTestCase: + description: str + argument: str + expected_values: tuple[str, ...] + + +@dataclass(frozen=True) +class RelativePartsTestCase: + description: str + path: str + under: str + expected_parts: tuple[str, ...] | None = None + expected_error: str = "" diff --git a/tests/unit/src/sqlbuild/rule_engine/classes/project_tree/test_queries.py b/tests/unit/src/sqlbuild/rule_engine/classes/project_tree/test_queries.py new file mode 100644 index 0000000000..d841596fc6 --- /dev/null +++ b/tests/unit/src/sqlbuild/rule_engine/classes/project_tree/test_queries.py @@ -0,0 +1,152 @@ +"""Project-tree queries select discovered paths exactly as pathlib and fnmatch would.""" + +from pathlib import Path + +import pytest + +from sqlbuild._native import RuleUsageError +from sqlbuild.rule_engine.classes.project_tree import ProjectTree +from tests.unit.src.sqlbuild.rule_engine.classes.project_tree._test_types import ( + RelativePartsTestCase, + TreeQueryTestCase, +) +from tests.unit.src.sqlbuild.rule_engine.classes.project_tree.helpers import linked_project_tree + + +@pytest.mark.parametrize( + "test_case", + ( + TreeQueryTestCase("project root", "", ("models", "rules")), + TreeQueryTestCase("current directory", ".", ("models", "rules")), + TreeQueryTestCase( + "folder with trailing slash", "models/", ("models/notes.txt", "models/orders.sql") + ), + TreeQueryTestCase("nested folder", "./rules//inputs", ("rules/inputs/limits.yaml",)), + TreeQueryTestCase("missing folder", "missing", ()), + ), + ids=lambda case: case.description, +) +def test_given_directory_when_listing_children_then_returns_direct_entries( + tmp_path: Path, test_case: TreeQueryTestCase +) -> None: + tree: ProjectTree = linked_project_tree(tmp_path) + + values: tuple[str, ...] = tuple(node.value for node in tree.children(test_case.argument)) + + assert values == test_case.expected_values + + +@pytest.mark.parametrize( + "test_case", + ( + TreeQueryTestCase( + "everything", + "", + ( + "models", + "models/notes.txt", + "models/orders.sql", + "rules", + "rules/inputs", + "rules/inputs/limits.yaml", + ), + ), + TreeQueryTestCase("folder", "rules/", ("rules/inputs", "rules/inputs/limits.yaml")), + TreeQueryTestCase("current directory spelled out", ".", ()), + ), + ids=lambda case: case.description, +) +def test_given_directory_when_listing_descendants_then_returns_entries_below_it( + tmp_path: Path, test_case: TreeQueryTestCase +) -> None: + tree: ProjectTree = linked_project_tree(tmp_path) + + values: tuple[str, ...] = tuple(node.value for node in tree.descendants(test_case.argument)) + + assert values == test_case.expected_values + + +@pytest.mark.parametrize( + "test_case", + ( + TreeQueryTestCase("star crosses folders", "*.sql", ("models/orders.sql",)), + TreeQueryTestCase("single character", "rule?", ("rules",)), + TreeQueryTestCase("negated set", "[!m]*s", ("rules", "rules/inputs")), + TreeQueryTestCase("no match", "*.py", ()), + ), + ids=lambda case: case.description, +) +def test_given_pattern_when_globbing_then_returns_fnmatch_matches( + tmp_path: Path, test_case: TreeQueryTestCase +) -> None: + tree: ProjectTree = linked_project_tree(tmp_path) + + values: tuple[str, ...] = tuple(node.value for node in tree.glob(test_case.argument)) + + assert values == test_case.expected_values + + +@pytest.mark.parametrize( + "test_case", + ( + RelativePartsTestCase("relative path", "models/orders.sql", "", ("models", "orders.sql")), + RelativePartsTestCase("beneath a folder", "models/orders.sql", "models/", ("orders.sql",)), + ), + ids=lambda case: case.description, +) +def test_given_path_when_reading_relative_parts_then_returns_project_relative_parts( + tmp_path: Path, test_case: RelativePartsTestCase +) -> None: + tree: ProjectTree = linked_project_tree(tmp_path) + + parts: tuple[str, ...] = tree.relative_parts(path=test_case.path, under=test_case.under) + + assert parts == test_case.expected_parts + + +@pytest.mark.parametrize( + "test_case", + ( + RelativePartsTestCase( + "through a linked folder", + "orders/linked_models/orders.sql", + "", + ("models", "orders.sql"), + ), + RelativePartsTestCase( + "beneath a folder", "orders/models/orders.sql", "models", ("orders.sql",) + ), + ), + ids=lambda case: case.description, +) +def test_given_absolute_path_inside_project_when_reading_relative_parts_then_strips_the_root( + tmp_path: Path, test_case: RelativePartsTestCase +) -> None: + tree: ProjectTree = linked_project_tree(tmp_path) + + parts: tuple[str, ...] = tree.relative_parts( + path=tmp_path / test_case.path, under=test_case.under + ) + + assert parts == test_case.expected_parts + + +@pytest.mark.parametrize( + "test_case", + ( + RelativePartsTestCase("not beneath", "models/orders.sql", "rules", None, "is not beneath"), + RelativePartsTestCase("absolute outside", "/", "", None, "escapes the project"), + ), + ids=lambda case: case.description, +) +def test_given_path_outside_scope_when_reading_relative_parts_then_raises_usage_error( + tmp_path: Path, test_case: RelativePartsTestCase +) -> None: + tree: ProjectTree = linked_project_tree(tmp_path) + + with pytest.raises(RuleUsageError, match=test_case.expected_error): + tree.relative_parts(path=test_case.path, under=test_case.under) + + +if __name__ == "__main__": + pytest.main([__file__, "-vv"]) From 3d1ccd6208e4212c4bcb77de2681865f43503693 Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Fri, 16 Oct 2026 01:47:30 +0100 Subject: [PATCH 45/48] test: split hook read cases into success and rejection tests --- .../_helpers/authored_values/python_nodes.rs | 3 +- .../python_nodes/decorators/_test_types.py | 4 +-- .../decorators/test_python_decorators.py | 36 ++++++++++++------- 3 files changed, 26 insertions(+), 17 deletions(-) diff --git a/crates/sqlbuild-python/src/bindings/_helpers/authored_values/python_nodes.rs b/crates/sqlbuild-python/src/bindings/_helpers/authored_values/python_nodes.rs index dc6fb69b7e..9a09f3ab30 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/authored_values/python_nodes.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/authored_values/python_nodes.rs @@ -173,7 +173,7 @@ fn validate_audit_case( Ok(()) } -/// `@hook(reads=...)` as distinct resource references in first-seen order. +/// `@hook(reads=...)` references in first-seen order; a leading invalid `None` is tolerated. #[pyfunction] fn normalize_hook_reads<'py>(value: &Bound<'py, PyAny>) -> PyResult> { let py: Python<'py> = value.py(); @@ -183,7 +183,6 @@ fn normalize_hook_reads<'py>(value: &Bound<'py, PyAny>) -> PyResult>()? }; - // Only the first non-reference item is checked, and a `None` there is tolerated, as before. for item in &reads { if !item.is_instance(&reference)? { if item.is_none() { diff --git a/tests/unit/src/sqlbuild/python_nodes/decorators/_test_types.py b/tests/unit/src/sqlbuild/python_nodes/decorators/_test_types.py index 346a42c7e6..7eb3747406 100644 --- a/tests/unit/src/sqlbuild/python_nodes/decorators/_test_types.py +++ b/tests/unit/src/sqlbuild/python_nodes/decorators/_test_types.py @@ -60,5 +60,5 @@ class FactoryDecoratorErrorTestCase: class HookReadsTestCase: description: str reads: object - expected_reads: tuple[str, ...] | None - expected_error: str | None = None + expected_reads: tuple[str, ...] = () + expected_error: str = "" diff --git a/tests/unit/src/sqlbuild/python_nodes/decorators/test_python_decorators.py b/tests/unit/src/sqlbuild/python_nodes/decorators/test_python_decorators.py index f155354d1d..2b40ee7ef0 100644 --- a/tests/unit/src/sqlbuild/python_nodes/decorators/test_python_decorators.py +++ b/tests/unit/src/sqlbuild/python_nodes/decorators/test_python_decorators.py @@ -10,7 +10,7 @@ from sqlbuild.checks import check, get_check_definition from sqlbuild.errors.contracts.exceptions import SharedInputError from sqlbuild.factories import factory, get_factory_definition -from sqlbuild.hooks import get_hook_definition, hook +from sqlbuild.hooks import HookDefinition, get_hook_definition, hook from sqlbuild.python_nodes.models import ( AssetDefinition, CheckDefinition, @@ -273,28 +273,38 @@ def test_given_factory_decorator_when_called_without_function_then_raises( reads=[None, 5, model("orders")], expected_reads=("orders",), ), + ], + ids=lambda case: case.description, +) +def test_given_hook_reads_when_decorating_then_keeps_distinct_references( + test_case: HookReadsTestCase, +) -> None: + def refresh_orders() -> None: + return None + + decorated: Any = cast(Any, hook(reads=cast(Any, test_case.reads)))(refresh_orders) + definition: HookDefinition | None = get_hook_definition(decorated) + + assert definition is not None + assert tuple(read.name for read in definition.reads) == test_case.expected_reads + + +@pytest.mark.parametrize( + "test_case", + [ HookReadsTestCase( description="a non-reference item is rejected", reads=[model("orders"), "customers"], - expected_reads=None, expected_error="got str", ), ], ids=lambda case: case.description, ) -def test_given_hook_reads_when_decorating_then_normalizes_like_the_documented_contract( +def test_given_non_reference_hook_read_when_decorating_then_raises( test_case: HookReadsTestCase, ) -> None: def refresh_orders() -> None: return None - if test_case.expected_error is not None: - with pytest.raises(SharedInputError, match=test_case.expected_error): - cast(Any, hook(reads=cast(Any, test_case.reads)))(refresh_orders) - return - - decorated: Any = cast(Any, hook(reads=cast(Any, test_case.reads)))(refresh_orders) - definition = get_hook_definition(decorated) - - assert definition is not None - assert tuple(read.name for read in definition.reads) == test_case.expected_reads + with pytest.raises(SharedInputError, match=test_case.expected_error): + cast(Any, hook(reads=cast(Any, test_case.reads)))(refresh_orders) From 1654c77fe7baa9bb10f9ea64dfd1b23fa000eceb Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Fri, 16 Oct 2026 01:58:17 +0100 Subject: [PATCH 46/48] fix: stamp startup inputs with platform path rules on Windows --- .../src/artifacts/_helpers/files.rs | 5 +- .../src/startup_inputs/_helpers/pure_paths.rs | 86 ++++++++++++++++--- .../src/startup_inputs/constants.rs | 4 +- .../src/startup_inputs/main/startup_stamps.rs | 32 ++++--- 4 files changed, 100 insertions(+), 27 deletions(-) diff --git a/crates/sqlbuild-cache/src/artifacts/_helpers/files.rs b/crates/sqlbuild-cache/src/artifacts/_helpers/files.rs index dcb580caa4..b1b26f57a2 100644 --- a/crates/sqlbuild-cache/src/artifacts/_helpers/files.rs +++ b/crates/sqlbuild-cache/src/artifacts/_helpers/files.rs @@ -5,7 +5,6 @@ use std::fs::{File, OpenOptions}; use std::io::{ErrorKind, Read, Write}; use std::path::{Path, PathBuf}; -use crate::artifacts::constants::NEW_FILE_MODE; use crate::artifacts::errors::ArtifactError; use crate::artifacts::models::{Publication, WrittenArtifacts}; @@ -261,7 +260,9 @@ fn copy_new_file(source: &Path, path: &Path) -> std::io::Result<()> { #[cfg(unix)] { use std::os::unix::fs::PermissionsExt; - let _ = builder.permissions(std::fs::Permissions::from_mode(NEW_FILE_MODE)); + let _ = builder.permissions(std::fs::Permissions::from_mode( + crate::artifacts::constants::NEW_FILE_MODE, + )); } let mut temporary = builder.tempfile_in(directory)?; temporary.write_all(&std::fs::read(source)?)?; diff --git a/crates/sqlbuild-cache/src/startup_inputs/_helpers/pure_paths.rs b/crates/sqlbuild-cache/src/startup_inputs/_helpers/pure_paths.rs index 2cc65747f7..9f219cbdab 100644 --- a/crates/sqlbuild-cache/src/startup_inputs/_helpers/pure_paths.rs +++ b/crates/sqlbuild-cache/src/startup_inputs/_helpers/pure_paths.rs @@ -1,12 +1,15 @@ -//! `pathlib.PurePosixPath` spelling and ordering over raw path bytes. +//! `pathlib` spelling, ordering and stat reads for startup stamps on each platform. -use std::ffi::OsString; -use std::os::unix::ffi::{OsStrExt, OsStringExt}; +use std::ffi::OsStr; +use std::fs::Metadata; use std::path::{Path, PathBuf}; +use crate::startup_inputs::constants::PATH_FILE_SUFFIX; +#[cfg(unix)] use crate::startup_inputs::constants::{CURRENT, SEPARATOR}; /// The root a POSIX path keeps: two leading slashes stay two, more collapse to one. +#[cfg(unix)] fn root(bytes: &[u8]) -> &[u8] { match bytes { [SEPARATOR, SEPARATOR, rest @ ..] if rest.first() != Some(&SEPARATOR) => &bytes[..2], @@ -16,11 +19,14 @@ fn root(bytes: &[u8]) -> &[u8] { } /// `str(PurePosixPath(path, *children))`. -pub(crate) fn pure(path: &Path, children: &[&[u8]]) -> PathBuf { +#[cfg(unix)] +pub(crate) fn pure(path: &Path, children: &[&OsStr]) -> PathBuf { + use std::os::unix::ffi::{OsStrExt, OsStringExt}; + let bytes: &[u8] = path.as_os_str().as_bytes(); let mut text: Vec = root(bytes).to_vec(); let parts = std::iter::once(bytes) - .chain(children.iter().copied()) + .chain(children.iter().map(|child| child.as_bytes())) .flat_map(|segment| segment.split(|byte| *byte == SEPARATOR)) .filter(|part| !part.is_empty() && *part != CURRENT); let mut first: bool = true; @@ -34,22 +40,82 @@ pub(crate) fn pure(path: &Path, children: &[&[u8]]) -> PathBuf { if text.is_empty() { text.extend_from_slice(CURRENT); } - PathBuf::from(OsString::from_vec(text)) + PathBuf::from(std::ffi::OsString::from_vec(text)) +} + +/// `str(PureWindowsPath(path, *children))` through the platform path rules. +#[cfg(not(unix))] +pub(crate) fn pure(path: &Path, children: &[&OsStr]) -> PathBuf { + let mut joined: PathBuf = path.components().collect(); + for child in children { + joined.push(child); + } + joined } /// `str(Path(path).absolute())` against the working directory `cwd`. pub(crate) fn absolute(path: &Path, cwd: &Path) -> PathBuf { - if root(path.as_os_str().as_bytes()).is_empty() { - pure(cwd, &[path.as_os_str().as_bytes()]) - } else { + if path.has_root() && (cfg!(unix) || path.is_absolute()) { pure(path, &[]) + } else { + pure(cwd, &[path.as_os_str()]) } } /// The `/`-separated parts `PurePosixPath` orders paths by. -pub(crate) fn order_key(path: &Path) -> Vec<&[u8]> { +#[cfg(unix)] +pub(crate) fn order_key(path: &Path) -> Vec> { + use std::os::unix::ffi::OsStrExt; + path.as_os_str() .as_bytes() .split(|byte| *byte == SEPARATOR) + .map(<[u8]>::to_vec) + .collect() +} + +/// The case-folded parts `PureWindowsPath` orders paths by. +#[cfg(not(unix))] +pub(crate) fn order_key(path: &Path) -> Vec> { + path.components() + .map(|part| { + part.as_os_str() + .to_string_lossy() + .to_lowercase() + .into_bytes() + }) .collect() } + +/// Whether `glob("*.pth")` matches a directory entry name on this platform. +pub(crate) fn is_path_file(name: &OsStr) -> bool { + if cfg!(unix) { + name.as_encoded_bytes().ends_with(PATH_FILE_SUFFIX) + } else { + name.to_string_lossy() + .to_lowercase() + .as_bytes() + .ends_with(PATH_FILE_SUFFIX) + } +} + +/// `os.stat(path).st_mtime_ns`. +#[cfg(unix)] +pub(crate) fn modified_ns(metadata: &Metadata) -> std::io::Result { + use std::os::unix::fs::MetadataExt; + + Ok(metadata.mtime() * 1_000_000_000 + metadata.mtime_nsec()) +} + +/// `os.stat(path).st_mtime_ns` from the platform modification time, negative before 1970. +#[cfg(not(unix))] +pub(crate) fn modified_ns(metadata: &Metadata) -> std::io::Result { + let modified: std::time::SystemTime = metadata.modified()?; + let (nanos, sign): (u128, i64) = match modified.duration_since(std::time::UNIX_EPOCH) { + Ok(elapsed) => (elapsed.as_nanos(), 1), + Err(before) => (before.duration().as_nanos(), -1), + }; + i64::try_from(nanos) + .map(|value| sign * value) + .map_err(std::io::Error::other) +} diff --git a/crates/sqlbuild-cache/src/startup_inputs/constants.rs b/crates/sqlbuild-cache/src/startup_inputs/constants.rs index 20ba4b2d99..f9f35121d3 100644 --- a/crates/sqlbuild-cache/src/startup_inputs/constants.rs +++ b/crates/sqlbuild-cache/src/startup_inputs/constants.rs @@ -1,8 +1,10 @@ //! Names the startup stamps read. /// The virtual-environment configuration file of an interpreter prefix. -pub(crate) const VENV_CONFIG: &[u8] = b"pyvenv.cfg"; +pub(crate) const VENV_CONFIG: &str = "pyvenv.cfg"; /// The suffix of the path configuration files `site` processes. pub(crate) const PATH_FILE_SUFFIX: &[u8] = b".pth"; +#[cfg(unix)] pub(crate) const SEPARATOR: u8 = b'/'; +#[cfg(unix)] pub(crate) const CURRENT: &[u8] = b"."; diff --git a/crates/sqlbuild-cache/src/startup_inputs/main/startup_stamps.rs b/crates/sqlbuild-cache/src/startup_inputs/main/startup_stamps.rs index e7384af4f6..8f502c2077 100644 --- a/crates/sqlbuild-cache/src/startup_inputs/main/startup_stamps.rs +++ b/crates/sqlbuild-cache/src/startup_inputs/main/startup_stamps.rs @@ -1,14 +1,14 @@ //! Stamp every startup directory, its `.pth` files, the interpreter and its venv configuration. use std::collections::BTreeSet; -use std::ffi::OsString; +use std::ffi::{OsStr, OsString}; use std::io; -use std::os::unix::ffi::OsStrExt; -use std::os::unix::fs::MetadataExt; use std::path::{Path, PathBuf}; -use crate::startup_inputs::_helpers::pure_paths::{absolute, order_key, pure}; -use crate::startup_inputs::constants::{PATH_FILE_SUFFIX, VENV_CONFIG}; +use crate::startup_inputs::_helpers::pure_paths::{ + absolute, is_path_file, modified_ns, order_key, pure, +}; +use crate::startup_inputs::constants::VENV_CONFIG; use crate::startup_inputs::models::{StartupLocations, StartupStamp}; /// Sorted stamps of the startup paths; missing paths are unstamped and other stat failures fail. @@ -32,7 +32,7 @@ pub fn startup_stamps( locations .prefixes .iter() - .map(|prefix| pure(prefix, &[VENV_CONFIG]).into_os_string()), + .map(|prefix| pure(prefix, &[OsStr::new(VENV_CONFIG)]).into_os_string()), ); for directory in &directories { let directory: &Path = Path::new(directory); @@ -40,21 +40,25 @@ pub fn startup_stamps( continue; }; for entry in listing.flatten() { - if entry.file_name().as_bytes().ends_with(PATH_FILE_SUFFIX) { - paths.insert(pure(directory, &[entry.file_name().as_bytes()]).into_os_string()); + let name: OsString = entry.file_name(); + if is_path_file(&name) { + paths.insert(pure(directory, &[name.as_os_str()]).into_os_string()); } } } let mut ordered: Vec = paths.into_iter().map(PathBuf::from).collect(); - ordered.sort_by(|left, right| order_key(left).cmp(&order_key(right))); + ordered.sort_by_key(|path| order_key(path)); ordered .into_iter() .map(|path| match std::fs::metadata(&path) { - Ok(metadata) => Ok(StartupStamp { - modified_ns: Some(metadata.mtime() * 1_000_000_000 + metadata.mtime_nsec()), - size: Some(metadata.len()), - path, - }), + Ok(metadata) => match modified_ns(&metadata) { + Ok(modified) => Ok(StartupStamp { + modified_ns: Some(modified), + size: Some(metadata.len()), + path, + }), + Err(error) => Err((path, error)), + }, Err(error) if error.kind() == io::ErrorKind::NotFound => Ok(StartupStamp { path, modified_ns: None, From 88453048fae43c2d414fc276a2ddeb44fd4e9aa8 Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Fri, 16 Oct 2026 02:18:49 +0100 Subject: [PATCH 47/48] fix: resolve project-tree paths with platform path rules on Windows --- .../sqlbuild-cache/src/artifacts/constants.rs | 1 + .../sqlbuild-core/src/posix_path/constants.rs | 1 - .../sqlbuild-core/src/posix_path/main/mod.rs | 1 - .../src/posix_path/main/posix_escapes_root.rs | 12 - .../src/posix_path/tests/test_posix_path.rs | 67 +----- .../src/posix_path/tests/test_types.rs | 6 - .../src/bindings/_helpers/rules/rule_api.rs | 49 +++- .../src/project_tree/_helpers/entries.rs | 25 ++- .../src/project_tree/_helpers/files.rs | 96 +++++++- .../project_tree/_helpers/relative_paths.rs | 209 +++++++++++++----- .../src/project_tree/constants.rs | 29 ++- .../sqlbuild-rules/src/project_tree/errors.rs | 2 +- .../src/project_tree/main/mod.rs | 1 + .../project_tree/main/project_path_escapes.rs | 11 + .../main/project_relative_parts.rs | 24 +- .../main/project_tree_children.rs | 9 +- .../main/project_tree_descendants.rs | 13 +- .../project_tree/main/project_tree_entries.rs | 4 +- .../project_tree/main/project_tree_file.rs | 16 +- .../project_tree/main/project_tree_glob.rs | 5 +- .../sqlbuild-rules/src/project_tree/models.rs | 8 + .../sqlbuild-rules/src/project_tree/tests.rs | 8 + .../src/project_tree/tests/test_escapes.rs | 68 ++++++ .../project_tree/tests/test_path_queries.rs | 72 +----- .../project_tree/tests/test_relative_parts.rs | 69 ++++++ .../src/project_tree/tests/test_types.rs | 11 + 26 files changed, 525 insertions(+), 292 deletions(-) delete mode 100644 crates/sqlbuild-core/src/posix_path/main/posix_escapes_root.rs create mode 100644 crates/sqlbuild-rules/src/project_tree/main/project_path_escapes.rs create mode 100644 crates/sqlbuild-rules/src/project_tree/tests/test_escapes.rs create mode 100644 crates/sqlbuild-rules/src/project_tree/tests/test_relative_parts.rs diff --git a/crates/sqlbuild-cache/src/artifacts/constants.rs b/crates/sqlbuild-cache/src/artifacts/constants.rs index 6625926d4d..3feacccd00 100644 --- a/crates/sqlbuild-cache/src/artifacts/constants.rs +++ b/crates/sqlbuild-cache/src/artifacts/constants.rs @@ -2,4 +2,5 @@ pub const NEW_FILE_MODE: u32 = 0o666; /// The extension every compiled SQL test artifact carries. pub(crate) const SQL_EXTENSION: &str = "sql"; +#[cfg(unix)] pub(crate) const NANOS_PER_SECOND: i64 = 1_000_000_000; diff --git a/crates/sqlbuild-core/src/posix_path/constants.rs b/crates/sqlbuild-core/src/posix_path/constants.rs index e58ea77ccb..9216305dc5 100644 --- a/crates/sqlbuild-core/src/posix_path/constants.rs +++ b/crates/sqlbuild-core/src/posix_path/constants.rs @@ -5,5 +5,4 @@ pub(crate) const ROOT: &str = "/"; /// POSIX keeps exactly two leading slashes as their own root. pub(crate) const DOUBLE_ROOT: &str = "//"; pub(crate) const CURRENT: &str = "."; -pub(crate) const PARENT: &str = ".."; pub(crate) const SUFFIX_MARK: char = '.'; diff --git a/crates/sqlbuild-core/src/posix_path/main/mod.rs b/crates/sqlbuild-core/src/posix_path/main/mod.rs index cbd3f33313..e4ab185549 100644 --- a/crates/sqlbuild-core/src/posix_path/main/mod.rs +++ b/crates/sqlbuild-core/src/posix_path/main/mod.rs @@ -1,4 +1,3 @@ -pub mod posix_escapes_root; pub mod posix_join; pub mod posix_name; pub mod posix_parent; diff --git a/crates/sqlbuild-core/src/posix_path/main/posix_escapes_root.rs b/crates/sqlbuild-core/src/posix_path/main/posix_escapes_root.rs deleted file mode 100644 index dec3d18b8b..0000000000 --- a/crates/sqlbuild-core/src/posix_path/main/posix_escapes_root.rs +++ /dev/null @@ -1,12 +0,0 @@ -//! Whether a `PurePosixPath` leaves the directory it is relative to. - -use crate::posix_path::constants::{DOUBLE_ROOT, PARENT, ROOT}; -use crate::posix_path::main::posix_parts::posix_parts; - -/// `PurePosixPath(path).is_absolute() or ".." in PurePosixPath(path).parts`. -#[must_use] -pub fn posix_escapes_root(path: &str) -> bool { - posix_parts(path) - .iter() - .any(|part| *part == ROOT || *part == DOUBLE_ROOT || *part == PARENT) -} diff --git a/crates/sqlbuild-core/src/posix_path/tests/test_posix_path.rs b/crates/sqlbuild-core/src/posix_path/tests/test_posix_path.rs index 2390da7919..4722f0737b 100644 --- a/crates/sqlbuild-core/src/posix_path/tests/test_posix_path.rs +++ b/crates/sqlbuild-core/src/posix_path/tests/test_posix_path.rs @@ -1,10 +1,9 @@ -use crate::posix_path::main::posix_escapes_root::posix_escapes_root; use crate::posix_path::main::posix_join::posix_join; use crate::posix_path::main::posix_name::posix_name; use crate::posix_path::main::posix_parent::posix_parent; use crate::posix_path::main::posix_parts::posix_parts; use crate::posix_path::main::posix_suffix::posix_suffix; -use crate::posix_path::tests::test_types::{EscapesRootTestCase, PosixPathTestCase}; +use crate::posix_path::tests::test_types::PosixPathTestCase; #[test] fn given_paths_when_reading_pure_posix_parts_then_they_match_pathlib() { @@ -141,67 +140,3 @@ fn given_paths_when_reading_pure_posix_parts_then_they_match_pathlib() { ); } } - -#[test] -fn given_paths_when_checking_for_escapes_then_absolute_and_parent_paths_escape() { - let test_cases = [ - EscapesRootTestCase { - description: "a relative file", - path: "models/a.sql", - expected_escapes: false, - }, - EscapesRootTestCase { - description: "current and empty parts", - path: "./a//b/", - expected_escapes: false, - }, - EscapesRootTestCase { - description: "an empty path", - path: "", - expected_escapes: false, - }, - EscapesRootTestCase { - description: "the current directory", - path: ".", - expected_escapes: false, - }, - EscapesRootTestCase { - description: "an absolute path", - path: "/abs/x.yml", - expected_escapes: true, - }, - EscapesRootTestCase { - description: "a double-slash root", - path: "//x", - expected_escapes: true, - }, - EscapesRootTestCase { - description: "a leading parent", - path: "../x.sql", - expected_escapes: true, - }, - EscapesRootTestCase { - description: "an inner parent", - path: "a/../x.sql", - expected_escapes: true, - }, - EscapesRootTestCase { - description: "only a parent", - path: "..", - expected_escapes: true, - }, - EscapesRootTestCase { - description: "dots inside names", - path: "a..b/..c", - expected_escapes: false, - }, - ]; - for test_case in test_cases { - assert_eq!( - posix_escapes_root(test_case.path), - test_case.expected_escapes, - "{}", - test_case.description - ); - } -} diff --git a/crates/sqlbuild-core/src/posix_path/tests/test_types.rs b/crates/sqlbuild-core/src/posix_path/tests/test_types.rs index da99d81c93..126c6e31d9 100644 --- a/crates/sqlbuild-core/src/posix_path/tests/test_types.rs +++ b/crates/sqlbuild-core/src/posix_path/tests/test_types.rs @@ -7,9 +7,3 @@ pub(super) struct PosixPathTestCase { /// `suffix` under Python 3.12 and under Python 3.14. pub(super) expected_suffix: &'static str, } - -pub(super) struct EscapesRootTestCase { - pub(super) description: &'static str, - pub(super) path: &'static str, - pub(super) expected_escapes: bool, -} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/rules/rule_api.rs b/crates/sqlbuild-python/src/bindings/_helpers/rules/rule_api.rs index 5de87ba555..415d36751c 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/rules/rule_api.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/rules/rule_api.rs @@ -1,6 +1,6 @@ //! Custom-rule API decisions: finding positions, harness option TOML and project-tree paths. -use std::path::PathBuf; +use std::path::{Path, PathBuf}; use pyo3::exceptions::{PyAttributeError, PyOSError, PyValueError}; use pyo3::prelude::{ @@ -12,8 +12,8 @@ use pyo3::types::{ use pyo3::{IntoPyObject, pyfunction, wrap_pyfunction}; use sqlbuild_config::toml::main::toml_literal::toml_literal; use sqlbuild_config::toml::models::TomlLiteral; -use sqlbuild_core::posix_path::main::posix_escapes_root::posix_escapes_root; use sqlbuild_rules::project_tree::errors::ProjectTreeFileError; +use sqlbuild_rules::project_tree::main::project_path_escapes::project_path_escapes; use sqlbuild_rules::project_tree::main::project_tree_entries::project_tree_entries; use sqlbuild_rules::project_tree::main::project_tree_file::project_tree_file; use sqlbuild_rules::project_tree::main::tracked_text_suffixes::tracked_text_suffixes; @@ -26,6 +26,7 @@ const RULE_USAGE_ERROR: &str = "RuleUsageError"; const PROJECT_PATH_CLASS: &str = "ProjectPath"; const BUILTINS_MODULE: &str = "builtins"; const DEFAULT_POSITION: i64 = 1; +#[cfg(unix)] const OS_MODULE: &str = "os"; /// The error `os.lstat` raises for a path holding a NUL byte. const EMBEDDED_NUL_MESSAGE: &str = "lstat: embedded null character in path"; @@ -87,7 +88,7 @@ fn finding_position<'py>( node: &Bound<'py, PyAny>, ) -> PyResult<(Bound<'py, PyAny>, Bound<'py, PyAny>)> { let py: Python<'py> = path.py(); - if posix_escapes_root(&path.str()?.to_string_lossy()) { + if escapes(path)? { return Err(usage_error_for( "finding path must be project-relative: ", path, @@ -141,12 +142,46 @@ fn rule_option_toml_value(value: &Bound<'_, PyAny>) -> PyResult { /// Reject a project path that is absolute or climbs out of the project. #[pyfunction] pub(crate) fn validate_project_relative(path: &Bound<'_, PyAny>) -> Result<(), PyErr> { - if posix_escapes_root(&path.str()?.to_string_lossy()) { + if escapes(path)? { return Err(usage_error_for("project path escapes the project: ", path)); } Ok(()) } +/// Whether `str(path)` is absolute or climbs out of the project with `..`. +fn escapes(path: &Bound<'_, PyAny>) -> PyResult { + Ok(project_path_escapes(Path::new( + &*path.str()?.to_string_lossy(), + ))) +} + +/// The `OSError` `Path.is_file()` raises for a stat failure it does not ignore. +#[cfg(unix)] +fn unreadable(py: Python<'_>, path: PathBuf, errno: i32) -> PyResult { + let message: String = imported(py, OS_MODULE, "strerror")? + .call1((errno,))? + .extract()?; + let Ok(path) = path.into_os_string().into_pyobject(py); + Ok(PyOSError::new_err((errno, message, path.unbind()))) +} + +/// The `OSError` `Path.is_file()` raises for a Windows stat failure it does not ignore. +#[cfg(not(unix))] +fn unreadable(py: Python<'_>, path: PathBuf, code: i32) -> PyResult { + let detail: String = std::io::Error::from_raw_os_error(code).to_string(); + let detail: &str = detail + .strip_suffix(&format!(" (os error {code})")) + .unwrap_or(&detail) + .trim_end_matches(|character: char| character <= ' ' || character == '.'); + let Ok(path) = path.into_os_string().into_pyobject(py); + Ok(PyOSError::new_err(( + code, + detail.to_owned(), + path.unbind(), + code, + ))) +} + /// Every supported path under the resolved project `root`, as sorted `ProjectPath`s. #[pyfunction] fn project_tree_paths<'py>(root: &Bound<'py, PyAny>) -> PyResult> { @@ -206,11 +241,7 @@ fn project_tree_file_path<'py>( ), Err(ProjectTreeFileError::Symlink) => "project file must not be a symlink: ".to_owned(), Err(ProjectTreeFileError::Unreadable { path, errno }) => { - let message: String = imported(py, OS_MODULE, "strerror")? - .call1((errno,))? - .extract()?; - let Ok(path) = path.into_os_string().into_pyobject(py); - return Err(PyOSError::new_err((errno, message, path.unbind()))); + return Err(unreadable(py, path, errno)?); } }; Err(usage_error_for(&prefix, path)) diff --git a/crates/sqlbuild-rules/src/project_tree/_helpers/entries.rs b/crates/sqlbuild-rules/src/project_tree/_helpers/entries.rs index c66261b5d2..0419da5111 100644 --- a/crates/sqlbuild-rules/src/project_tree/_helpers/entries.rs +++ b/crates/sqlbuild-rules/src/project_tree/_helpers/entries.rs @@ -1,9 +1,10 @@ -//! How the project walk classifies and filters directory entries. +//! How the project walk classifies, filters and names directory entries. +use std::ffi::{OsStr, OsString}; use std::fs::DirEntry; -use std::os::unix::ffi::OsStrExt; +use std::path::{Path, PathBuf}; -use crate::project_tree::constants::{EXCLUDED_ROOTS, HIDDEN_PREFIX}; +use crate::project_tree::constants::{EXCLUDED_ROOTS, HIDDEN_PREFIX, POSIX_SEPARATOR}; use crate::project_tree::models::EntryKind; /// How the walk treats one directory entry, without following links. @@ -17,12 +18,24 @@ pub(crate) fn kind(entry: &DirEntry) -> EntryKind { /// Whether the walk lists and enters a directory entry. pub(crate) fn listed_directory(entry: &DirEntry, at_root: bool) -> bool { - let name = entry.file_name(); - if name.as_bytes().first() == Some(&HIDDEN_PREFIX) { + let name: OsString = entry.file_name(); + let name: &[u8] = name.as_encoded_bytes(); + if name.first() == Some(&HIDDEN_PREFIX) { return false; } !(at_root && EXCLUDED_ROOTS .iter() - .any(|excluded| name.as_bytes() == excluded.as_bytes())) + .any(|excluded| name == excluded.as_bytes())) +} + +/// The project-relative `as_posix()` spelling of `name` inside `relative`. +pub(crate) fn posix_child(relative: &Path, name: &OsStr) -> PathBuf { + if relative.as_os_str().is_empty() { + return PathBuf::from(name); + } + let mut child: OsString = relative.as_os_str().to_os_string(); + child.push(POSIX_SEPARATOR); + child.push(name); + PathBuf::from(child) } diff --git a/crates/sqlbuild-rules/src/project_tree/_helpers/files.rs b/crates/sqlbuild-rules/src/project_tree/_helpers/files.rs index b036a3a5c0..a37fa5f9d5 100644 --- a/crates/sqlbuild-rules/src/project_tree/_helpers/files.rs +++ b/crates/sqlbuild-rules/src/project_tree/_helpers/files.rs @@ -1,26 +1,94 @@ //! Resolution of one project-relative path to a real, readable project file. -use std::ffi::OsStr; -use std::os::unix::ffi::OsStrExt; use std::path::{Path, PathBuf}; -use crate::project_tree::constants::MISSING_FILE_ERRNOS; +use crate::project_tree::_helpers::relative_paths::{pure_parts, pure_relative_to}; +use crate::project_tree::constants::MISSING_FILE_ERRORS; use crate::project_tree::errors::ProjectTreeFileError; use crate::project_tree::main::tracked_text_path::tracked_text_path; +use crate::project_tree::models::PureParts; -/// `root` joined with the raw path components. -pub(crate) fn joined(root: &Path, parts: &[&[u8]]) -> PathBuf { +/// `root / path` for a relative POSIX path: the root joined with each component. +#[cfg(unix)] +pub(crate) fn joined(root: &Path, parts: &PureParts) -> PathBuf { let mut path: PathBuf = root.to_path_buf(); - path.extend(parts.iter().map(|part| OsStr::from_bytes(part))); + path.extend(&parts.tail); path } +/// `root / path` as `ntpath.join` builds it, keeping or replacing the root's drive. +#[cfg(not(unix))] +pub(crate) fn joined(root: &Path, parts: &PureParts) -> PathBuf { + use std::ffi::OsString; + use std::path::MAIN_SEPARATOR_STR; + + use crate::project_tree::_helpers::relative_paths::same_part; + + let anchor: &str = &parts.anchor.to_string_lossy(); + let drive: &str = anchor.trim_end_matches(MAIN_SEPARATOR_STR); + let root_anchor: OsString = pure_parts(root).anchor; + let root_anchor: &str = &root_anchor.to_string_lossy(); + let root_drive: &str = root_anchor.trim_end_matches(MAIN_SEPARATOR_STR); + let mut path: PathBuf = if drive.len() < anchor.len() { + let drive: &str = if drive.is_empty() { root_drive } else { drive }; + PathBuf::from(format!("{drive}{MAIN_SEPARATOR_STR}")) + } else if drive.is_empty() || same_part(drive.as_ref(), root_drive.as_ref()) { + root.components().collect() + } else { + PathBuf::from(drive) + }; + path.extend(&parts.tail); + path +} + +/// `path.resolve()` of an existing path, spelled without Windows' verbatim `\\?\` prefix. +#[cfg(unix)] +pub(crate) fn real_path(path: &Path) -> std::io::Result { + std::fs::canonicalize(path) +} + +/// `path.resolve()` of an existing path, spelled without Windows' verbatim `\\?\` prefix. +#[cfg(not(unix))] +pub(crate) fn real_path(path: &Path) -> std::io::Result { + use std::ffi::OsString; + use std::path::{Component, Prefix}; + + let real: PathBuf = std::fs::canonicalize(path)?; + let mut components = real.components(); + let Some(Component::Prefix(prefix)) = components.next() else { + return Ok(real); + }; + let anchor: OsString = match prefix.kind() { + Prefix::VerbatimDisk(drive) => OsString::from(format!("{}:", char::from(drive))), + Prefix::VerbatimUNC(server, share) => { + let mut anchor: OsString = OsString::from(r"\\"); + anchor.push(server); + anchor.push(r"\"); + anchor.push(share); + anchor + } + _ => return Ok(real), + }; + let mut plain: PathBuf = PathBuf::from(anchor); + plain.extend(components); + Ok(plain) +} + +/// `path.is_relative_to(root)`, case-folded on Windows as `pathlib` compares there. +pub(crate) fn inside(path: &Path, root: &Path) -> bool { + if cfg!(unix) { + path.starts_with(root) + } else { + pure_relative_to(&pure_parts(path), &pure_parts(root)).is_some() + } +} + /// `Path.is_file()`: false for the failures `pathlib` ignores, the failure otherwise. pub(crate) fn points_to_file(path: &Path) -> Result { match std::fs::metadata(path) { Ok(metadata) => Ok(metadata.is_file()), Err(error) => match error.raw_os_error() { - Some(errno) if !MISSING_FILE_ERRNOS.contains(&errno) => { + Some(errno) if !MISSING_FILE_ERRORS.contains(&errno) => { Err(ProjectTreeFileError::Unreadable { path: path.to_path_buf(), errno, @@ -32,16 +100,20 @@ pub(crate) fn points_to_file(path: &Path) -> Result } /// A regular, non-link file under the project's resolved parent directory of the path. -pub(crate) fn regular_file(root: &Path, parts: &[&[u8]]) -> Option { - let (name, parent) = parts.split_last()?; - let directory: PathBuf = match std::fs::canonicalize(joined(root, parent)) { +pub(crate) fn regular_file(root: &Path, parts: &PureParts) -> Option { + let (name, parent) = parts.tail.split_last()?; + let parent: PureParts = PureParts { + anchor: parts.anchor.clone(), + tail: parent.to_vec(), + }; + let directory: PathBuf = match real_path(&joined(root, &parent)) { Ok(directory) => directory, Err(_unresolved) => return None, }; - if !directory.starts_with(root) { + if !inside(&directory, root) { return None; } - let candidate: PathBuf = directory.join(OsStr::from_bytes(name)); + let candidate: PathBuf = directory.join(name); match std::fs::symlink_metadata(&candidate) { Ok(metadata) => metadata.file_type().is_file().then_some(candidate), Err(_unreadable) => None, diff --git a/crates/sqlbuild-rules/src/project_tree/_helpers/relative_paths.rs b/crates/sqlbuild-rules/src/project_tree/_helpers/relative_paths.rs index 3fc2996153..4cd03fe999 100644 --- a/crates/sqlbuild-rules/src/project_tree/_helpers/relative_paths.rs +++ b/crates/sqlbuild-rules/src/project_tree/_helpers/relative_paths.rs @@ -1,88 +1,179 @@ -//! `PurePosixPath` reads of a project-relative path over its raw bytes. +//! `pathlib` reads of project paths: POSIX over raw bytes, Windows through the platform path rules. -use std::os::unix::ffi::OsStrExt; -use std::path::Path; +use std::ffi::{OsStr, OsString}; +use std::path::{MAIN_SEPARATOR, MAIN_SEPARATOR_STR, Path}; -use crate::project_tree::constants::{ - CURRENT, DOUBLE_ROOT, ESCAPED_BYTE_BASE, PARENT, ROOT, SEPARATOR, -}; +use crate::project_tree::constants::{CURRENT, PARENT, POSIX_SEPARATOR_BYTE}; +use crate::project_tree::models::PureParts; -/// The components `PurePosixPath(path).parts` keeps after any root. -pub(crate) fn parts(path: &Path) -> Vec<&[u8]> { - path.as_os_str() - .as_bytes() - .split(|byte| *byte == SEPARATOR) - .filter(|part| !part.is_empty() && *part != CURRENT) - .collect() +/// `PurePosixPath(path)`: its root (`/` or exactly `//`) and the components after it. +#[cfg(unix)] +pub(crate) fn pure_parts(path: &Path) -> PureParts { + use std::os::unix::ffi::OsStrExt; + + let separator: u8 = POSIX_SEPARATOR_BYTE; + let bytes: &[u8] = path.as_os_str().as_bytes(); + let anchor: &[u8] = match bytes { + [first, second, rest @ ..] + if *first == separator && *second == separator && rest.first() != Some(&separator) => + { + &bytes[..2] + } + [first, ..] if *first == separator => &bytes[..1], + _ => &[], + }; + PureParts { + anchor: OsStr::from_bytes(anchor).to_os_string(), + tail: bytes + .split(|byte| *byte == separator) + .filter(|part| !part.is_empty() && *part != CURRENT.as_bytes()) + .map(|part| OsStr::from_bytes(part).to_os_string()) + .collect(), + } } -/// `PurePosixPath(path).is_absolute() or ".." in PurePosixPath(path).parts`. -pub(crate) fn escapes(path: &Path) -> bool { - path.as_os_str().as_bytes().first() == Some(&SEPARATOR) || parts(path).contains(&PARENT) +/// `PureWindowsPath(path)`: its drive and root with `\` separators, then each component. +#[cfg(not(unix))] +pub(crate) fn pure_parts(path: &Path) -> PureParts { + use std::path::Component; + + use crate::project_tree::constants::POSIX_SEPARATOR; + + let mut parts: PureParts = PureParts { + anchor: OsString::new(), + tail: Vec::new(), + }; + for component in path.components() { + match component { + Component::Prefix(prefix) => parts.anchor.push( + prefix + .as_os_str() + .to_string_lossy() + .replace(POSIX_SEPARATOR, MAIN_SEPARATOR_STR), + ), + Component::RootDir => parts.anchor.push(MAIN_SEPARATOR_STR), + Component::CurDir => {} + Component::ParentDir => parts.tail.push(OsString::from(PARENT)), + Component::Normal(part) => parts.tail.push(part.to_os_string()), + } + } + parts } -/// `PurePosixPath(path).parts`: its root (`/` or exactly `//`) first, then each component. -pub(crate) fn rooted_parts(path: &Path) -> Vec<&[u8]> { - let bytes: &[u8] = path.as_os_str().as_bytes(); - let mut rooted: Vec<&[u8]> = Vec::new(); - if bytes.starts_with(DOUBLE_ROOT) && bytes.get(DOUBLE_ROOT.len()) != Some(&SEPARATOR) { - rooted.push(DOUBLE_ROOT); - } else if bytes.first() == Some(&SEPARATOR) { - rooted.push(ROOT); +/// Whether two parts are equal as `pathlib` compares them: exactly, or case-folded on Windows. +pub(crate) fn same_part(left: &OsStr, right: &OsStr) -> bool { + if cfg!(unix) { + left == right + } else { + left.to_string_lossy().to_lowercase() == right.to_string_lossy().to_lowercase() } - rooted.extend(parts(path)); - rooted } -/// Whether a part list starts with a root. -pub(crate) fn is_rooted(parts: &[&[u8]]) -> bool { - parts - .first() - .is_some_and(|first| *first == ROOT || *first == DOUBLE_ROOT) +/// Whether `pathlib` considers two pure paths equal. +pub(crate) fn same_path(left: &PureParts, right: &PureParts) -> bool { + same_part(&left.anchor, &right.anchor) + && left.tail.len() == right.tail.len() + && left + .tail + .iter() + .zip(&right.tail) + .all(|(left, right)| same_part(left, right)) } -/// `str(PurePosixPath(*parts))` as bytes: the root, the components joined by `/`, or `.`. -pub(crate) fn joined_text(parts: &[&[u8]]) -> Vec { - let (root, components): (&[u8], &[&[u8]]) = match parts.split_first() { - Some((first, rest)) if *first == ROOT || *first == DOUBLE_ROOT => (first, rest), - _ => (&[], parts), - }; - if root.is_empty() && components.is_empty() { - return CURRENT.to_vec(); +/// `str(path)`: the anchor, then the components joined by the platform separator, or `.`. +pub(crate) fn pure_text(parts: &PureParts) -> OsString { + if parts.anchor.is_empty() && parts.tail.is_empty() { + return OsString::from(CURRENT); + } + let mut text: OsString = parts.anchor.clone(); + for (index, part) in parts.tail.iter().enumerate() { + if index > 0 { + text.push(MAIN_SEPARATOR_STR); + } + text.push(part); } - let mut text: Vec = root.to_vec(); - text.extend(components.join(&SEPARATOR)); text } -/// `PurePosixPath(*parts).parent` as parts: a bare root or `.` is its own parent. -pub(crate) fn parent_parts<'a>(parts: &[&'a [u8]]) -> Vec<&'a [u8]> { - match parts { - [] => Vec::new(), - [only] if is_rooted(&[only]) => vec![*only], - [init @ .., _] => init.to_vec(), - } +/// `path.as_posix()` as encoded bytes. +pub(crate) fn posix_bytes(parts: &PureParts) -> Vec { + pure_text(parts) + .as_encoded_bytes() + .iter() + .map(|byte| { + if char::from(*byte) == MAIN_SEPARATOR { + POSIX_SEPARATOR_BYTE + } else { + *byte + } + }) + .collect() +} + +/// `path.parent`: a bare anchor or `.` is its own parent. +pub(crate) fn pure_parent(parts: &PureParts) -> PureParts { + let mut parent: PureParts = parts.clone(); + parent.tail.pop(); + parent +} + +/// `path.relative_to(base)`, or `None` when `path` is not beneath `base`. +pub(crate) fn pure_relative_to(parts: &PureParts, base: &PureParts) -> Option { + let beneath: bool = same_part(&parts.anchor, &base.anchor) + && parts.tail.len() >= base.tail.len() + && parts + .tail + .iter() + .zip(&base.tail) + .all(|(part, prefix)| same_part(part, prefix)); + beneath.then(|| PureParts { + anchor: OsString::new(), + tail: parts.tail[base.tail.len()..].to_vec(), + }) +} + +/// `path.parts`: the anchor, when there is one, then each component. +pub(crate) fn part_list(parts: PureParts) -> Vec { + let anchor: Option = (!parts.anchor.is_empty()).then_some(parts.anchor); + anchor.into_iter().chain(parts.tail).collect() } -/// `PurePosixPath(*parts).relative_to(PurePosixPath(*base)).parts`, or `None` when not beneath. -pub(crate) fn relative_parts_to<'a>(parts: &[&'a [u8]], base: &[&[u8]]) -> Option> { - let beneath: bool = parts.len() >= base.len() - && parts.iter().zip(base).all(|(part, prefix)| part == prefix) - && !(base.is_empty() && is_rooted(parts)); - beneath.then(|| parts[base.len()..].to_vec()) +/// `path.is_absolute() or ".." in path.parts`. +pub(crate) fn escapes(path: &Path) -> bool { + path.is_absolute() || pure_parts(path).tail.iter().any(|part| part == PARENT) } -/// Path bytes as text with one character per byte UTF-8 cannot decode, as `surrogateescape` reads. -pub(crate) fn escaped_text(bytes: &[u8]) -> String { - let mut text: String = String::with_capacity(bytes.len()); - for chunk in bytes.utf8_chunks() { +/// Path text with one private-use character per undecodable byte, as `surrogateescape` reads. +#[cfg(unix)] +pub(crate) fn escaped_text(path: &Path) -> String { + use std::os::unix::ffi::OsStrExt; + + use crate::project_tree::constants::ESCAPED_BASE; + + let mut text: String = String::new(); + for chunk in path.as_os_str().as_bytes().utf8_chunks() { text.push_str(chunk.valid()); text.extend( chunk .invalid() .iter() - .filter_map(|byte| char::from_u32(ESCAPED_BYTE_BASE + u32::from(*byte))), + .filter_map(|byte| char::from_u32(ESCAPED_BASE + u32::from(*byte))), ); } text } + +/// Path text with one private-use character per unpaired surrogate, as Python keeps them. +#[cfg(windows)] +pub(crate) fn escaped_text(path: &Path) -> String { + use std::os::windows::ffi::OsStrExt; + + use crate::project_tree::constants::ESCAPED_BASE; + + char::decode_utf16(path.as_os_str().encode_wide()) + .filter_map(|decoded| match decoded { + Ok(character) => Some(character), + Err(error) => char::from_u32(ESCAPED_BASE + u32::from(error.unpaired_surrogate())), + }) + .collect() +} diff --git a/crates/sqlbuild-rules/src/project_tree/constants.rs b/crates/sqlbuild-rules/src/project_tree/constants.rs index 4e4f7e9c41..7759efbbb0 100644 --- a/crates/sqlbuild-rules/src/project_tree/constants.rs +++ b/crates/sqlbuild-rules/src/project_tree/constants.rs @@ -8,15 +8,20 @@ pub(crate) const TRACKED_TEXT_SUFFIXES: [&str; 5] = [".py", ".sql", ".toml", ".y pub(crate) const HIDDEN_PREFIX: u8 = b'.'; /// `errno` values `pathlib` treats as a missing file when it tests `is_file()`. #[cfg(target_os = "macos")] -pub(crate) const MISSING_FILE_ERRNOS: [i32; 4] = [2, 20, 9, 62]; -#[cfg(not(target_os = "macos"))] -pub(crate) const MISSING_FILE_ERRNOS: [i32; 4] = [2, 20, 9, 40]; -pub(crate) const SEPARATOR: u8 = b'/'; -pub(crate) const ROOT: &[u8] = b"/"; -/// POSIX keeps exactly two leading slashes as their own root. -pub(crate) const DOUBLE_ROOT: &[u8] = b"//"; -pub(crate) const DESCENDANT_SEPARATOR: &[u8] = b"/"; -pub(crate) const CURRENT: &[u8] = b"."; -pub(crate) const PARENT: &[u8] = b".."; -/// Private-use characters standing for undecodable path bytes, one per byte value. -pub(crate) const ESCAPED_BYTE_BASE: u32 = 0xF_0000; +pub(crate) const MISSING_FILE_ERRORS: [i32; 4] = [2, 20, 9, 62]; +/// `errno` values `pathlib` treats as a missing file when it tests `is_file()`. +#[cfg(all(unix, not(target_os = "macos")))] +pub(crate) const MISSING_FILE_ERRORS: [i32; 4] = [2, 20, 9, 40]; +/// Windows error codes `pathlib` treats as a missing file: those mapping to its ignored `errno`s. +#[cfg(not(unix))] +pub(crate) const MISSING_FILE_ERRORS: [i32; 15] = [ + 2, 3, 15, 18, 53, 67, 161, 206, 6, 114, 130, 267, 21, 123, 1921, +]; +/// The separator `as_posix()` spells and the project walk joins listed paths with. +pub(crate) const POSIX_SEPARATOR: &str = "/"; +pub(crate) const POSIX_SEPARATOR_BYTE: u8 = b'/'; +/// `str()` of a path with no anchor and no components. +pub(crate) const CURRENT: &str = "."; +pub(crate) const PARENT: &str = ".."; +/// Private-use characters standing for undecodable path bytes or unpaired surrogates. +pub(crate) const ESCAPED_BASE: u32 = 0xF_0000; diff --git a/crates/sqlbuild-rules/src/project_tree/errors.rs b/crates/sqlbuild-rules/src/project_tree/errors.rs index 9a49f8f65f..94b63825bd 100644 --- a/crates/sqlbuild-rules/src/project_tree/errors.rs +++ b/crates/sqlbuild-rules/src/project_tree/errors.rs @@ -15,7 +15,7 @@ pub enum ProjectTreeFileError { Untracked, /// The path resolves through a symbolic link. Symlink, - /// Testing the resolved path failed with this `errno`. + /// Testing the resolved path failed with this `errno`, or Windows error code. Unreadable { path: PathBuf, errno: i32 }, } diff --git a/crates/sqlbuild-rules/src/project_tree/main/mod.rs b/crates/sqlbuild-rules/src/project_tree/main/mod.rs index 1045d8d2b0..4ef4911b9a 100644 --- a/crates/sqlbuild-rules/src/project_tree/main/mod.rs +++ b/crates/sqlbuild-rules/src/project_tree/main/mod.rs @@ -1,3 +1,4 @@ +pub mod project_path_escapes; pub mod project_relative_parts; pub mod project_tree_children; pub mod project_tree_descendants; diff --git a/crates/sqlbuild-rules/src/project_tree/main/project_path_escapes.rs b/crates/sqlbuild-rules/src/project_tree/main/project_path_escapes.rs new file mode 100644 index 0000000000..a4d285536e --- /dev/null +++ b/crates/sqlbuild-rules/src/project_tree/main/project_path_escapes.rs @@ -0,0 +1,11 @@ +//! Whether a project path leaves the project. + +use std::path::Path; + +use crate::project_tree::_helpers::relative_paths::escapes; + +/// `Path(path).is_absolute() or ".." in Path(path).parts` with this platform's path rules. +#[must_use] +pub fn project_path_escapes(path: &Path) -> bool { + escapes(path) +} diff --git a/crates/sqlbuild-rules/src/project_tree/main/project_relative_parts.rs b/crates/sqlbuild-rules/src/project_tree/main/project_relative_parts.rs index 1d70ed58a8..90009bdd69 100644 --- a/crates/sqlbuild-rules/src/project_tree/main/project_relative_parts.rs +++ b/crates/sqlbuild-rules/src/project_tree/main/project_relative_parts.rs @@ -1,11 +1,11 @@ //! The parts of a path relative to the project and an optional directory inside it. -use std::ffi::{OsStr, OsString}; -use std::os::unix::ffi::OsStrExt; +use std::ffi::OsString; use std::path::{Path, PathBuf}; -use crate::project_tree::_helpers::relative_paths::{is_rooted, relative_parts_to, rooted_parts}; +use crate::project_tree::_helpers::relative_paths::{part_list, pure_parts, pure_relative_to}; use crate::project_tree::errors::RelativePartsError; +use crate::project_tree::models::PureParts; /// `path`'s parts, made project-relative through `resolve` when absolute, then relative to `under`. pub fn project_relative_parts( @@ -14,19 +14,15 @@ pub fn project_relative_parts( under: Option<&Path>, resolve: impl FnOnce(&Path) -> Result, ) -> Result, RelativePartsError> { - let resolved: PathBuf; - let mut parts: Vec<&[u8]> = rooted_parts(path); - if is_rooted(&parts) { - resolved = resolve(path).map_err(RelativePartsError::Resolve)?; - parts = relative_parts_to(&rooted_parts(&resolved), &rooted_parts(root)) + let mut parts: PureParts = pure_parts(path); + if path.is_absolute() { + let resolved: PathBuf = resolve(path).map_err(RelativePartsError::Resolve)?; + parts = pure_relative_to(&pure_parts(&resolved), &pure_parts(root)) .ok_or(RelativePartsError::Escapes)?; } if let Some(under) = under { - parts = relative_parts_to(&parts, &rooted_parts(under)) - .ok_or(RelativePartsError::NotBeneath)?; + parts = + pure_relative_to(&parts, &pure_parts(under)).ok_or(RelativePartsError::NotBeneath)?; } - Ok(parts - .into_iter() - .map(|part| OsStr::from_bytes(part).to_os_string()) - .collect()) + Ok(part_list(parts)) } diff --git a/crates/sqlbuild-rules/src/project_tree/main/project_tree_children.rs b/crates/sqlbuild-rules/src/project_tree/main/project_tree_children.rs index d8ba41b719..6494af41f8 100644 --- a/crates/sqlbuild-rules/src/project_tree/main/project_tree_children.rs +++ b/crates/sqlbuild-rules/src/project_tree/main/project_tree_children.rs @@ -2,16 +2,17 @@ use std::path::Path; -use crate::project_tree::_helpers::relative_paths::{joined_text, parent_parts, rooted_parts}; +use crate::project_tree::_helpers::relative_paths::{pure_parent, pure_parts, same_path}; +use crate::project_tree::models::PureParts; -/// Indexes of the `values` whose `PurePosixPath(value).parent == PurePosixPath(directory)`. +/// Indexes of the `values` whose `Path(value).parent == Path(directory)`. #[must_use] pub fn project_tree_children(values: &[&Path], directory: &Path) -> Vec { - let directory: Vec = joined_text(&rooted_parts(directory)); + let directory: PureParts = pure_parts(directory); values .iter() .enumerate() - .filter(|(_, value)| joined_text(&parent_parts(&rooted_parts(value))) == directory) + .filter(|(_, value)| same_path(&pure_parent(&pure_parts(value)), &directory)) .map(|(index, _)| index) .collect() } diff --git a/crates/sqlbuild-rules/src/project_tree/main/project_tree_descendants.rs b/crates/sqlbuild-rules/src/project_tree/main/project_tree_descendants.rs index 0450fad701..83283fedf8 100644 --- a/crates/sqlbuild-rules/src/project_tree/main/project_tree_descendants.rs +++ b/crates/sqlbuild-rules/src/project_tree/main/project_tree_descendants.rs @@ -1,21 +1,20 @@ //! The listed paths below one project directory. -use std::os::unix::ffi::OsStrExt; use std::path::Path; -use crate::project_tree::_helpers::relative_paths::{joined_text, rooted_parts}; -use crate::project_tree::constants::{DESCENDANT_SEPARATOR, SEPARATOR}; +use crate::project_tree::_helpers::relative_paths::{posix_bytes, pure_parts}; +use crate::project_tree::constants::POSIX_SEPARATOR_BYTE; /// Indexes of the `values` below `directory` as `as_posix().rstrip("/") + "/"` prefixes them. #[must_use] pub fn project_tree_descendants(values: &[&Path], directory: Option<&Path>) -> Vec { let prefix: Vec = match directory { Some(directory) => { - let mut text: Vec = joined_text(&rooted_parts(directory)); - while text.last() == Some(&SEPARATOR) { + let mut text: Vec = posix_bytes(&pure_parts(directory)); + while text.last() == Some(&POSIX_SEPARATOR_BYTE) { text.pop(); } - text.extend_from_slice(DESCENDANT_SEPARATOR); + text.push(POSIX_SEPARATOR_BYTE); text } None => Vec::new(), @@ -23,7 +22,7 @@ pub fn project_tree_descendants(values: &[&Path], directory: Option<&Path>) -> V values .iter() .enumerate() - .filter(|(_, value)| value.as_os_str().as_bytes().starts_with(&prefix)) + .filter(|(_, value)| value.as_os_str().as_encoded_bytes().starts_with(&prefix)) .map(|(index, _)| index) .collect() } diff --git a/crates/sqlbuild-rules/src/project_tree/main/project_tree_entries.rs b/crates/sqlbuild-rules/src/project_tree/main/project_tree_entries.rs index ed57e4aa4e..a925fc6b62 100644 --- a/crates/sqlbuild-rules/src/project_tree/main/project_tree_entries.rs +++ b/crates/sqlbuild-rules/src/project_tree/main/project_tree_entries.rs @@ -2,7 +2,7 @@ use std::path::{Path, PathBuf}; -use crate::project_tree::_helpers::entries::{kind, listed_directory}; +use crate::project_tree::_helpers::entries::{kind, listed_directory, posix_child}; use crate::project_tree::models::{EntryKind, ProjectTreeEntry}; /// Files and directories under `root`, without links, hidden or top-level tooling directories. @@ -15,7 +15,7 @@ pub fn project_tree_entries(root: &Path) -> Vec { continue; }; for entry in listing.flatten() { - let child: PathBuf = relative.join(entry.file_name()); + let child: PathBuf = posix_child(&relative, &entry.file_name()); match kind(&entry) { EntryKind::Link => {} EntryKind::Directory => { diff --git a/crates/sqlbuild-rules/src/project_tree/main/project_tree_file.rs b/crates/sqlbuild-rules/src/project_tree/main/project_tree_file.rs index 10c97ad871..3f76fe8cc9 100644 --- a/crates/sqlbuild-rules/src/project_tree/main/project_tree_file.rs +++ b/crates/sqlbuild-rules/src/project_tree/main/project_tree_file.rs @@ -1,27 +1,29 @@ //! The real file a project-relative path names, when rules may read it. -use std::os::unix::ffi::OsStrExt; use std::path::{Path, PathBuf}; -use crate::project_tree::_helpers::files::{joined, points_to_file, regular_file, tracked}; -use crate::project_tree::_helpers::relative_paths::{escapes, parts}; +use crate::project_tree::_helpers::files::{ + inside, joined, points_to_file, real_path, regular_file, tracked, +}; +use crate::project_tree::_helpers::relative_paths::{escapes, pure_parts}; use crate::project_tree::errors::ProjectTreeFileError; +use crate::project_tree::models::PureParts; /// The tracked file inside the resolved `root` that the project-relative `path` names. pub fn project_tree_file(root: &Path, path: &Path) -> Result { if escapes(path) { return Err(ProjectTreeFileError::Escapes); } - if path.as_os_str().as_bytes().contains(&0) { + if path.as_os_str().as_encoded_bytes().contains(&0) { return Err(ProjectTreeFileError::EmbeddedNul); } - let parts: Vec<&[u8]> = parts(path); + let parts: PureParts = pure_parts(path); if let Some(regular) = regular_file(root, &parts) { return tracked(regular); } let lexical: PathBuf = joined(root, &parts); - let candidate: PathBuf = std::fs::canonicalize(&lexical).unwrap_or(lexical); - if !candidate.starts_with(root) || !points_to_file(&candidate)? { + let candidate: PathBuf = real_path(&lexical).unwrap_or(lexical); + if !inside(&candidate, root) || !points_to_file(&candidate)? { return Err(ProjectTreeFileError::Missing); } let candidate: PathBuf = tracked(candidate)?; diff --git a/crates/sqlbuild-rules/src/project_tree/main/project_tree_glob.rs b/crates/sqlbuild-rules/src/project_tree/main/project_tree_glob.rs index a94a230d9c..adec478a24 100644 --- a/crates/sqlbuild-rules/src/project_tree/main/project_tree_glob.rs +++ b/crates/sqlbuild-rules/src/project_tree/main/project_tree_glob.rs @@ -1,6 +1,5 @@ //! The listed paths a shell-style pattern matches. -use std::os::unix::ffi::OsStrExt; use std::path::Path; use sqlbuild_core::text::main::glob_match::glob_match; @@ -10,11 +9,11 @@ use crate::project_tree::_helpers::relative_paths::escaped_text; /// Indexes of the `values` that `fnmatch.fnmatchcase(value, pattern)` accepts. #[must_use] pub fn project_tree_glob(values: &[&Path], pattern: &Path) -> Vec { - let pattern: String = escaped_text(pattern.as_os_str().as_bytes()); + let pattern: String = escaped_text(pattern); values .iter() .enumerate() - .filter(|(_, value)| glob_match(&escaped_text(value.as_os_str().as_bytes()), &pattern)) + .filter(|(_, value)| glob_match(&escaped_text(value), &pattern)) .map(|(index, _)| index) .collect() } diff --git a/crates/sqlbuild-rules/src/project_tree/models.rs b/crates/sqlbuild-rules/src/project_tree/models.rs index 7a5e0f5a13..65a53b938f 100644 --- a/crates/sqlbuild-rules/src/project_tree/models.rs +++ b/crates/sqlbuild-rules/src/project_tree/models.rs @@ -1,5 +1,6 @@ //! One discovered project path. +use std::ffi::OsString; use std::path::PathBuf; /// A project-relative file or directory the tree lists. @@ -15,3 +16,10 @@ pub(crate) enum EntryKind { Directory, Other, } + +/// A pure path as `pathlib` splits it: its anchor (drive and root), then each component. +#[derive(Clone, Debug, PartialEq, Eq)] +pub(crate) struct PureParts { + pub(crate) anchor: OsString, + pub(crate) tail: Vec, +} diff --git a/crates/sqlbuild-rules/src/project_tree/tests.rs b/crates/sqlbuild-rules/src/project_tree/tests.rs index f6bd42db5d..30903ec9e3 100644 --- a/crates/sqlbuild-rules/src/project_tree/tests.rs +++ b/crates/sqlbuild-rules/src/project_tree/tests.rs @@ -1,8 +1,16 @@ +#[cfg(unix)] +#[path = "tests/test_escapes.rs"] +mod escapes; +#[cfg(unix)] #[path = "tests/helpers.rs"] mod helpers; #[path = "tests/test_path_queries.rs"] mod path_queries; +#[cfg(unix)] #[path = "tests/test_project_tree.rs"] mod project_tree; +#[cfg(unix)] +#[path = "tests/test_relative_parts.rs"] +mod relative_parts; #[path = "tests/test_types.rs"] mod test_types; diff --git a/crates/sqlbuild-rules/src/project_tree/tests/test_escapes.rs b/crates/sqlbuild-rules/src/project_tree/tests/test_escapes.rs new file mode 100644 index 0000000000..7240d19d0f --- /dev/null +++ b/crates/sqlbuild-rules/src/project_tree/tests/test_escapes.rs @@ -0,0 +1,68 @@ +use std::path::Path; + +use crate::project_tree::main::project_path_escapes::project_path_escapes; +use crate::project_tree::tests::test_types::EscapesTestCase; + +#[test] +fn given_paths_when_checking_for_escapes_then_absolute_and_parent_paths_escape() { + let test_cases = [ + EscapesTestCase { + description: "a relative file", + path: "models/a.sql", + expected_escapes: false, + }, + EscapesTestCase { + description: "current and empty parts", + path: "./a//b/", + expected_escapes: false, + }, + EscapesTestCase { + description: "an empty path", + path: "", + expected_escapes: false, + }, + EscapesTestCase { + description: "the current directory", + path: ".", + expected_escapes: false, + }, + EscapesTestCase { + description: "an absolute path", + path: "/abs/x.yml", + expected_escapes: true, + }, + EscapesTestCase { + description: "a double-slash root", + path: "//x", + expected_escapes: true, + }, + EscapesTestCase { + description: "a leading parent", + path: "../x.sql", + expected_escapes: true, + }, + EscapesTestCase { + description: "an inner parent", + path: "a/../x.sql", + expected_escapes: true, + }, + EscapesTestCase { + description: "only a parent", + path: "..", + expected_escapes: true, + }, + EscapesTestCase { + description: "dots inside names", + path: "a..b/..c", + expected_escapes: false, + }, + ]; + for test_case in test_cases { + assert_eq!( + project_path_escapes(Path::new(test_case.path)), + test_case.expected_escapes, + "{}", + test_case.description + ); + } +} diff --git a/crates/sqlbuild-rules/src/project_tree/tests/test_path_queries.rs b/crates/sqlbuild-rules/src/project_tree/tests/test_path_queries.rs index b56f909415..964c6ebe4f 100644 --- a/crates/sqlbuild-rules/src/project_tree/tests/test_path_queries.rs +++ b/crates/sqlbuild-rules/src/project_tree/tests/test_path_queries.rs @@ -1,14 +1,10 @@ -use std::path::{Path, PathBuf}; +use std::path::Path; -use crate::project_tree::errors::RelativePartsError; -use crate::project_tree::main::project_relative_parts::project_relative_parts; use crate::project_tree::main::project_tree_children::project_tree_children; use crate::project_tree::main::project_tree_descendants::project_tree_descendants; use crate::project_tree::main::project_tree_glob::project_tree_glob; use crate::project_tree::main::tracked_text_path::tracked_text_path; -use crate::project_tree::tests::test_types::{ - PathSelectionTestCase, RelativePartsTestCase, TrackedTextTestCase, -}; +use crate::project_tree::tests::test_types::{PathSelectionTestCase, TrackedTextTestCase}; const VALUES: [&str; 6] = [ "models", @@ -77,70 +73,6 @@ fn given_listed_paths_when_selecting_then_children_descendants_and_globs_match_p } } -#[test] -fn given_paths_when_made_project_relative_then_parts_follow_relative_to() { - let test_cases = [ - RelativePartsTestCase { - description: "a relative path keeps its parts", - path: "models/../a.sql", - resolved: "", - under: None, - expected_parts: Ok(vec!["models", "..", "a.sql"]), - }, - RelativePartsTestCase { - description: "an absolute path inside the project", - path: "/project/link/a.sql", - resolved: "/project/models/a.sql", - under: Some("models"), - expected_parts: Ok(vec!["a.sql"]), - }, - RelativePartsTestCase { - description: "an absolute path outside the project", - path: "/elsewhere/a.sql", - resolved: "/elsewhere/a.sql", - under: None, - expected_parts: Err(RelativePartsError::Escapes), - }, - RelativePartsTestCase { - description: "a path outside the requested folder", - path: "models/a.sql", - resolved: "", - under: Some("rules"), - expected_parts: Err(RelativePartsError::NotBeneath), - }, - RelativePartsTestCase { - description: "a relative path is beneath the current folder", - path: "a.sql", - resolved: "", - under: Some("."), - expected_parts: Ok(vec!["a.sql"]), - }, - ]; - for test_case in test_cases { - let parts = project_relative_parts( - Path::new(test_case.path), - Path::new("/project"), - test_case.under.map(Path::new), - |_| Ok::(PathBuf::from(test_case.resolved)), - ) - .map(|parts| { - parts - .into_iter() - .map(|part| part.to_string_lossy().into_owned()) - .collect::>() - }); - assert_eq!( - parts - .as_ref() - .map(|parts| parts.iter().map(String::as_str).collect::>()) - .map_err(Clone::clone), - test_case.expected_parts, - "{}", - test_case.description - ); - } -} - #[test] fn given_file_names_when_checking_tracked_text_then_only_known_suffixes_count() { let test_cases = [ diff --git a/crates/sqlbuild-rules/src/project_tree/tests/test_relative_parts.rs b/crates/sqlbuild-rules/src/project_tree/tests/test_relative_parts.rs new file mode 100644 index 0000000000..03fddedadd --- /dev/null +++ b/crates/sqlbuild-rules/src/project_tree/tests/test_relative_parts.rs @@ -0,0 +1,69 @@ +use std::path::{Path, PathBuf}; + +use crate::project_tree::errors::RelativePartsError; +use crate::project_tree::main::project_relative_parts::project_relative_parts; +use crate::project_tree::tests::test_types::RelativePartsTestCase; + +#[test] +fn given_paths_when_made_project_relative_then_parts_follow_relative_to() { + let test_cases = [ + RelativePartsTestCase { + description: "a relative path keeps its parts", + path: "models/../a.sql", + resolved: "", + under: None, + expected_parts: Ok(vec!["models", "..", "a.sql"]), + }, + RelativePartsTestCase { + description: "an absolute path inside the project", + path: "/project/link/a.sql", + resolved: "/project/models/a.sql", + under: Some("models"), + expected_parts: Ok(vec!["a.sql"]), + }, + RelativePartsTestCase { + description: "an absolute path outside the project", + path: "/elsewhere/a.sql", + resolved: "/elsewhere/a.sql", + under: None, + expected_parts: Err(RelativePartsError::Escapes), + }, + RelativePartsTestCase { + description: "a path outside the requested folder", + path: "models/a.sql", + resolved: "", + under: Some("rules"), + expected_parts: Err(RelativePartsError::NotBeneath), + }, + RelativePartsTestCase { + description: "a relative path is beneath the current folder", + path: "a.sql", + resolved: "", + under: Some("."), + expected_parts: Ok(vec!["a.sql"]), + }, + ]; + for test_case in test_cases { + let parts = project_relative_parts( + Path::new(test_case.path), + Path::new("/project"), + test_case.under.map(Path::new), + |_| Ok::(PathBuf::from(test_case.resolved)), + ) + .map(|parts| { + parts + .into_iter() + .map(|part| part.to_string_lossy().into_owned()) + .collect::>() + }); + assert_eq!( + parts + .as_ref() + .map(|parts| parts.iter().map(String::as_str).collect::>()) + .map_err(Clone::clone), + test_case.expected_parts, + "{}", + test_case.description + ); + } +} diff --git a/crates/sqlbuild-rules/src/project_tree/tests/test_types.rs b/crates/sqlbuild-rules/src/project_tree/tests/test_types.rs index 3e05f20299..7918aa2a67 100644 --- a/crates/sqlbuild-rules/src/project_tree/tests/test_types.rs +++ b/crates/sqlbuild-rules/src/project_tree/tests/test_types.rs @@ -1,10 +1,13 @@ +#[cfg(unix)] use crate::project_tree::errors::{ProjectTreeFileError, RelativePartsError}; +#[cfg(unix)] pub(crate) struct ProjectTreeEntriesTestCase { pub(crate) description: &'static str, pub(crate) expected_entries: Vec<(&'static str, bool)>, } +#[cfg(unix)] pub(crate) struct ProjectTreeFileTestCase { pub(crate) description: &'static str, pub(crate) path: &'static str, @@ -20,6 +23,7 @@ pub(crate) struct PathSelectionTestCase { pub(crate) expected_glob: Vec<&'static str>, } +#[cfg(unix)] pub(crate) struct RelativePartsTestCase { pub(crate) description: &'static str, pub(crate) path: &'static str, @@ -33,3 +37,10 @@ pub(crate) struct TrackedTextTestCase { pub(crate) path: &'static str, pub(crate) expected_tracked: bool, } + +#[cfg(unix)] +pub(crate) struct EscapesTestCase { + pub(crate) description: &'static str, + pub(crate) path: &'static str, + pub(crate) expected_escapes: bool, +} From 78f547076ee7e466744f31bc34070211ff130c57 Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Fri, 16 Oct 2026 03:20:13 +0100 Subject: [PATCH 48/48] fix: set the allocator purge delay before the launched entry loads native code The `sqb` launcher hands off to `sqlbuild.cli.entry.main.launched`, which imported `LAUNCH_RECORD_OPTION` from `sqlbuild._native` at module import. That loaded the extension, and its mimalloc allocator, before `entry.main` defaulted `MIMALLOC_PURGE_DELAY`, so mimalloc kept its default purge delay and returned far less freed memory to the OS. Dense 10000-model compiles peaked about 200-400 MiB higher. Default the purge delay in the launched entry before importing the native constant, and extend the fresh-process allocator entry test to cover the launched entry. --- src/sqlbuild/cli/entry/main/launched.py | 13 ++++++++-- .../sqlbuild/cli/commands/main/_test_types.py | 1 + .../main/test_native_allocator_entry.py | 24 ++++++++++++++++--- 3 files changed, 33 insertions(+), 5 deletions(-) diff --git a/src/sqlbuild/cli/entry/main/launched.py b/src/sqlbuild/cli/entry/main/launched.py index 8632729d51..b544413b74 100644 --- a/src/sqlbuild/cli/entry/main/launched.py +++ b/src/sqlbuild/cli/entry/main/launched.py @@ -1,13 +1,22 @@ """Python CLI entrypoint the native `sqb` executable hands every invocation it does not serve.""" +import os import sys -from sqlbuild._native import LAUNCH_RECORD_OPTION +from sqlbuild.cli.entry.constants import ( + NATIVE_ALLOCATOR_PURGE_DELAY_MILLISECONDS, + NATIVE_ALLOCATOR_PURGE_DELAY_VARIABLE, +) from sqlbuild.cli.entry.main.entry import main as run_cli def main() -> int: - """Run the CLI with the arguments `sqb` received; the CLI reads a leading launch record.""" + """Run the CLI with the `sqb` arguments, setting the allocator purge delay first.""" + + os.environ.setdefault( + NATIVE_ALLOCATOR_PURGE_DELAY_VARIABLE, NATIVE_ALLOCATOR_PURGE_DELAY_MILLISECONDS + ) + from sqlbuild._native import LAUNCH_RECORD_OPTION arguments: list[str] = sys.argv[1:] offered: bool = bool(arguments) and arguments[0].startswith(LAUNCH_RECORD_OPTION) diff --git a/tests/integration/src/sqlbuild/cli/commands/main/_test_types.py b/tests/integration/src/sqlbuild/cli/commands/main/_test_types.py index 8b95a482f2..0b688f3b54 100644 --- a/tests/integration/src/sqlbuild/cli/commands/main/_test_types.py +++ b/tests/integration/src/sqlbuild/cli/commands/main/_test_types.py @@ -928,6 +928,7 @@ class DiscoveryWorkTestCase: @dataclass(frozen=True) class NativeAllocatorEntryTestCase: description: str + entry_module: str inherited_environment: tuple[tuple[str, str], ...] expected_purge_delay: str diff --git a/tests/integration/src/sqlbuild/cli/commands/main/test_native_allocator_entry.py b/tests/integration/src/sqlbuild/cli/commands/main/test_native_allocator_entry.py index f284fb5cef..802f5d1af7 100644 --- a/tests/integration/src/sqlbuild/cli/commands/main/test_native_allocator_entry.py +++ b/tests/integration/src/sqlbuild/cli/commands/main/test_native_allocator_entry.py @@ -14,14 +14,18 @@ ) _PURGE_DELAY_VARIABLE: str = "MIMALLOC_PURGE_DELAY" +_ENTRY_MODULE: str = "sqlbuild.cli.entry.main.entry" +_LAUNCHED_MODULE: str = "sqlbuild.cli.entry.main.launched" _SCRIPT: str = """ +import importlib import json import os import sys -from sqlbuild.cli.entry.main.entry import main +entry = importlib.import_module(sys.argv[1]) loaded_at_import = "sqlbuild._native" in sys.modules -exit_code = main(["--version"]) +sys.argv = ["sqlbuild-python", "--version"] +exit_code = entry.main() if entry.__name__.endswith(".launched") else entry.main(["--version"]) loaded_by_version = "sqlbuild._native" in sys.modules import sqlbuild._native print(json.dumps({ @@ -38,11 +42,25 @@ [ NativeAllocatorEntryTestCase( description="unset purge delay defaults to ten milliseconds", + entry_module=_ENTRY_MODULE, inherited_environment=(), expected_purge_delay="10", ), NativeAllocatorEntryTestCase( description="user purge delay is preserved", + entry_module=_ENTRY_MODULE, + inherited_environment=((_PURGE_DELAY_VARIABLE, "250"),), + expected_purge_delay="250", + ), + NativeAllocatorEntryTestCase( + description="launched entry defaults the purge delay before native load", + entry_module=_LAUNCHED_MODULE, + inherited_environment=(), + expected_purge_delay="10", + ), + NativeAllocatorEntryTestCase( + description="launched entry preserves a user purge delay", + entry_module=_LAUNCHED_MODULE, inherited_environment=((_PURGE_DELAY_VARIABLE, "250"),), expected_purge_delay="250", ), @@ -57,7 +75,7 @@ def test_given_fresh_process_when_running_cli_entry_then_purge_delay_is_set_befo environment.update(test_case.inherited_environment) result: subprocess.CompletedProcess[str] = subprocess.run( - [sys.executable, "-c", _SCRIPT], + [sys.executable, "-c", _SCRIPT, test_case.entry_module], check=True, capture_output=True, env=environment,