diff --git a/src/sqlbuild/.agents/skills/sqlbuild/SKILL.md b/src/sqlbuild/.agents/skills/sqlbuild/SKILL.md index ebbd8a205..a0123a235 100644 --- a/src/sqlbuild/.agents/skills/sqlbuild/SKILL.md +++ b/src/sqlbuild/.agents/skills/sqlbuild/SKILL.md @@ -6592,6 +6592,17 @@ columns, and 120 characters per value, and pass through diagnostic redaction. Th as structured `unexpected_samples` and `missing_samples` in JSON output. Sampling failure never replaces the known test failure. +When exactly one row exists in each difference direction and the bounded samples have the same +columns, SQLBuild also aligns the rows and reports only changed columns with their redacted actual +and expected values. It does not guess an alignment when several rows differ or sampling is +incomplete. + +Before opening a warehouse connection, SQLBuild statically checks fixture shapes when SQL analysis +is enabled. Missing columns are reported together with the test path, fixture resource, and models +that read them. Statically provable collection-versus-scalar type conflicts identify the affected +column and suggest explicit `CAST`, `ARRAY_CONSTRUCT`, or `PARSE_JSON` expressions. SQLBuild never +invents missing values. + The trailing `SELECT 1` is required as a ceremonial closing statement. ### CTE conventions @@ -6634,6 +6645,17 @@ SELECT 1 SQLBuild topologically sorts the expected models, resolves each intermediate model's real SQL with mocks substituted, and chains the outputs forward. Every model between the mocked sources and the expected model is computed automatically. +Inspect that boundary without opening a warehouse connection: + +```bash +sqb test --select fact_orders --inspect +``` + +The resolved plan lists mocked sources, refs, seeds, real models in execution order, expected +models, and any unsatisfied leaf dependencies. A mocked ref is called out explicitly because its +real model SQL will not execute. Inspection exits non-zero when the plan contains unresolved +dependency errors. + ### Mocking refs and seeds You can mock models directly with `__ref__` and seeds with `__seed__`, not just sources. This skips the model's real SQL (or the seed's real CSV data) and provides controlled data instead: @@ -6965,8 +6987,15 @@ SELECT 1 ``` The cases report independently as `order status: maps source states [completed]`, `[cancelled]`, -and `[pending]`. Authored order controls display order. Selection remains at the parent test's -resolved model or resource, so selecting `stg_orders` selects every case; there is no case selector. +and `[pending]`. Authored order controls display order. Selecting `stg_orders` selects every case by +default. Add `--case` to run one named case without changing the test file: + +```bash +sqb test --select stg_orders --case cancelled +``` + +If the case does not exist within the parent/model selection, SQLBuild reports the available case +names. Supported scalar types are `string`, `integer`, `boolean`, `float`, and exact `decimal`. Decimal values are quoted, such as `tax_rate "0.2000"`, so they never pass through binary float. Boolean @@ -7028,6 +7057,9 @@ SELECT 1 This produces one aggregate test result. Use native cases when each row must pass or fail independently. +Quoted strings in `VALUES` rows may contain commas, brackets, and parentheses. SQLBuild preserves +these literals while extracting test CTEs; use normal SQL quote escaping for embedded quotes. + ### Test placement Place unit test files under `tests/unit/` in your project directory. SQLBuild discovers all `.sql` files in this directory recursively. diff --git a/src/sqlbuild/cli/commands/_helpers/entry/dispatch.py b/src/sqlbuild/cli/commands/_helpers/entry/dispatch.py index 8ead76604..5bf585edd 100644 --- a/src/sqlbuild/cli/commands/_helpers/entry/dispatch.py +++ b/src/sqlbuild/cli/commands/_helpers/entry/dispatch.py @@ -247,6 +247,8 @@ def dispatch_cli_command(*, args: CliNamespace, handlers: CliEntrypointHandlers) json_output=args.json, json_output_path=args.json_output, concurrency=args.concurrency, + case_name=args.case, + inspect=args.inspect, ) ) if args.command == CliCommand.CHECK: diff --git a/src/sqlbuild/cli/commands/_helpers/entry/parser.py b/src/sqlbuild/cli/commands/_helpers/entry/parser.py index 9faf501a6..331b661af 100644 --- a/src/sqlbuild/cli/commands/_helpers/entry/parser.py +++ b/src/sqlbuild/cli/commands/_helpers/entry/parser.py @@ -276,6 +276,10 @@ def _add_quality_parsers( test_parser.add_argument("--json", action="store_true", default=False) test_parser.add_argument("--target", default=None) test_parser.add_argument("--concurrency", type=int, default=None) + test_parser.add_argument("--case", default=None, help="Run one named parameter case") + test_parser.add_argument( + "--inspect", action="store_true", default=False, help="Show the resolved test plan" + ) _ = add_execution_json_output_arg(test_parser) _ = add_select_args(test_parser) _ = add_vars_args(test_parser) diff --git a/src/sqlbuild/cli/commands/_helpers/entry/parsing.py b/src/sqlbuild/cli/commands/_helpers/entry/parsing.py index 6dae1d549..df422db7b 100644 --- a/src/sqlbuild/cli/commands/_helpers/entry/parsing.py +++ b/src/sqlbuild/cli/commands/_helpers/entry/parsing.py @@ -133,6 +133,8 @@ def parse_cli_invocation( parser.error(str(error)) if args.command == CliCommand.SCOPE: _validate_scope_args(args=args, parser=parser) + if args.command == CliCommand.TEST: + _validate_test_args(args=args, parser=parser) except SystemExit as error: exit_code: int = error.code if isinstance(error.code, int) else 1 return ParsedCliInvocation(args=None, exit_code=exit_code) @@ -181,3 +183,8 @@ def _validate_scope_args(*, args: CliNamespace, parser: argparse.ArgumentParser) parser.error("scope --explain requires a declaration-qualified identity") if not qualified_name: parser.error("scope --explain requires a declaration-qualified identity") + + +def _validate_test_args(*, args: CliNamespace, parser: argparse.ArgumentParser) -> None: + if args.inspect and (args.json or args.json_output is not None): + parser.error("test --inspect cannot be combined with --json or --json-output") diff --git a/src/sqlbuild/cli/commands/_helpers/test/inspection.py b/src/sqlbuild/cli/commands/_helpers/test/inspection.py new file mode 100644 index 000000000..4ef4f5fb7 --- /dev/null +++ b/src/sqlbuild/cli/commands/_helpers/test/inspection.py @@ -0,0 +1,59 @@ +"""Resolved SQL-test plan inspection output.""" + +from __future__ import annotations + +from collections.abc import Iterable +from typing import TextIO + +from sqlbuild.compiler.planner.models import PlanOutput, PlanWarning, SqlTestPlanEntry +from sqlbuild.compiler.planner.types import WarningSeverity + + +def write_test_plan_inspection(*, stream: TextIO, plan_output: PlanOutput) -> int: + """Write selected test fixture boundaries and real model chains without executing them.""" + + stream.write("\nResolved test plan\n\n") + for entry in plan_output.test_entries: + stream.write(f"{_entry_label(entry)}\n") + _write_names(stream=stream, label="mocked sources", names=entry.mock_source_names) + _write_names(stream=stream, label="mocked refs", names=entry.mock_ref_names) + _write_names(stream=stream, label="mocked seeds", names=entry.mock_seed_names) + _write_names(stream=stream, label="mocked dbt refs", names=entry.mock_dbt_ref_names) + _write_names( + stream=stream, + label="real models", + names=tuple(step.model_name for step in entry.chain), + ) + _write_names( + stream=stream, + label="expected models", + names=tuple( + step.model_name for step in entry.chain if step.expected_cte_sql is not None + ), + ) + for mock_name in entry.mock_ref_names: + stream.write(f" boundary: {mock_name} is replaced by __ref__{mock_name}\n") + stream.write("\n") + error_warnings: tuple[PlanWarning, ...] = tuple( + warning for warning in plan_output.warnings if warning.severity == WarningSeverity.ERROR + ) + for warning in plan_output.warnings: + stream.write(f" {warning.severity.value}: {warning.message}\n") + outcome: str = "failed" if error_warnings else "complete" + stream.write( + f"Test plan inspection {outcome}: {len(plan_output.test_entries)} selected, " + f"{len(error_warnings)} errors.\n" + ) + stream.flush() + return 1 if error_warnings else 0 + + +def _write_names(*, stream: TextIO, label: str, names: Iterable[str]) -> None: + values: tuple[str, ...] = tuple(names) + stream.write(f" {label}: {', '.join(values) if values else 'none'}\n") + + +def _entry_label(entry: SqlTestPlanEntry) -> str: + if entry.case_name is None: + return entry.name + return f"{entry.parent_name or entry.name} [{entry.case_name}]" diff --git a/src/sqlbuild/cli/commands/_helpers/test/planning.py b/src/sqlbuild/cli/commands/_helpers/test/planning.py index 72f72a909..38016b9db 100644 --- a/src/sqlbuild/cli/commands/_helpers/test/planning.py +++ b/src/sqlbuild/cli/commands/_helpers/test/planning.py @@ -13,7 +13,9 @@ CompilePipelineResult, StaticCommandContext, ) +from sqlbuild.compiler.planner.exceptions import PlannerInputError from sqlbuild.compiler.planner.main.commands.sql_test import build_test_command_plan +from sqlbuild.compiler.planner.models import PlanOutput def compile_test_plan( @@ -42,12 +44,33 @@ def compile_test_plan( options=options, on_progress=invocation.planning_progress.on_progress, ) + plan_output: PlanOutput = build_test_command_plan( + project=context.project, + adapter=invocation.adapter, + scope=context.scope, + relations=context.relations, + case_name=request.case_name, + ) + if request.case_name is not None: + plan_output = _validate_parameter_case_selection( + plan_output=plan_output, + case_name=request.case_name, + available_cases=plan_output.available_test_case_names, + ) return build_static_pipeline_result( context=context, - plan_output=build_test_command_plan( - project=context.project, - adapter=invocation.adapter, - scope=context.scope, - relations=context.relations, - ), + plan_output=plan_output, + ) + + +def _validate_parameter_case_selection( + *, plan_output: PlanOutput, case_name: str, available_cases: tuple[str, ...] +) -> PlanOutput: + if plan_output.test_entries: + return plan_output + available_text: str = ", ".join(available_cases) or "none" + raise PlannerInputError( + f"SQL test case '{case_name}' did not match the selected tests; available cases: " + f"{available_text}", + code="S008", ) diff --git a/src/sqlbuild/cli/commands/classes/cli_namespace.py b/src/sqlbuild/cli/commands/classes/cli_namespace.py index 1731499bd..eea0cb5b2 100644 --- a/src/sqlbuild/cli/commands/classes/cli_namespace.py +++ b/src/sqlbuild/cli/commands/classes/cli_namespace.py @@ -25,6 +25,8 @@ "defer_clone_from": None, "defer_sources_to": None, "target": None, + "case": None, + "inspect": False, "target_range": None, "from_target": None, "to_target": None, @@ -175,6 +177,8 @@ class CliNamespace: defer_clone_from: str | None defer_sources_to: str | None target: str | None + case: str | None + inspect: bool target_range: str | None from_target: str | None to_target: str | None diff --git a/src/sqlbuild/cli/commands/main/execution/_test.py b/src/sqlbuild/cli/commands/main/execution/_test.py index 5b43ad818..c28661b7a 100644 --- a/src/sqlbuild/cli/commands/main/execution/_test.py +++ b/src/sqlbuild/cli/commands/main/execution/_test.py @@ -7,6 +7,7 @@ prepare_test_execution, resolve_test_concurrency, ) +from sqlbuild.cli.commands._helpers.test.inspection import write_test_plan_inspection from sqlbuild.cli.commands._helpers.test.invocation import resolve_test_invocation from sqlbuild.cli.commands._helpers.test.outputs import ( resolve_test_exit_code, @@ -32,6 +33,11 @@ def run_test(request: TestCommandRequest) -> int: request=request, invocation=invocation, ) + if request.inspect: + return write_test_plan_inspection( + stream=invocation.progress_stream, + plan_output=pipeline_result.plan_output, + ) invocation.progress_stream.write("\n") write_execution_header( stream=invocation.progress_stream, diff --git a/src/sqlbuild/cli/commands/models.py b/src/sqlbuild/cli/commands/models.py index 91204d888..5ba2fa2fa 100644 --- a/src/sqlbuild/cli/commands/models.py +++ b/src/sqlbuild/cli/commands/models.py @@ -1288,6 +1288,8 @@ class TestCommandRequest: json_output: bool = False json_output_path: Path | None = None concurrency: int | None = None + case_name: str | None = None + inspect: bool = False @dataclass(frozen=True) diff --git a/src/sqlbuild/cli/output/_helpers/execution_result_document.py b/src/sqlbuild/cli/output/_helpers/execution_result_document.py index c0fbb63dd..e59457965 100644 --- a/src/sqlbuild/cli/output/_helpers/execution_result_document.py +++ b/src/sqlbuild/cli/output/_helpers/execution_result_document.py @@ -958,6 +958,15 @@ def _format_sql_test_step(step: StepResult) -> dict[str, object]: "missing_row_count": step.missing_row_count, "unexpected_samples": _format_sql_test_difference_samples(step.unexpected_samples), "missing_samples": _format_sql_test_difference_samples(step.missing_samples), + "column_differences": [ + { + "name": difference.name, + "actual": difference.actual, + "expected": difference.expected, + } + for difference in step.column_differences + ] + or None, "error_code": step.error_code, "error_help": step.error_help, "error_message": step.error_message, diff --git a/src/sqlbuild/cli/progress/main/_expectation_detail.py b/src/sqlbuild/cli/progress/main/_expectation_detail.py index 71a9e42a4..910adc22f 100644 --- a/src/sqlbuild/cli/progress/main/_expectation_detail.py +++ b/src/sqlbuild/cli/progress/main/_expectation_detail.py @@ -34,6 +34,11 @@ def format_expectation_detail(step_result: StepResult) -> str: ) parts.extend(_format_samples(label="unexpected sample", samples=step_result.unexpected_samples)) parts.extend(_format_samples(label="missing sample", samples=step_result.missing_samples)) + for difference in step_result.column_differences: + parts.append( + f"differing column {difference.name}: actual={difference.actual}, " + f"expected={difference.expected}" + ) return f" {'; '.join(parts)}" diff --git a/src/sqlbuild/compiler/compile/_helpers/assembly/project.py b/src/sqlbuild/compiler/compile/_helpers/assembly/project.py index c6b36cea6..2e5033e03 100644 --- a/src/sqlbuild/compiler/compile/_helpers/assembly/project.py +++ b/src/sqlbuild/compiler/compile/_helpers/assembly/project.py @@ -1364,6 +1364,7 @@ def _assemble_compiled_sql_test( mock_source_names=model_payload.mock_source_names, mock_seed_names=model_payload.mock_seed_names, mock_dbt_ref_names=model_payload.mock_dbt_ref_names, + expected_ctes=model_payload.expected_ctes, expected_model_names=model_payload.expected_model_names, assertion_ctes=model_payload.assertion_ctes, assertion_names=model_payload.assertion_names, diff --git a/src/sqlbuild/compiler/compile/_helpers/attachment/sql_tests.py b/src/sqlbuild/compiler/compile/_helpers/attachment/sql_tests.py index b65203bbd..a1240c99e 100644 --- a/src/sqlbuild/compiler/compile/_helpers/attachment/sql_tests.py +++ b/src/sqlbuild/compiler/compile/_helpers/attachment/sql_tests.py @@ -370,6 +370,7 @@ def _build_test_input_payload( mock_source_names=model_payload.mock_source_names, mock_seed_names=model_payload.mock_seed_names, mock_dbt_ref_names=model_payload.mock_dbt_ref_names, + expected_ctes=model_payload.expected_ctes, expected_model_names=model_payload.expected_model_names, assertion_ctes=model_payload.assertion_ctes, assertion_names=model_payload.assertion_names, diff --git a/src/sqlbuild/compiler/compile/_helpers/sql_tests/core.py b/src/sqlbuild/compiler/compile/_helpers/sql_tests/core.py index 623345d63..b3324606e 100644 --- a/src/sqlbuild/compiler/compile/_helpers/sql_tests/core.py +++ b/src/sqlbuild/compiler/compile/_helpers/sql_tests/core.py @@ -376,6 +376,7 @@ def _classify_model_sql_test_ctes( mock_source_names: list[str] = [] mock_seed_names: list[str] = [] mock_dbt_ref_names: list[str] = [] + expected_ctes: list[CompileSqlTestCte] = [] expected_model_names: list[str] = [] assertion_ctes: list[CompileSqlTestCte] = [] assertion_names: list[str] = [] @@ -460,6 +461,7 @@ def _classify_model_sql_test_ctes( ) ) _validate_expected_cte_query(cte=cte, file_label=file_label) + expected_ctes.append(cte) continue if cte.name.startswith(ASSERT_TEST_CTE_PREFIX): assertion_names.append( @@ -502,6 +504,7 @@ def _classify_model_sql_test_ctes( mock_source_names=tuple(mock_source_names), mock_seed_names=tuple(mock_seed_names), mock_dbt_ref_names=tuple(mock_dbt_ref_names), + expected_ctes=tuple(expected_ctes), expected_model_names=tuple(expected_model_names), assertion_ctes=tuple(assertion_ctes), assertion_names=tuple(assertion_names), diff --git a/src/sqlbuild/compiler/compile/constants.py b/src/sqlbuild/compiler/compile/constants.py index bfbe158ab..761594c3e 100644 --- a/src/sqlbuild/compiler/compile/constants.py +++ b/src/sqlbuild/compiler/compile/constants.py @@ -82,6 +82,10 @@ TEMPLATE_ESCAPE_TOKEN: str = "\\" TEMPLATE_OPEN_TOKEN: str = "${" TEMPLATE_FALSE_VALUES: frozenset[str] = frozenset({"", "0", "false"}) +POLYGLOT_LITERAL_KIND: str = "literal" +POLYGLOT_ARRAY_KIND: str = "array_func" +POLYGLOT_STRUCTURED_KINDS: frozenset[str] = frozenset({"parse_json", "struct"}) +POLYGLOT_FUNCTION_KIND: str = "function" MISSING_TEMPLATE_VALUE_MESSAGE_PARTS: frozenset[str] = frozenset( { "references missing ENV variable", diff --git a/src/sqlbuild/compiler/compile/main/_infer_fixture_columns.py b/src/sqlbuild/compiler/compile/main/_infer_fixture_columns.py new file mode 100644 index 000000000..0417802d1 --- /dev/null +++ b/src/sqlbuild/compiler/compile/main/_infer_fixture_columns.py @@ -0,0 +1,112 @@ +"""Public fixture-column inference entrypoint.""" + +from __future__ import annotations + +from typing import Any, cast + +from sqlbuild.adapter.contract.models import ExpressionInferenceProfile +from sqlbuild.compiler.compile._helpers.analysis.columns import ( + analyze_columns_and_lineage_with_polyglot, +) +from sqlbuild.compiler.compile.constants import ( + POLYGLOT_ARRAY_KIND, + POLYGLOT_FUNCTION_KIND, + POLYGLOT_LITERAL_KIND, + POLYGLOT_STRUCTURED_KINDS, +) +from sqlbuild.compiler.compile.models import InferredColumn, PolyglotAnalysisResult +from sqlbuild.compiler.sql_analysis.main.import_polyglot_sql import import_polyglot_sql + + +def infer_fixture_columns( + *, query_sql: str, inference_profile: ExpressionInferenceProfile +) -> tuple[InferredColumn, ...] | None: + """Infer fixture columns and conservative simple-literal types without warehouse access.""" + + analysis: PolyglotAnalysisResult = analyze_columns_and_lineage_with_polyglot( + query_sql=query_sql, + inference_profile=inference_profile, + ) + if not analysis.analysis_succeeded or analysis.has_star or analysis.columns is None: + return None + simple_types: dict[str, str] = _simple_projection_types( + query_sql=query_sql, + dialect=inference_profile.sql_analysis_dialect, + ) + return tuple( + InferredColumn( + name=column.name, + type=column.type or simple_types.get(column.name.casefold()), + nullability=column.nullability, + ) + for column in analysis.columns + ) + + +def _simple_projection_types(*, query_sql: str, dialect: str | None) -> dict[str, str]: + polyglot_sql: Any = import_polyglot_sql() + try: + payload: dict[str, Any] = polyglot_sql.parse_one( + query_sql, dialect=dialect or "generic" + ).to_dict() + except polyglot_sql.PolyglotError: + return {} + select_payload: dict[str, Any] | None = _first_select(payload) + if select_payload is None: + return {} + result: dict[str, str] = {} + for expression in select_payload.get("expressions", []): + alias_payload: object = expression.get("alias") if isinstance(expression, dict) else None + if not isinstance(alias_payload, dict): + continue + alias: object = alias_payload.get("alias") + value: object = alias_payload.get("this") + if not isinstance(alias, dict) or not isinstance(value, dict): + continue + name: object = alias.get("name") + if not isinstance(name, str): + continue + raw_value_kind: object = next(iter(value), "") + if not isinstance(raw_value_kind, str): + continue + value_kind: str = raw_value_kind + value_payload: object = value.get(value_kind) + inferred_type: str | None = None + if value_kind == POLYGLOT_LITERAL_KIND and isinstance(value_payload, dict): + inferred_type = { + "string": "VARCHAR", + "number": "NUMBER", + "boolean": "BOOLEAN", + }.get(str(value_payload.get("literal_type"))) + elif value_kind == POLYGLOT_ARRAY_KIND: + inferred_type = "ARRAY" + elif value_kind in POLYGLOT_STRUCTURED_KINDS: + inferred_type = "VARIANT" + elif value_kind == POLYGLOT_FUNCTION_KIND and isinstance(value_payload, dict): + function_name: str = str(value_payload.get("name") or "").upper() + inferred_type = { + "ARRAY_CONSTRUCT": "ARRAY", + "OBJECT_CONSTRUCT": "OBJECT", + "PARSE_JSON": "VARIANT", + }.get(function_name) + if inferred_type is not None: + result[name.casefold()] = inferred_type + return result + + +def _first_select(payload: object) -> dict[str, Any] | None: + if isinstance(payload, dict): + payload_dict: dict[object, object] = cast(dict[object, object], payload) + select_payload: object = payload_dict.get("select") + if isinstance(select_payload, dict): + return cast(dict[str, Any], select_payload) + for value in payload_dict.values(): + found: dict[str, Any] | None = _first_select(value) + if found is not None: + return found + if isinstance(payload, list): + for value in payload: + found = _first_select(value) + if found is not None: + return found + return None diff --git a/src/sqlbuild/compiler/compile/models.py b/src/sqlbuild/compiler/compile/models.py index 816211aa0..890e97540 100644 --- a/src/sqlbuild/compiler/compile/models.py +++ b/src/sqlbuild/compiler/compile/models.py @@ -885,6 +885,7 @@ class CompileModelSqlTestCtes: mock_source_names: tuple[str, ...] = field(default_factory=tuple) mock_seed_names: tuple[str, ...] = field(default_factory=tuple) mock_dbt_ref_names: tuple[str, ...] = field(default_factory=tuple) + expected_ctes: tuple[CompileSqlTestCte, ...] = field(default_factory=tuple) expected_model_names: tuple[str, ...] = field(default_factory=tuple) assertion_ctes: tuple[CompileSqlTestCte, ...] = field(default_factory=tuple) assertion_names: tuple[str, ...] = field(default_factory=tuple) @@ -918,6 +919,7 @@ class CompileModelSqlTestInputPayload: mock_source_names: tuple[str, ...] = field(default_factory=tuple) mock_seed_names: tuple[str, ...] = field(default_factory=tuple) mock_dbt_ref_names: tuple[str, ...] = field(default_factory=tuple) + expected_ctes: tuple[CompileSqlTestCte, ...] = field(default_factory=tuple) expected_model_names: tuple[str, ...] = field(default_factory=tuple) assertion_ctes: tuple[CompileSqlTestCte, ...] = field(default_factory=tuple) assertion_names: tuple[str, ...] = field(default_factory=tuple) @@ -964,6 +966,7 @@ class CompiledModelSqlTestPayload: mock_source_names: tuple[str, ...] = field(default_factory=tuple) mock_seed_names: tuple[str, ...] = field(default_factory=tuple) mock_dbt_ref_names: tuple[str, ...] = field(default_factory=tuple) + expected_ctes: tuple[CompileSqlTestCte, ...] = field(default_factory=tuple) expected_model_names: tuple[str, ...] = field(default_factory=tuple) assertion_ctes: tuple[CompileSqlTestCte, ...] = field(default_factory=tuple) assertion_names: tuple[str, ...] = field(default_factory=tuple) diff --git a/src/sqlbuild/compiler/planner/_helpers/command_planning/static.py b/src/sqlbuild/compiler/planner/_helpers/command_planning/static.py index e795994e1..6ebdb1f67 100644 --- a/src/sqlbuild/compiler/planner/_helpers/command_planning/static.py +++ b/src/sqlbuild/compiler/planner/_helpers/command_planning/static.py @@ -12,6 +12,7 @@ from sqlbuild.compiler.planner._helpers.output.plan_entry import ( build_planner_relations_context, extract_seed_columns, + scope_overlaps, ) from sqlbuild.compiler.planner._helpers.output.plan_output import ( build_selected_audit_entries, @@ -111,6 +112,7 @@ def build_test_command_plan_impl( adapter: BaseAdapter, scope: PlannerScope, relations: PlannerRelationsContext, + case_name: str | None = None, ) -> PlanOutput: """Project selected SQL tests from static compile state.""" @@ -118,6 +120,17 @@ def build_test_command_plan_impl( project=project, adapter=adapter, selected_keys=scope.selected_keys, + case_name=case_name, + ) + available_test_case_names: tuple[str, ...] = tuple( + sorted( + { + test.case_name + for test in project.sql_tests + if test.case_name is not None + and scope_overlaps(scope_deps=test.scope_deps, selected_keys=scope.selected_keys) + } + ) ) test_keys: frozenset[CompiledObjectKey] = frozenset(entry.key for entry in test_entries) function_keys_set: set[CompiledObjectKey] = set() @@ -142,6 +155,7 @@ def build_test_command_plan_impl( key for key in scope.execution_order if key in scope.selected_keys | test_keys ), test_entries=tuple(test_entries), + available_test_case_names=available_test_case_names, function_entries=function_entries, warnings=tuple(warnings), ) @@ -210,6 +224,7 @@ def _base_plan_output( function_entries: tuple[FunctionPlanEntry, ...] = (), audit_entries: tuple[AuditPlanEntry, ...] = (), test_entries: tuple[SqlTestPlanEntry, ...] = (), + available_test_case_names: tuple[str, ...] = (), warnings: tuple[PlanWarning, ...] = (), ) -> PlanOutput: return PlanOutput( @@ -222,6 +237,7 @@ def _base_plan_output( function_entries=function_entries, audit_entries=audit_entries, test_entries=test_entries, + available_test_case_names=available_test_case_names, selected_keys=scope.selected_keys, warnings=warnings, upstream_deps=scope.upstream_deps, diff --git a/src/sqlbuild/compiler/planner/_helpers/output/plan_output.py b/src/sqlbuild/compiler/planner/_helpers/output/plan_output.py index 1cc2d3d73..559fcdea6 100644 --- a/src/sqlbuild/compiler/planner/_helpers/output/plan_output.py +++ b/src/sqlbuild/compiler/planner/_helpers/output/plan_output.py @@ -31,7 +31,10 @@ ) from sqlbuild.compiler.planner._helpers.resolve.resolve import resolve_function_sql from sqlbuild.compiler.planner._helpers.sql_tests.assembly import plan_test -from sqlbuild.compiler.planner.exceptions import PlannerInputError +from sqlbuild.compiler.planner.exceptions import ( + PlannerInputError, + SqlTestFixtureValidationError, +) from sqlbuild.compiler.planner.models import ( AuditPlanEntry, BackfillResult, @@ -433,21 +436,32 @@ def build_selected_test_entries( project: CompiledProject, adapter: BaseAdapter, selected_keys: frozenset[CompiledObjectKey], + case_name: str | None = None, ) -> tuple[list[SqlTestPlanEntry], list[PlanWarning]]: entries: list[SqlTestPlanEntry] = [] warnings: list[PlanWarning] = [] + fixture_diagnostics: list[str] = [] sql_test: CompiledSqlTest for sql_test in project.sql_tests: if not scope_overlaps(scope_deps=sql_test.scope_deps, selected_keys=selected_keys): continue + if case_name is not None and sql_test.case_name != case_name: + continue test_entry: SqlTestPlanEntry test_warnings: tuple[PlanWarning, ...] - test_entry, test_warnings = plan_test( - test=sql_test, - project=project, - adapter=adapter, - sql_analysis_enabled=project.settings.sql_analysis, - ) + try: + test_entry, test_warnings = plan_test( + test=sql_test, + project=project, + adapter=adapter, + sql_analysis_enabled=project.settings.sql_analysis, + validate_fixtures=True, + ) + except SqlTestFixtureValidationError as error: + fixture_diagnostics.extend(error.diagnostics) + continue entries.append(test_entry) warnings.extend(test_warnings) + if fixture_diagnostics: + raise PlannerInputError("Invalid SQL test fixtures:\n- " + "\n- ".join(fixture_diagnostics)) return entries, warnings diff --git a/src/sqlbuild/compiler/planner/_helpers/sql_tests/assembly.py b/src/sqlbuild/compiler/planner/_helpers/sql_tests/assembly.py index 62b766c4f..866548e57 100644 --- a/src/sqlbuild/compiler/planner/_helpers/sql_tests/assembly.py +++ b/src/sqlbuild/compiler/planner/_helpers/sql_tests/assembly.py @@ -37,6 +37,7 @@ uncommented_matches_by_pattern, uncommented_pattern_matches, ) +from sqlbuild.compiler.planner._helpers.sql_tests.fixture_validation import validate_test_fixtures from sqlbuild.compiler.planner.exceptions import PlannerInputError from sqlbuild.compiler.planner.models import ( ChainStep, @@ -125,6 +126,7 @@ def plan_test( project: CompiledProject, adapter: BaseAdapter, sql_analysis_enabled: bool = False, + validate_fixtures: bool = False, ) -> tuple[SqlTestPlanEntry, tuple[PlanWarning, ...]]: """Build a test plan entry with chained resolution.""" @@ -174,6 +176,17 @@ def plan_test( model_query_overrides=model_payload.model_query_overrides, mock_ref_names=frozenset(mock_refs), ) + if sql_analysis_enabled and validate_fixtures: + validate_test_fixtures( + test=test, + project=project, + adapter=adapter, + ordered_model_names=ordered_names, + mock_refs=mock_refs, + mock_sources=mock_sources, + mock_seeds=mock_seeds, + expected_outputs=expected_map, + ) warnings: list[PlanWarning] = [] reachable_mocks: set[str] = set() @@ -325,6 +338,10 @@ def plan_test( case_fingerprint=test.case_fingerprint, parameter_schema=test.parameter_schema, parameter_values=test.parameter_values, + mock_ref_names=tuple(sorted(mock_refs)), + mock_source_names=tuple(sorted(mock_sources)), + mock_seed_names=tuple(sorted(mock_seeds)), + mock_dbt_ref_names=tuple(sorted(mock_dbt_refs)), chain=tuple(chain_steps), assertions=assertion_steps, scope_deps=test.scope_deps, @@ -990,16 +1007,13 @@ def _extract_expected_ctes( """Build expected model name to CTE SQL body mapping.""" result: dict[str, str] = {} - pattern: re.Pattern[str] = re.compile( - r"(__expected__\w+)\s+AS\s*\(((?:[^()]*|\((?:[^()]*|\([^()]*\))*\))*)\)", - re.IGNORECASE, - ) - match: re.Match[str] - for match in pattern.finditer(test.sql_body): - cte_name: str = match.group(1) + if not isinstance(test.payload, CompiledModelSqlTestPayload): + return result + cte: CompileSqlTestCte + for cte in test.payload.expected_ctes: + cte_name: str = cte.name model_name: str = cte_name.removeprefix(EXPECTED_TEST_CTE_PREFIX) - cte_body: str = match.group(2).strip() - result[model_name] = cte_body + result[model_name] = cte.sql_body return result diff --git a/src/sqlbuild/compiler/planner/_helpers/sql_tests/fixture_validation.py b/src/sqlbuild/compiler/planner/_helpers/sql_tests/fixture_validation.py new file mode 100644 index 000000000..9770d1ca5 --- /dev/null +++ b/src/sqlbuild/compiler/planner/_helpers/sql_tests/fixture_validation.py @@ -0,0 +1,202 @@ +"""Static SQL-test fixture shape validation.""" + +from __future__ import annotations + +import re + +from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter +from sqlbuild.compiler.compile.main._infer_fixture_columns import infer_fixture_columns +from sqlbuild.compiler.compile.models import ( + CompiledModel, + CompiledProject, + CompiledSqlTest, + InferredColumn, +) +from sqlbuild.compiler.compile.types import CompiledResourceType +from sqlbuild.compiler.lineage.types import ColumnTransformKind +from sqlbuild.compiler.planner.exceptions import SqlTestFixtureValidationError + +_COLLECTION_TYPES: frozenset[str] = frozenset( + {"ARRAY", "LIST", "MAP", "OBJECT", "STRUCT", "VARIANT"} +) + + +def validate_test_fixtures( + *, + test: CompiledSqlTest, + project: CompiledProject, + adapter: BaseAdapter, + ordered_model_names: tuple[str, ...], + mock_refs: dict[str, str], + mock_sources: dict[str, str], + mock_seeds: dict[str, str], + expected_outputs: dict[str, str], +) -> None: + """Report all statically provable missing columns and collection-type mismatches.""" + + model_map: dict[str, CompiledModel] = {model.name: model for model in project.models} + inferred_by_fixture: dict[tuple[CompiledResourceType, str], tuple[InferredColumn, ...]] = {} + fixture_groups: tuple[tuple[CompiledResourceType, dict[str, str]], ...] = ( + (CompiledResourceType.MODEL, mock_refs), + (CompiledResourceType.SOURCE, mock_sources), + (CompiledResourceType.SEED, mock_seeds), + ) + for resource_type, fixtures in fixture_groups: + for name, sql in fixtures.items(): + inferred: tuple[InferredColumn, ...] | None = infer_fixture_columns( + query_sql=sql, + inference_profile=adapter.expression_inference_profile(), + ) + if inferred is not None: + inferred_by_fixture[(resource_type, name)] = inferred + + required: dict[tuple[CompiledResourceType, str], dict[str, set[str]]] = {} + for model_name in ordered_model_names: + model: CompiledModel | None = model_map.get(model_name) + if model is None: + continue + for lineage_column in model.fast_lineage_columns or (): + for upstream in lineage_column.upstream_columns: + key: tuple[CompiledResourceType, str] = ( + CompiledResourceType(upstream.resource_type), + upstream.resource_name, + ) + if key not in inferred_by_fixture: + continue + required.setdefault(key, {}).setdefault(upstream.column_name, set()).add(model_name) + + errors: list[str] = [] + for key, required_columns in sorted( + required.items(), key=lambda item: (item[0][0], item[0][1]) + ): + available: dict[str, InferredColumn] = { + column.name.casefold(): column for column in inferred_by_fixture[key] + } + missing: tuple[str, ...] = tuple( + sorted(name for name in required_columns if name.casefold() not in available) + ) + if missing: + model_names: set[str] = set() + for name in missing: + model_names.update(required_columns[name]) + models: tuple[str, ...] = tuple(sorted(model_names)) + errors.append( + f"{_fixture_location(test=test, resource_type=key[0], name=key[1])}: " + f"mock {key[0].value} '{key[1]}' is missing required columns: " + f"{', '.join(missing)} (read by: {', '.join(models)})" + ) + + expected_types: dict[tuple[CompiledResourceType, str], dict[str, str]] = {} + for model_name, model in model_map.items(): + columns: tuple[InferredColumn, ...] = model.inferred_columns or () + expected_types[(CompiledResourceType.MODEL, model_name)] = { + column.name.casefold(): column.type for column in columns if column.type is not None + } + if model.schema_entry is not None: + expected_types[(CompiledResourceType.MODEL, model_name)].update( + { + column.name.casefold(): column.type + for column in model.schema_entry.columns + if column.type is not None + } + ) + for source in project.sources: + expected_types[(CompiledResourceType.SOURCE, source.name)] = { + column.name.casefold(): column.type + for column in source.source_entry.columns + if column.type is not None + } + for seed in project.seeds: + expected_types[(CompiledResourceType.SEED, seed.name)] = { + column.name.casefold(): column.type + for column in seed.schema_entry.columns + if column.type is not None + } + for model_name in ordered_model_names: + model: CompiledModel | None = model_map.get(model_name) + if model is None: + continue + model_types: dict[str, str] = expected_types[(CompiledResourceType.MODEL, model_name)] + for lineage_column in model.fast_lineage_columns or (): + if lineage_column.transform_kind != ColumnTransformKind.DIRECT: + continue + upstream_types: set[str] = { + upstream_type + for upstream in lineage_column.upstream_columns + if ( + upstream_type := expected_types.get( + (CompiledResourceType(upstream.resource_type), upstream.resource_name), {} + ).get(upstream.column_name.casefold()) + ) + } + if len(upstream_types) == 1: + model_types.setdefault( + lineage_column.output_column.casefold(), upstream_types.pop() + ) + type_fixture_groups: tuple[tuple[CompiledResourceType, dict[str, str]], ...] = ( + *fixture_groups, + (CompiledResourceType.SQL_TEST, expected_outputs), + ) + for resource_type, fixtures in type_fixture_groups: + for name, sql in fixtures.items(): + inferred_columns: tuple[InferredColumn, ...] | None + if resource_type == CompiledResourceType.SQL_TEST: + inferred_columns = infer_fixture_columns( + query_sql=sql, + inference_profile=adapter.expression_inference_profile(), + ) + else: + inferred_columns = inferred_by_fixture.get((resource_type, name)) + if inferred_columns is None: + continue + type_key: tuple[CompiledResourceType, str] = ( + CompiledResourceType.MODEL + if resource_type == CompiledResourceType.SQL_TEST + else resource_type, + name, + ) + for column in inferred_columns: + expected_type: str | None = expected_types.get(type_key, {}).get( + column.name.casefold() + ) + if not _types_are_incompatible(expected=expected_type, inferred=column.type): + continue + fixture_label: str = ( + "expected output" + if resource_type == CompiledResourceType.SQL_TEST + else f"mock {resource_type.value}" + ) + errors.append( + f"{_fixture_location(test=test, resource_type=resource_type, name=name)}: " + f"{fixture_label} '{name}' column '{column.name}' has incompatible type " + f"{column.type}; resource type is {expected_type}. Use an explicit CAST, " + "ARRAY_CONSTRUCT, or PARSE_JSON as appropriate" + ) + if errors: + raise SqlTestFixtureValidationError( + tuple(f"SQL test '{test.name}': {diagnostic}" for diagnostic in sorted(set(errors))) + ) + + +def _fixture_location( + *, test: CompiledSqlTest, resource_type: CompiledResourceType, name: str +) -> str: + prefix: str = { + CompiledResourceType.MODEL: "__ref__", + CompiledResourceType.SOURCE: "__source__", + CompiledResourceType.SEED: "__seed__", + CompiledResourceType.SQL_TEST: "__expected__", + }[resource_type] + match: re.Match[str] | None = re.search( + rf"(?im)^\s*{re.escape(prefix + name)}\s+AS\s*\(", test.test_file.contents + ) + line: int = 1 if match is None else test.test_file.contents.count("\n", 0, match.start()) + 1 + return f"{test.test_file.relative_path.as_posix()}:{line}" + + +def _types_are_incompatible(*, expected: str | None, inferred: str | None) -> bool: + if expected is None or inferred is None: + return False + expected_base: str = expected.upper().split("(", maxsplit=1)[0].strip() + inferred_base: str = inferred.upper().split("(", maxsplit=1)[0].strip() + return (expected_base in _COLLECTION_TYPES) != (inferred_base in _COLLECTION_TYPES) diff --git a/src/sqlbuild/compiler/planner/exceptions.py b/src/sqlbuild/compiler/planner/exceptions.py index c0e7dacda..743f2121e 100644 --- a/src/sqlbuild/compiler/planner/exceptions.py +++ b/src/sqlbuild/compiler/planner/exceptions.py @@ -19,6 +19,14 @@ def __init__(self, message: str, *, code: str | None = None, help: str | None = self.help = help +class SqlTestFixtureValidationError(PlannerInputError): + """Raised with every invalid fixture diagnostic for one SQL test.""" + + def __init__(self, diagnostics: tuple[str, ...]) -> None: + self.diagnostics = diagnostics + super().__init__("\n- ".join(diagnostics)) + + class FutureCursorSafetyError(PlannerInputError): """Raised when an effective cursor exceeds the configured future limit.""" diff --git a/src/sqlbuild/compiler/planner/main/commands/sql_test.py b/src/sqlbuild/compiler/planner/main/commands/sql_test.py index 18105a190..20f767e92 100644 --- a/src/sqlbuild/compiler/planner/main/commands/sql_test.py +++ b/src/sqlbuild/compiler/planner/main/commands/sql_test.py @@ -12,9 +12,14 @@ def build_test_command_plan( adapter: BaseAdapter, scope: PlannerScope, relations: PlannerRelationsContext, + case_name: str | None = None, ) -> PlanOutput: """Project selected SQL tests from canonical static state.""" return build_test_command_plan_impl( - project=project, adapter=adapter, scope=scope, relations=relations + project=project, + adapter=adapter, + scope=scope, + relations=relations, + case_name=case_name, ) diff --git a/src/sqlbuild/compiler/planner/models.py b/src/sqlbuild/compiler/planner/models.py index b6542b361..abd3019c1 100644 --- a/src/sqlbuild/compiler/planner/models.py +++ b/src/sqlbuild/compiler/planner/models.py @@ -1124,6 +1124,10 @@ class SqlTestPlanEntry: case_fingerprint: str | None = None parameter_schema: tuple[SqlTestParameterDeclaration, ...] = field(default_factory=tuple) parameter_values: tuple[tuple[str, SqlValue], ...] = field(default_factory=tuple) + mock_ref_names: tuple[str, ...] = field(default_factory=tuple) + mock_source_names: tuple[str, ...] = field(default_factory=tuple) + mock_seed_names: tuple[str, ...] = field(default_factory=tuple) + mock_dbt_ref_names: tuple[str, ...] = field(default_factory=tuple) chain: tuple[ChainStep, ...] = field(default_factory=tuple) assertions: tuple[SqlTestAssertionStep, ...] = field(default_factory=tuple) scope_deps: tuple[CompiledObjectKey, ...] = field(default_factory=tuple) @@ -1263,6 +1267,7 @@ class PlanOutput: function_entries: tuple[FunctionPlanEntry, ...] = field(default_factory=tuple) audit_entries: tuple[AuditPlanEntry, ...] = field(default_factory=tuple) test_entries: tuple[SqlTestPlanEntry, ...] = field(default_factory=tuple) + available_test_case_names: tuple[str, ...] = field(default_factory=tuple) selected_keys: frozenset[CompiledObjectKey] = field(default_factory=frozenset) warnings: tuple[PlanWarning, ...] = field(default_factory=tuple) retention_entries: tuple[RetentionPlanEntry, ...] = field(default_factory=tuple) diff --git a/src/sqlbuild/executor/testing/_helpers/difference_samples.py b/src/sqlbuild/executor/testing/_helpers/difference_samples.py index 660fd199b..7306b71d8 100644 --- a/src/sqlbuild/executor/testing/_helpers/difference_samples.py +++ b/src/sqlbuild/executor/testing/_helpers/difference_samples.py @@ -13,12 +13,17 @@ from sqlbuild.diagnostics.classes.diagnostic_record_redactor import DiagnosticRecordRedactor from sqlbuild.diagnostics.main.log_debug_event import log_debug_event from sqlbuild.executor.testing._helpers.comparison_sql import format_sql, lift_step_ctes -from sqlbuild.executor.testing.models import SqlTestDifferenceSample, StepResult +from sqlbuild.executor.testing.models import ( + SqlTestColumnDifference, + SqlTestDifferenceSample, + StepResult, +) from sqlbuild.executor.testing.types import SqlTestDifferenceDirection, SqlTestOutcome _ROW_LIMIT: Final[int] = 3 _COLUMN_LIMIT: Final[int] = 12 _VALUE_LIMIT: Final[int] = 120 +_OMITTED_COLUMN_NAME: Final[str] = "..." _LOGGER: logging.Logger = logging.getLogger("sqlbuild.execution") @@ -63,11 +68,41 @@ def add_difference_samples( step_result, unexpected_samples=unexpected_samples, missing_samples=missing_samples, + column_differences=_align_single_difference( + step_result=step_result, + unexpected_samples=unexpected_samples, + missing_samples=missing_samples, + ), ) ) return sampled_results +def _align_single_difference( + *, + step_result: StepResult, + unexpected_samples: tuple[SqlTestDifferenceSample, ...], + missing_samples: tuple[SqlTestDifferenceSample, ...], +) -> tuple[SqlTestColumnDifference, ...]: + """Align values only when each set-difference direction contains exactly one row.""" + + if step_result.unexpected_row_count != 1 or step_result.missing_row_count != 1: + return () + if len(unexpected_samples) != 1 or len(missing_samples) != 1: + return () + actual_values: tuple[tuple[str, str], ...] = unexpected_samples[0].values + expected_values: tuple[tuple[str, str], ...] = missing_samples[0].values + if any(name == _OMITTED_COLUMN_NAME for name, _ in (*actual_values, *expected_values)): + return () + if tuple(name for name, _ in actual_values) != tuple(name for name, _ in expected_values): + return () + return tuple( + SqlTestColumnDifference(name=name, actual=actual, expected=expected) + for (name, actual), (_, expected) in zip(actual_values, expected_values, strict=True) + if actual != expected + ) + + def build_sql_test_difference_sample_sql( *, test_entry: SqlTestPlanEntry, @@ -145,7 +180,7 @@ def _fetch_difference_samples( ] omitted_columns: int = max(0, len(column_names) - _COLUMN_LIMIT) if omitted_columns: - values.append(("...", f"{omitted_columns} columns omitted")) + values.append((_OMITTED_COLUMN_NAME, f"{omitted_columns} columns omitted")) samples.append(SqlTestDifferenceSample(values=tuple(values))) return tuple(samples) diff --git a/src/sqlbuild/executor/testing/models.py b/src/sqlbuild/executor/testing/models.py index e51954d56..9935ec741 100644 --- a/src/sqlbuild/executor/testing/models.py +++ b/src/sqlbuild/executor/testing/models.py @@ -17,6 +17,15 @@ class SqlTestDifferenceSample: values: tuple[tuple[str, str], ...] = field(default_factory=tuple) +@dataclass(frozen=True) +class SqlTestColumnDifference: + """One safely aligned, redacted actual and expected column value.""" + + name: str + actual: str + expected: str + + @dataclass(frozen=True) class StepResult: """Outcome of one chain step in a SQL unit test.""" @@ -30,6 +39,7 @@ class StepResult: missing_row_count: int | None = None unexpected_samples: tuple[SqlTestDifferenceSample, ...] = field(default_factory=tuple) missing_samples: tuple[SqlTestDifferenceSample, ...] = field(default_factory=tuple) + column_differences: tuple[SqlTestColumnDifference, ...] = field(default_factory=tuple) error_code: str | None = None error_help: str | None = None error_message: str | None = None diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/test/_test_types.py b/tests/e2e/src/sqlbuild/cli/commands/main/test/_test_types.py index 23f90f582..615d258f0 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/test/_test_types.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/test/_test_types.py @@ -24,3 +24,61 @@ class SqlAnalysisChainSqlTestE2ETestCase: sql_analysis_enabled: bool expected_artifact_fragments: tuple[str, ...] unexpected_artifact_fragments: tuple[str, ...] + + +@dataclass(frozen=True) +class ParameterCaseSelectionE2ETestCase: + """Test case for one parameterized SQL-test case selection.""" + + description: str + case_name: str + expected_exit_code: int + expected_stdout_fragments: tuple[str, ...] = field(default_factory=tuple) + expected_stderr_fragments: tuple[str, ...] = field(default_factory=tuple) + + +@dataclass(frozen=True) +class SqlTestPlanInspectionE2ETestCase: + """Test case for offline SQL-test plan inspection.""" + + description: str + repo_files: dict[str, str] + expected_stdout_fragments: tuple[str, ...] + expected_exit_code: int = 0 + + +@dataclass(frozen=True) +class SqlTestFixtureValidationE2ETestCase: + """Test case for static SQL-test fixture validation.""" + + description: str + repo_files: dict[str, str] + expected_stderr_fragments: tuple[str, ...] + + +@dataclass(frozen=True) +class ComplexValuesFixtureE2ETestCase: + """Test case for complex string literals inside VALUES fixtures.""" + + description: str + adapter_name: str + command: tuple[str, ...] + expected_stdout_fragment: str + + +@dataclass(frozen=True) +class FixtureCompatibilityE2ETestCase: + """Test case for valid fixture shapes that require conservative analysis.""" + + description: str + repo_files: dict[str, str] + expected_stdout_fragment: str + + +@dataclass(frozen=True) +class SqlTestInspectConflictE2ETestCase: + """Test case for mutually exclusive inspection output modes.""" + + description: str + expected_exit_code: int + expected_stderr_fragment: str diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/test/helpers.py b/tests/e2e/src/sqlbuild/cli/commands/main/test/helpers.py index 73610c9aa..465a74979 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/test/helpers.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/test/helpers.py @@ -67,6 +67,267 @@ def build_chain_test_project_files(*, sql_analysis_enabled: bool) -> dict[str, s } +def build_parameterized_test_project_files() -> dict[str, str]: + """Build an inline project with two independently selectable test cases.""" + + return { + "sqlbuild_project.toml": ( + 'name = "parameter_case_demo"\n' + 'adapter = "duckdb"\n\n' + "[connection]\n" + 'database = "parameter_case_demo.duckdb"\n' + ), + "models/orders.sql": 'MODEL ();\n\nSELECT status FROM __source("raw_orders")\n', + "models/customers.sql": ('MODEL ();\n\nSELECT status FROM __source("raw_orders")\n'), + "sources/raw_orders.yml": ( + "sources:\n - name: raw_orders\n schema: main\n table: raw_orders\n" + ), + "tests/unit/test_orders.sql": ( + 'TEST (name "order_status", parameters (status string), cases (' + 'open_case (status "open"), closed_case (status "closed")));\n\n' + "WITH\n" + '__source__raw_orders AS (SELECT @param("status") AS status),\n' + '__expected__orders AS (SELECT @param("status") AS status)\n' + "SELECT 1\n" + ), + "tests/unit/test_customers.sql": ( + 'TEST (name "customer_status", parameters (status string), ' + 'cases (customer_only (status "active")));\n\n' + "WITH\n" + '__source__raw_orders AS (SELECT @param("status") AS status),\n' + '__expected__customers AS (SELECT @param("status") AS status)\n' + "SELECT 1\n" + ), + } + + +def build_mock_boundary_test_project_files() -> dict[str, str]: + """Build a test whose ref mock intentionally replaces one real model.""" + + return { + "sqlbuild_project.toml": ( + 'name = "mock_boundary_demo"\n' + 'adapter = "duckdb"\n\n' + "[connection]\n" + 'database = "mock_boundary_demo.duckdb"\n' + ), + "models/stg_orders.sql": "MODEL ();\n\nSELECT 1 AS order_id\n", + "models/int_orders.sql": ('MODEL ();\n\nSELECT order_id FROM __ref("stg_orders")\n'), + "models/fact_orders.sql": ('MODEL ();\n\nSELECT order_id FROM __ref("int_orders")\n'), + "tests/unit/test_fact_orders.sql": ( + "TEST();\n\n" + "WITH\n" + "__ref__stg_orders AS (SELECT 1 AS order_id),\n" + "__expected__fact_orders AS (SELECT 1 AS order_id)\n" + "SELECT 1\n" + ), + } + + +def build_unsatisfied_leaf_test_project_files() -> dict[str, str]: + """Build a test that omits the source mock required by its real model chain.""" + + files: dict[str, str] = build_chain_test_project_files(sql_analysis_enabled=True) + files["sources/raw.yml"] = ( + "sources:\n" + " - name: raw\n schema: main\n table: raw\n" + " - name: unused\n schema: main\n table: unused\n" + ) + files["tests/unit/test_chain.sql"] = ( + "TEST();\n\n" + "WITH\n" + "__source__unused AS (SELECT 1 AS id),\n" + "__expected__fact_orders AS (\n" + " SELECT 1 AS id, 101 AS adjusted, 'US' AS country, " + "'literal' AS literal_text, 'active' AS status\n" + ")\n" + "SELECT 1\n" + ) + return files + + +def build_missing_mock_columns_project_files() -> dict[str, str]: + """Build a test whose source fixture omits two statically required columns.""" + + return { + "sqlbuild_project.toml": ( + 'name = "missing_fixture_columns"\n' + 'adapter = "duckdb"\n\n' + "[connection]\n" + 'database = "missing_fixture_columns.duckdb"\n' + ), + "models/orders.sql": ( + 'MODEL ();\n\nSELECT customer_id, status FROM __source("raw_orders")\n' + ), + "sources/raw_orders.yml": ( + "sources:\n" + " - name: raw_orders\n" + " schema: main\n" + " table: raw_orders\n" + " columns:\n" + " - name: customer_id\n type: INTEGER\n" + " - name: status\n type: VARCHAR\n" + ), + "tests/unit/test_orders.sql": ( + "TEST();\n\n" + "WITH\n" + "__source__raw_orders AS (SELECT 1 AS order_id),\n" + "__expected__orders AS (SELECT 100 AS customer_id, 'open' AS status)\n" + "SELECT 1\n" + ), + } + + +def build_incompatible_fixture_type_project_files(*, adapter_name: str) -> dict[str, str]: + """Build a test whose expected scalar conflicts with an array model column.""" + + array_expression: str = { + "duckdb": "[1, 2]", + "snowflake": "ARRAY_CONSTRUCT(1, 2)", + }[adapter_name] + return { + "sqlbuild_project.toml": ( + 'name = "incompatible_fixture_type"\n' + f'adapter = "{adapter_name}"\n\n' + "[connection]\n" + 'database = "incompatible_fixture_type.duckdb"\n' + 'schema = "main"\n' + ), + "models/orders.sql": ( + 'MODEL (database "fixture_db", schema "main");\n\n' + 'SELECT item_ids FROM __source("raw_orders")\n' + ), + "sources/raw_orders.yml": ( + "sources:\n" + " - name: raw_orders\n" + " schema: main\n" + " table: raw_orders\n" + " columns:\n" + " - name: item_ids\n type: ARRAY\n" + ), + "tests/unit/test_orders.sql": ( + "TEST();\n\n" + "WITH\n" + f"__source__raw_orders AS (SELECT {array_expression} AS item_ids),\n" + "__expected__orders AS (SELECT '1,2' AS item_ids)\n" + "SELECT 1\n" + ), + } + + +def build_complex_values_fixture_project_files(*, adapter_name: str) -> dict[str, str]: + """Build a minimal complex-string VALUES fixture for one adapter dialect.""" + + return { + "sqlbuild_project.toml": ( + 'name = "complex_values_fixture"\n' + f'adapter = "{adapter_name}"\n\n' + "[connection]\n" + 'database = "complex_values_fixture.duckdb"\n' + 'schema = "main"\n' + ), + "models/orders.sql": ( + 'MODEL (database "fixture_db", schema "main");\n\n' + 'SELECT order_id, mapping_text FROM __source("raw_orders")\n' + ), + "sources/raw_orders.yml": ( + "sources:\n" + " - name: raw_orders\n" + " schema: main\n" + " table: raw_orders\n" + " columns:\n" + " - name: order_id\n type: INTEGER\n" + " - name: mapping_text\n type: VARCHAR\n" + ), + "tests/unit/test_orders.sql": ( + "TEST();\n\n" + "WITH\n" + "__source__raw_orders AS (\n" + " SELECT order_id, mapping_text FROM (VALUES\n" + " (1, 'alpha,beta [one] (two)'),\n" + " (2, 'gamma(delta),[epsilon]')\n" + ") AS fixture(order_id, mapping_text)\n" + "),\n" + "__expected__orders AS (\n" + " SELECT order_id, mapping_text FROM (VALUES\n" + " (1, 'alpha,beta [one] (two)'),\n" + " (2, 'gamma(delta),[epsilon]')\n" + ") AS fixture(order_id, mapping_text)\n" + ")\n" + "SELECT 1\n" + ), + } + + +def build_star_mock_fixture_project_files() -> dict[str, str]: + """Build a valid source fixture whose columns come from a star projection.""" + + files: dict[str, str] = build_missing_mock_columns_project_files() + files["tests/unit/test_orders.sql"] = ( + "TEST();\n\n" + "WITH\n" + "__source__raw_orders AS (\n" + " SELECT * FROM (VALUES (100, 'open')) AS fixture(customer_id, status)\n" + "),\n" + "__expected__orders AS (SELECT 100 AS customer_id, 'open' AS status)\n" + "SELECT 1\n" + ) + return files + + +def build_transformed_collection_project_files() -> dict[str, str]: + """Build a valid aggregation that changes a scalar input into a collection output.""" + + return { + "sqlbuild_project.toml": ( + 'name = "transformed_collection"\n' + 'adapter = "duckdb"\n\n' + "[connection]\n" + 'database = "transformed_collection.duckdb"\n' + ), + "models/order_statuses.sql": ( + 'MODEL ();\n\nSELECT list(status) AS statuses FROM __source("raw_orders")\n' + ), + "sources/raw_orders.yml": ( + "sources:\n" + " - name: raw_orders\n" + " schema: main\n" + " table: raw_orders\n" + " columns:\n" + " - name: status\n type: VARCHAR\n" + ), + "tests/unit/test_order_statuses.sql": ( + "TEST();\n\n" + "WITH\n" + "__source__raw_orders AS (SELECT 'open' AS status),\n" + "__expected__order_statuses AS (SELECT ['open'] AS statuses)\n" + "SELECT 1\n" + ), + } + + +def build_multiple_invalid_fixtures_project_files() -> dict[str, str]: + """Build two invalid tests so planning must aggregate both diagnostics.""" + + files: dict[str, str] = build_missing_mock_columns_project_files() + del files["tests/unit/test_orders.sql"] + files["tests/unit/test_orders_a.sql"] = ( + "TEST();\n\n" + "WITH\n" + "__source__raw_orders AS (SELECT 'open' AS status),\n" + "__expected__orders AS (SELECT 100 AS customer_id, 'open' AS status)\n" + "SELECT 1\n" + ) + files["tests/unit/test_orders_b.sql"] = ( + "TEST();\n\n" + "WITH\n" + "__source__raw_orders AS (SELECT 100 AS customer_id),\n" + "__expected__orders AS (SELECT 100 AS customer_id, 'open' AS status)\n" + "SELECT 1\n" + ) + return files + + def build_assertion_test_project_files(*, failing: bool) -> dict[str, str]: """Build an inline project with a SQL unit-test zero-row assertion.""" diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/test/test_test.py b/tests/e2e/src/sqlbuild/cli/commands/main/test/test_test.py index 73b2bd510..6492d83d5 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/test/test_test.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/test/test_test.py @@ -8,13 +8,28 @@ import pytest from tests.e2e.src.sqlbuild.cli.commands.main.test._test_types import ( + ComplexValuesFixtureE2ETestCase, + FixtureCompatibilityE2ETestCase, + ParameterCaseSelectionE2ETestCase, SqlAnalysisChainSqlTestE2ETestCase, SqlTestE2ETestCase, + SqlTestFixtureValidationE2ETestCase, + SqlTestInspectConflictE2ETestCase, + SqlTestPlanInspectionE2ETestCase, ) from tests.e2e.src.sqlbuild.cli.commands.main.test.helpers import ( build_assertion_test_project_files, build_chain_test_project_files, + build_complex_values_fixture_project_files, + build_incompatible_fixture_type_project_files, build_macro_test_project_files, + build_missing_mock_columns_project_files, + build_mock_boundary_test_project_files, + build_multiple_invalid_fixtures_project_files, + build_parameterized_test_project_files, + build_star_mock_fixture_project_files, + build_transformed_collection_project_files, + build_unsatisfied_leaf_test_project_files, ) from tests.e2e.src.sqlbuild.cli.commands.shared.helpers import ( assert_fragments_in_order, @@ -24,6 +39,290 @@ ) +@pytest.mark.parametrize( + "test_case", + [ + SqlTestInspectConflictE2ETestCase( + description="inspect rejects structured output file", + expected_exit_code=2, + expected_stderr_fragment=( + "test --inspect cannot be combined with --json or --json-output" + ), + ), + ], + ids=lambda case: case.description, +) +def test_given_inspection_when_requesting_json_file_then_parser_rejects_conflict( + test_case: SqlTestInspectConflictE2ETestCase, + tmp_path: Path, +) -> None: + project_dir: Path = prepare_inline_project( + tmp_path=tmp_path, + project_name="inspect_output_conflict_project", + repo_files=build_parameterized_test_project_files(), + ) + + result: subprocess.CompletedProcess[str] = run_sqb( + command=("test", "--inspect", "--json-output", str(tmp_path / "result.json")), + project_dir=project_dir, + ) + + assert result.returncode == test_case.expected_exit_code + assert test_case.expected_stderr_fragment in result.stderr + + +@pytest.mark.parametrize( + "test_case", + [ + FixtureCompatibilityE2ETestCase( + description="star mock fixture defers static shape validation", + repo_files=build_star_mock_fixture_project_files(), + expected_stdout_fragment="PASS=1", + ), + FixtureCompatibilityE2ETestCase( + description="aggregation output does not inherit scalar input type", + repo_files=build_transformed_collection_project_files(), + expected_stdout_fragment="PASS=1", + ), + ], + ids=lambda case: case.description, +) +def test_given_valid_fixture_when_static_inference_is_ambiguous_then_test_still_executes( + test_case: FixtureCompatibilityE2ETestCase, + tmp_path: Path, +) -> None: + project_dir: Path = prepare_inline_project( + tmp_path=tmp_path, + project_name="fixture_compatibility_project", + repo_files=test_case.repo_files, + ) + + result: subprocess.CompletedProcess[str] = run_sqb( + command=("--no-color", "test"), + project_dir=project_dir, + ) + + assert result.returncode == 0, result.stdout + result.stderr + assert test_case.expected_stdout_fragment in result.stdout + + +@pytest.mark.parametrize( + "test_case", + [ + ComplexValuesFixtureE2ETestCase( + description="duckdb executes complex string values", + adapter_name="duckdb", + command=("--no-color", "test"), + expected_stdout_fragment="PASS=1", + ), + ComplexValuesFixtureE2ETestCase( + description="snowflake compiles complex string values unchanged", + adapter_name="snowflake", + command=("--no-color", "compile"), + expected_stdout_fragment="Project compiled", + ), + ], + ids=lambda case: case.description, +) +def test_given_complex_strings_in_values_when_processing_then_literals_remain_intact( + test_case: ComplexValuesFixtureE2ETestCase, + tmp_path: Path, +) -> None: + project_dir: Path = prepare_inline_project( + tmp_path=tmp_path, + project_name="complex_values_project", + repo_files=build_complex_values_fixture_project_files(adapter_name=test_case.adapter_name), + ) + + result: subprocess.CompletedProcess[str] = run_sqb( + command=test_case.command, + project_dir=project_dir, + ) + + assert result.returncode == 0, result.stdout + result.stderr + assert test_case.expected_stdout_fragment in result.stdout + artifacts: tuple[Path, ...] = tuple((project_dir / "target").glob("**/test_orders.sql")) + assert artifacts + artifact_sql: str = artifacts[0].read_text(encoding="utf-8") + assert "alpha,beta [one] (two)" in artifact_sql + assert "gamma(delta),[epsilon]" in artifact_sql + + +@pytest.mark.parametrize( + "test_case", + [ + SqlTestFixtureValidationE2ETestCase( + description="missing source columns are reported together", + repo_files=build_missing_mock_columns_project_files(), + expected_stderr_fragments=( + "tests/unit/test_orders.sql:4", + "mock source 'raw_orders' is missing required columns: customer_id, status", + "read by: orders", + ), + ), + SqlTestFixtureValidationE2ETestCase( + description="duckdb array and scalar types identify the expected column", + repo_files=build_incompatible_fixture_type_project_files(adapter_name="duckdb"), + expected_stderr_fragments=( + "tests/unit/test_orders.sql:5", + "expected output 'orders' column 'item_ids' has incompatible type VARCHAR", + "resource type is ARRAY", + "ARRAY_CONSTRUCT, or PARSE_JSON", + ), + ), + SqlTestFixtureValidationE2ETestCase( + description="snowflake array and scalar types identify the expected column", + repo_files=build_incompatible_fixture_type_project_files(adapter_name="snowflake"), + expected_stderr_fragments=( + "tests/unit/test_orders.sql:5", + "expected output 'orders' column 'item_ids' has incompatible type VARCHAR", + "resource type is ARRAY", + "ARRAY_CONSTRUCT, or PARSE_JSON", + ), + ), + SqlTestFixtureValidationE2ETestCase( + description="invalid fixtures are aggregated across selected tests", + repo_files=build_multiple_invalid_fixtures_project_files(), + expected_stderr_fragments=( + "SQL test 'test_orders_a'", + "tests/unit/test_orders_a.sql:4", + "missing required columns: customer_id", + "SQL test 'test_orders_b'", + "tests/unit/test_orders_b.sql:4", + "missing required columns: status", + ), + ), + ], + ids=lambda case: case.description, +) +def test_given_invalid_fixture_when_testing_then_static_diagnostics_prevent_connection( + test_case: SqlTestFixtureValidationE2ETestCase, + tmp_path: Path, +) -> None: + project_dir: Path = prepare_inline_project( + tmp_path=tmp_path, + project_name="fixture_validation_project", + repo_files=test_case.repo_files, + ) + + result: subprocess.CompletedProcess[str] = run_sqb( + command=("--no-color", "test"), + project_dir=project_dir, + ) + + assert result.returncode == 1, result.stdout + result.stderr + for fragment in test_case.expected_stderr_fragments: + assert fragment in result.stderr, result.stdout + result.stderr + assert "Connecting to" not in result.stdout + + +@pytest.mark.parametrize( + "test_case", + [ + SqlTestPlanInspectionE2ETestCase( + description="multi model plan shows real chain", + repo_files=build_chain_test_project_files(sql_analysis_enabled=True), + expected_stdout_fragments=( + "Resolved test plan", + "mocked sources: raw", + "real models: stg_orders, fact_orders", + "expected models: stg_orders, fact_orders", + "Test plan inspection complete: 1 selected, 0 errors.", + ), + ), + SqlTestPlanInspectionE2ETestCase( + description="missing leaf mock is reported before execution", + repo_files=build_unsatisfied_leaf_test_project_files(), + expected_exit_code=1, + expected_stdout_fragments=( + "model 'stg_orders' references __source('raw') which has no mock", + "model 'fact_orders' references __source('raw') which has no mock", + "Test plan inspection failed: 1 selected, 2 errors.", + ), + ), + SqlTestPlanInspectionE2ETestCase( + description="ref mock shows replacement boundary", + repo_files=build_mock_boundary_test_project_files(), + expected_stdout_fragments=( + "mocked refs: stg_orders", + "real models: int_orders, fact_orders", + "boundary: stg_orders is replaced by __ref__stg_orders", + ), + ), + ], + ids=lambda case: case.description, +) +def test_given_sql_test_when_inspecting_then_fixture_boundaries_and_chain_are_shown( + test_case: SqlTestPlanInspectionE2ETestCase, + tmp_path: Path, +) -> None: + project_dir: Path = prepare_inline_project( + tmp_path=tmp_path, + project_name="test_plan_inspection_project", + repo_files=test_case.repo_files, + ) + + result: subprocess.CompletedProcess[str] = run_sqb( + command=("--no-color", "test", "--inspect"), + project_dir=project_dir, + ) + + assert result.returncode == test_case.expected_exit_code, result.stdout + result.stderr + for fragment in test_case.expected_stdout_fragments: + assert fragment in result.stdout, result.stdout + result.stderr + assert "Connecting to" not in result.stdout + + +@pytest.mark.parametrize( + "test_case", + [ + ParameterCaseSelectionE2ETestCase( + description="named case runs without sibling cases", + case_name="open_case", + expected_exit_code=0, + expected_stdout_fragments=("Test ready 1 selected", "PASS=1", "open_case"), + ), + ParameterCaseSelectionE2ETestCase( + description="unknown case reports available selected cases", + case_name="missing_case", + expected_exit_code=1, + expected_stderr_fragments=( + "SQL test case 'missing_case' did not match the selected tests", + "available cases: closed_case, open_case", + ), + ), + ], + ids=lambda case: case.description, +) +def test_given_parameterized_test_when_selecting_case_then_only_named_case_runs( + test_case: ParameterCaseSelectionE2ETestCase, + tmp_path: Path, +) -> None: + project_dir: Path = prepare_inline_project( + tmp_path=tmp_path, + project_name="parameter_case_project", + repo_files=build_parameterized_test_project_files(), + ) + + result: subprocess.CompletedProcess[str] = run_sqb( + command=( + "--no-color", + "test", + "--select", + "orders", + "--case", + test_case.case_name, + ), + project_dir=project_dir, + ) + + assert result.returncode == test_case.expected_exit_code, result.stdout + result.stderr + for fragment in test_case.expected_stdout_fragments: + assert fragment in result.stdout, result.stdout + result.stderr + for fragment in test_case.expected_stderr_fragments: + assert fragment in result.stderr, result.stdout + result.stderr + + @pytest.mark.parametrize( "test_case", [ diff --git a/tests/integration/src/sqlbuild/executor/testing/_test_types.py b/tests/integration/src/sqlbuild/executor/testing/_test_types.py index 69c99868d..43d3acc77 100644 --- a/tests/integration/src/sqlbuild/executor/testing/_test_types.py +++ b/tests/integration/src/sqlbuild/executor/testing/_test_types.py @@ -48,3 +48,4 @@ class SqlTestDiagnosticsTestCase: expected_sample_count: int expect_redaction: bool = False expect_truncation: bool = False + expected_column_differences: tuple[tuple[str, str, str], ...] = () diff --git a/tests/integration/src/sqlbuild/executor/testing/test_sql_test_execution.py b/tests/integration/src/sqlbuild/executor/testing/test_sql_test_execution.py index 9a43d45a4..3f638c963 100644 --- a/tests/integration/src/sqlbuild/executor/testing/test_sql_test_execution.py +++ b/tests/integration/src/sqlbuild/executor/testing/test_sql_test_execution.py @@ -14,7 +14,11 @@ ) from sqlbuild.executor.testing.main._execute import execute_sql_test from sqlbuild.executor.testing.main.comparison_sql import build_sql_test_comparison_sql -from sqlbuild.executor.testing.models import SqlTestExecutionResult, StepResult +from sqlbuild.executor.testing.models import ( + SqlTestColumnDifference, + SqlTestExecutionResult, + StepResult, +) from sqlbuild.executor.testing.types import SqlTestDifferenceDirection, SqlTestOutcome from tests.integration.src.sqlbuild.executor.testing._test_types import ( SqlTestComparisonSqlTestCase, @@ -367,6 +371,53 @@ def test_given_two_way_mismatch_when_executing_test_then_samples_are_bounded_and assert unexpected_values["detail"].endswith("...") == test_case.expect_truncation +@pytest.mark.parametrize( + "test_case", + ( + SqlTestDiagnosticsTestCase( + description="one row each way aligns changed columns", + chain_steps=( + ( + "orders", + "SELECT 1 AS id, 'actual' AS status", + "SELECT 1 AS id, 'expected' AS status", + ), + ), + expected_actual_count=1, + expected_expected_count=1, + expected_unexpected_count=1, + expected_missing_count=1, + expected_sample_count=1, + expected_column_differences=(("status", "actual", "expected"),), + ), + ), + ids=lambda case: case.description, +) +def test_given_one_row_each_way_when_executing_then_changed_columns_are_aligned( + test_case: SqlTestDiagnosticsTestCase, + adapter: DuckDbAdapter, + connection: Any, +) -> None: + entry: SqlTestPlanEntry = build_sql_test_plan_entry( + name="aligned_diagnostics", + chain_steps=test_case.chain_steps, + ) + + result: SqlTestExecutionResult = execute_sql_test( + test_entry=entry, + adapter=adapter, + connection=connection, + ) + + assert result.outcome == SqlTestOutcome.FAIL + assert len(result.step_results) == 1 + differences: tuple[SqlTestColumnDifference, ...] = result.step_results[0].column_differences + assert ( + tuple((value.name, value.actual, value.expected) for value in differences) + == test_case.expected_column_differences + ) + + @pytest.mark.parametrize( "test_case", ( diff --git a/tests/unit/src/sqlbuild/cli/output/_helpers/_test_types.py b/tests/unit/src/sqlbuild/cli/output/_helpers/_test_types.py index ce0aff13f..743edf479 100644 --- a/tests/unit/src/sqlbuild/cli/output/_helpers/_test_types.py +++ b/tests/unit/src/sqlbuild/cli/output/_helpers/_test_types.py @@ -38,6 +38,7 @@ class SqlTestDifferenceOutputTestCase: description: str expected_unexpected_count: int expected_missing_count: int + expected_column_differences: tuple[tuple[str, str, str], ...] @dataclass(frozen=True) diff --git a/tests/unit/src/sqlbuild/cli/output/_helpers/test_execution_result_document.py b/tests/unit/src/sqlbuild/cli/output/_helpers/test_execution_result_document.py index 8c46a5cec..d20dc6dec 100644 --- a/tests/unit/src/sqlbuild/cli/output/_helpers/test_execution_result_document.py +++ b/tests/unit/src/sqlbuild/cli/output/_helpers/test_execution_result_document.py @@ -44,6 +44,7 @@ from sqlbuild.executor.run.types import ExecutionPhase from sqlbuild.executor.scheduling.types import ExecutionStatus from sqlbuild.executor.testing.models import ( + SqlTestColumnDifference, SqlTestDifferenceSample, SqlTestExecutionResult, StepResult, @@ -102,6 +103,7 @@ def test_given_failed_model_when_formatting_json_then_last_recorded_sql_is_prese description="two-way samples remain structured", expected_unexpected_count=1, expected_missing_count=1, + expected_column_differences=(("status", "unexpected", "expected"),), ), ), ids=lambda case: case.description, @@ -118,6 +120,9 @@ def test_given_difference_samples_when_formatting_json_then_structured_diagnosti SqlTestDifferenceSample(values=(("id", "2"), ("status", "unexpected"))), ), missing_samples=(SqlTestDifferenceSample(values=(("id", "1"), ("status", "expected"))),), + column_differences=( + SqlTestColumnDifference(name="status", actual="unexpected", expected="expected"), + ), ) payload: dict[str, object] = _format_sql_test_step(step) @@ -130,6 +135,11 @@ def test_given_difference_samples_when_formatting_json_then_structured_diagnosti assert payload["missing_samples"] == [ [{"name": "id", "value": "1"}, {"name": "status", "value": "expected"}] ] + expected_column_differences: list[dict[str, str]] = [ + {"name": name, "actual": actual, "expected": expected} + for name, actual, expected in test_case.expected_column_differences + ] + assert payload["column_differences"] == expected_column_differences @pytest.mark.parametrize( diff --git a/tests/unit/src/sqlbuild/cli/progress/main/test_expectation_detail.py b/tests/unit/src/sqlbuild/cli/progress/main/test_expectation_detail.py index 9c8a97843..9eeb54bcf 100644 --- a/tests/unit/src/sqlbuild/cli/progress/main/test_expectation_detail.py +++ b/tests/unit/src/sqlbuild/cli/progress/main/test_expectation_detail.py @@ -3,7 +3,11 @@ import pytest from sqlbuild.cli.progress.main._expectation_detail import format_expectation_detail -from sqlbuild.executor.testing.models import SqlTestDifferenceSample, StepResult +from sqlbuild.executor.testing.models import ( + SqlTestColumnDifference, + SqlTestDifferenceSample, + StepResult, +) from sqlbuild.executor.testing.types import SqlTestOutcome from tests.unit.src.sqlbuild.cli.progress.main._test_types import ExpectationDetailTestCase @@ -27,6 +31,11 @@ missing_samples=( SqlTestDifferenceSample(values=(("id", "1"), ("status", "expected"))), ), + column_differences=( + SqlTestColumnDifference( + name="status", actual="unexpected", expected="expected" + ), + ), ), expected_fragments=( "unexpected=1", @@ -34,6 +43,7 @@ "row counts actual=2 expected=1", "unexpected sample 1: id=2, status=unexpected", "missing sample 1: id=1, status=expected", + "differing column status: actual=unexpected, expected=expected", ), ), ExpectationDetailTestCase(