Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
27 changes: 8 additions & 19 deletions js/packages/ui/src/components/histogram/HistogramDiffForm.tsx
Original file line number Diff line number Diff line change
Expand Up @@ -15,6 +15,7 @@ import NativeSelect from "@mui/material/NativeSelect";
import type { HistogramDiffParams } from "../../api";
import { useModelColumns } from "../../hooks";
import type { RunFormProps } from "../run";
import { classifyType } from "../ui/DataTypeIcon/classifyType";

// ============================================================================
// Type Utilities
Expand Down Expand Up @@ -63,25 +64,13 @@ function isBooleanDataType(columnType: string) {
}

function isDateTimeType(columnType: string) {
const sql_datetime_types = [
"DATE",
"DATETIME",
"TIMESTAMP",
"TIME",
"YEAR", // Specific to MySQL/MariaDB
"DATETIME2",
"SMALLDATETIME",
"DATETIMEOFFSET", // Specific to SQL Server
"INTERVAL", // Common in PostgreSQL and Oracle
"TIMESTAMPTZ",
"TIMETZ", // Specific to PostgreSQL
"TIMESTAMP WITH TIME ZONE",
"TIMESTAMP WITH LOCAL TIME ZONE", // Oracle
"TIMESTAMP_LTZ",
"TIMESTAMP_NTZ",
"TIMESTAMP_TZ", // Specific to Snowflake
];
return sql_datetime_types.includes(columnType.toUpperCase());
const category = classifyType(columnType);
if (["date", "datetime", "time"].includes(category)) {
return true;
}

const normalizedType = columnType.trim().toUpperCase();
return normalizedType === "YEAR" || normalizedType === "INTERVAL";
}

// ============================================================================
Expand Down
Original file line number Diff line number Diff line change
@@ -1,7 +1,29 @@
import { readFileSync } from "node:fs";
import { resolve } from "node:path";
import { describe, expect, it } from "vitest";
import { supportsHistogramDiff } from "../HistogramDiffForm";

interface HistogramTypePolicyCase {
type: string;
backend_supported: boolean;
picker_supported: boolean;
}

const histogramTypePolicy = JSON.parse(
readFileSync(
resolve(process.cwd(), "../tests/fixtures/histogram_type_policy.json"),
"utf8",
),
) as HistogramTypePolicyCase[];

describe("supportsHistogramDiff", () => {
it.each(histogramTypePolicy)(
"matches the shared backend policy for '$type'",
({ type, picker_supported: pickerSupported }) => {
expect(supportsHistogramDiff(type)).toBe(pickerSupported);
},
);

it("should return true for supported numeric types", () => {
const supportedNumericTypes = [
"INT",
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -144,7 +144,7 @@ const CATEGORY_MAP: Record<string, TypeCategory> = {
* - Special case: TINYINT(1) maps to "boolean", plain TINYINT maps to "integer"
*/
export function classifyType(rawType: string): TypeCategory {
const trimmed = rawType.trim().toUpperCase();
const trimmed = rawType.trim().toUpperCase().replace(/\s+/g, " ");

if (!trimmed) {
return "unknown";
Expand Down
16 changes: 14 additions & 2 deletions recce/apis/check_api.py
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,7 @@
from pydantic import BaseModel

from recce.apis.check_func import (
_validate_check,
create_check_from_run,
create_check_without_run,
export_persistent_state,
Expand Down Expand Up @@ -165,6 +166,8 @@ async def run_check_handler(check_id: UUID, input: RunCheckIn):
raise HTTPException(status_code=404, detail=f"Check ID '{check_id}' not found")

try:
if check.type == RunType.HISTOGRAM_DIFF:
_validate_check(check.type, check.params)
log_api_event(
"rerun_check",
dict(
Expand All @@ -173,7 +176,7 @@ async def run_check_handler(check_id: UUID, input: RunCheckIn):
),
)
run, future = submit_run(check.type, check.params, check_id=check_id, triggered_by="user")
except RecceException as e:
except (RecceException, ValueError) as e:
raise HTTPException(status_code=400, detail=str(e))

if input.nowait:
Expand Down Expand Up @@ -237,11 +240,20 @@ class PatchCheckIn(BaseModel):
response_model_exclude_none=True,
)
async def update_check_handler(check_id: UUID, patch: PatchCheckIn, background_tasks: BackgroundTasks):
check_dao = CheckDAO()
try:
check = CheckDAO().update_check_by_id(check_id, patch)
if patch.params is not None:
existing_check = check_dao.find_check_by_id(check_id)
if existing_check is None:
raise HTTPException(status_code=404, detail="Not Found")
if existing_check.type == RunType.HISTOGRAM_DIFF:
_validate_check(existing_check.type, patch.params)
check = check_dao.update_check_by_id(check_id, patch)
except RecceCloudException as e:
log_cloud_exception(f"Failed to update check {check_id}: {e}", e)
raise HTTPException(status_code=e.status_code, detail=str(e.reason))
except ValueError as e:
raise HTTPException(status_code=400, detail=str(e))

if check is None:
raise HTTPException(status_code=404, detail="Not Found")
Expand Down
13 changes: 12 additions & 1 deletion recce/apis/check_func.py
Original file line number Diff line number Diff line change
Expand Up @@ -28,7 +28,16 @@ def get_node_name_by_id(node_id):
def _validate_check(check_type, params):
if check_type == RunType.SCHEMA_DIFF:
validate_schema_diff_check(params)
pass
elif check_type == RunType.HISTOGRAM_DIFF:
from recce.tasks.histogram import HistogramDiffCheckValidator

HistogramDiffCheckValidator().validate(
{
"name": "Histogram diff",
"type": check_type,
"params": params,
}
)


def _get_ref_model(sql_template: str) -> Optional[str]:
Expand Down Expand Up @@ -102,6 +111,8 @@ def create_check_from_run(
def create_check_without_run(
check_name, check_description, check_type, params, check_view_options, is_preset=False, is_checked=False
):
if check_type == RunType.HISTOGRAM_DIFF:
_validate_check(check_type, params)
name = check_name if check_name is not None else _generate_check_name(check_type, params, check_view_options)

# Get cloud user context for attribution if available
Expand Down
2 changes: 1 addition & 1 deletion recce/apis/run_api.py
Original file line number Diff line number Diff line change
Expand Up @@ -70,7 +70,7 @@ async def create_run_handler(input: CreateRunIn):
)
try:
run, future = submit_run(input.type, input.params, check_id=input.check_id, triggered_by="user")
except RecceException as e:
except (RecceException, ValueError) as e:
raise HTTPException(status_code=400, detail=str(e))

if input.nowait:
Expand Down
4 changes: 4 additions & 0 deletions recce/mcp_server.py
Original file line number Diff line number Diff line change
Expand Up @@ -2799,6 +2799,7 @@ async def _tool_create_check(self, arguments: Dict[str, Any]) -> Dict[str, Any]:
"""Create a persistent check from analysis findings."""
from recce.apis.check_api import PatchCheckIn
from recce.apis.check_func import (
_validate_check,
create_check_without_run,
export_persistent_state,
)
Expand All @@ -2812,6 +2813,9 @@ async def _tool_create_check(self, arguments: Dict[str, Any]) -> Dict[str, Any]:
description = arguments.get("description", "")
approve = arguments.get("approve", True)

if check_type == RunType.HISTOGRAM_DIFF:
_validate_check(check_type, params)

# Idempotency: find existing check with same (type, params)
check_dao = CheckDAO()
existing_checks = check_dao.list()
Expand Down
42 changes: 34 additions & 8 deletions recce/tasks/histogram.py
Original file line number Diff line number Diff line change
Expand Up @@ -25,6 +25,7 @@
"INTERVAL", # Common in PostgreSQL and Oracle
"TIMESTAMPTZ",
"TIMESTAMP WITH TIME ZONE",
"TIMESTAMP WITHOUT TIME ZONE",
"TIMESTAMP WITH LOCAL TIME ZONE", # Oracle
"TIMESTAMP_LTZ",
"TIMESTAMP_NTZ",
Expand Down Expand Up @@ -81,13 +82,33 @@
r"^(CHAR|VARCHAR|NCHAR|NVARCHAR|VARCHAR2|NVARCHAR2)\(\d+\)$", # String types with lengths
]

sql_time_only_type_pattern = re.compile(
r"(?:TIMETZ\s*(?:\(\s*\d+\s*\))?"
r"|TIME\s*(?:\(\s*\d+\s*\))?"
r"(?:\s+(?:WITH|WITHOUT)\s+TIME\s+ZONE\s*(?:\(\s*\d+\s*\))?)?)"
)
sql_type_precision_pattern = re.compile(r"\s*\(\s*\d+\s*\)")


def _normalize_column_type(column_type: str) -> str:
return " ".join(column_type.strip().upper().split())


def _is_datetime_histogram_type(column_type: str) -> bool:
normalized_type = sql_type_precision_pattern.sub("", _normalize_column_type(column_type), count=1)
return normalized_type in sql_datetime_types


def _is_histogram_supported(column_type):
if column_type.upper() in sql_not_supported_types:
normalized_type = _normalize_column_type(column_type)
if sql_time_only_type_pattern.fullmatch(normalized_type):
return False

if normalized_type in sql_not_supported_types:
return False

for pattern in sql_not_supported_types_pattern:
if re.match(pattern, column_type.upper()):
if re.match(pattern, normalized_type):
return False
return True

Expand Down Expand Up @@ -276,6 +297,11 @@ class HistogramDiffParams(BaseModel):
num_bins: Optional[int] = 50


def _validate_histogram_column_type(column_type: str) -> None:
if _is_histogram_supported(column_type) is False:
raise ValueError(f"Column type {column_type} is not supported for histogram analysis")


def query_numeric_histogram(task, node, column, column_type, min_value, max_value, num_bins=50):
is_integer = is_integral_histogram_type(column_type)
geometry = numeric_histogram_geometry(min_value, max_value, num_bins, is_integer=is_integer)
Expand Down Expand Up @@ -441,6 +467,7 @@ class HistogramDiffTask(Task, QueryMixin):
def __init__(self, params):
super().__init__()
self.params = HistogramDiffParams(**params)
_validate_histogram_column_type(self.params.column_type)
self.connection = None

def execute(self):
Expand All @@ -454,8 +481,8 @@ def execute(self):
num_bins = self.params.num_bins or 50
column_type = self.params.column_type

if _is_histogram_supported(column_type) is False:
raise ValueError(f"Column type {column_type} is not supported for histogram analysis")
_validate_histogram_column_type(column_type)
is_datetime_type = _is_datetime_histogram_type(column_type)

with dbt_adapter.connection_named("query"):
self.connection = dbt_adapter.get_thread_connection()
Expand Down Expand Up @@ -496,7 +523,7 @@ def get_min_max(fn, base, curr):
}
bin_edges = []
labels = []
elif column_type.upper() in sql_datetime_types:
elif is_datetime_type:
base_result, current_result, bin_edges = query_datetime_histogram(
self, node, column, min_value, max_value
)
Expand All @@ -510,7 +537,7 @@ def get_min_max(fn, base, curr):
current_result["total"] = curr_total
result["base"] = base_result
result["current"] = current_result
if column_type.upper() in sql_datetime_types or min_value is None:
if is_datetime_type or min_value is None:
result["min"] = min_value
result["max"] = max_value
else:
Expand Down Expand Up @@ -539,5 +566,4 @@ def validate_check(self, check: Check):
except Exception as e:
raise ValueError(f"Invalid check: {str(e)}")

if _is_histogram_supported(params.column_type) is False:
raise ValueError(f"Column type {params.column_type} is not supported for histogram analysis")
_validate_histogram_column_type(params.column_type)
Loading
Loading