diff --git a/pointblank/_interrogation.py b/pointblank/_interrogation.py index 062345f63..8fab8f8e4 100644 --- a/pointblank/_interrogation.py +++ b/pointblank/_interrogation.py @@ -821,212 +821,6 @@ def convert(tbl: Any, from_backend: str, to_backend: str, which: str) -> Any: return data_tbl, tbl_compare -def tbl_match(data_tbl: IntoFrame, tbl_compare: IntoFrame) -> bool: - """ - Check if two tables match exactly in schema, row count, and data. - - This function performs a comprehensive comparison between two tables, - checking progressively stricter conditions from least to most stringent: - - 1. Column count match - 2. Row count match - 3. Schema match (case-insensitive column names, any order) - 4. Schema match (case-insensitive column names, correct order) - 5. Schema match (case-sensitive column names, correct order) - 6. Data match: compares values column-by-column - - If the two tables have different backends (e.g., one is Polars and one is Pandas), - the comparison table will be automatically coerced to match the data table's backend - before comparison. This ensures consistent dtype handling. - - Parameters - ---------- - data_tbl - The target table to validate. - tbl_compare - The comparison table to validate against. - - Returns - ------- - bool - True if tables match completely, False otherwise. - """ - from pointblank.schema import Schema, _check_schema_match - from pointblank.validate import get_column_count, get_row_count - - # Coerce to common backend if needed - data_tbl, tbl_compare = _coerce_to_common_backend(data_tbl, tbl_compare) - - # Convert both tables to narwhals for compatibility - tbl = _convert_to_narwhals(df=data_tbl) - tbl_cmp = _convert_to_narwhals(df=tbl_compare) - - # Stage 1: Check column count (least stringent) - col_count_matching = get_column_count(data=data_tbl) == get_column_count(data=tbl_compare) - - if not col_count_matching: - return False - - # Stage 2: Check row count - row_count_matching = get_row_count(data=data_tbl) == get_row_count(data=tbl_compare) - - if not row_count_matching: - return False - - # Stage 3: Check schema match for case-insensitive column names, any order - schema = Schema(tbl=tbl_compare) - - col_schema_matching_any_order = _check_schema_match( - data_tbl=data_tbl, - schema=schema, - complete=True, - in_order=False, - case_sensitive_colnames=False, - case_sensitive_dtypes=False, - full_match_dtypes=False, - ) - - if not col_schema_matching_any_order: - return False - - # Stage 4: Check schema match for case-insensitive column names, correct order - col_schema_matching_in_order = _check_schema_match( - data_tbl=data_tbl, - schema=schema, - complete=True, - in_order=True, - case_sensitive_colnames=False, - case_sensitive_dtypes=False, - full_match_dtypes=False, - ) - - if not col_schema_matching_in_order: - return False - - # Stage 5: Check schema match for case-sensitive column names, correct order - col_schema_matching_exact = _check_schema_match( - data_tbl=data_tbl, - schema=schema, - complete=True, - in_order=True, - case_sensitive_colnames=True, - case_sensitive_dtypes=False, - full_match_dtypes=False, - ) - - if not col_schema_matching_exact: - return False - - # Stage 6: Check for exact data by cell across matched columns (most stringent) - # Handle edge case where both tables have zero rows (they match) - if get_row_count(data=data_tbl) == 0: - return True - - column_count = get_column_count(data=data_tbl) - - # Compare column-by-column - for i in range(column_count): - # Get column name - col_name = tbl.columns[i] - - # Get column data from both tables - col_data_1 = tbl.select(col_name) - col_data_2 = tbl_cmp.select(col_name) - - # Convert to native format for comparison - # We need to collect if lazy frames - if is_narwhals_lazyframe(col_data_1): # pragma: no cover - col_data_1 = col_data_1.collect() - - if is_narwhals_lazyframe(col_data_2): # pragma: no cover - col_data_2 = col_data_2.collect() - - # Convert to native and then to lists for comparison - # Native frames could be Polars, Pandas, or Ibis - use Any for dynamic access - col_1_native: Any = col_data_1.to_native() - col_2_native: Any = col_data_2.to_native() - - # Extract values as lists for comparison - # Note: We use hasattr for runtime detection but maintain Any typing - values_1: list[Any] - values_2: list[Any] - if hasattr(col_1_native, "to_list"): # pragma: no cover # Polars DataFrame - values_1 = col_1_native[col_name].to_list() - values_2 = col_2_native[col_name].to_list() - - elif hasattr(col_1_native, "tolist"): # pragma: no cover # Pandas DataFrame - values_1 = col_1_native[col_name].tolist() - values_2 = col_2_native[col_name].tolist() - - elif hasattr(col_1_native, "collect"): # pragma: no cover # Ibis - values_1 = col_1_native[col_name].to_pandas().tolist() - values_2 = col_2_native[col_name].to_pandas().tolist() - - else: # pragma: no cover - # Fallback: try direct comparison - values_1 = list(col_1_native[col_name]) - values_2 = list(col_2_native[col_name]) - - # Compare the two lists element by element, handling NaN/None - if len(values_1) != len(values_2): # pragma: no cover - return False - - for v1, v2 in zip(values_1, values_2): - # Handle None/NaN comparisons and check both None and NaN - # Note: When Pandas NaN is converted to Polars, it may become None - v1_is_null = v1 is None - v2_is_null = v2 is None - - # Check if v1 is NaN - if not v1_is_null: - try: - import math - - if math.isnan(v1): - v1_is_null = True - except (TypeError, ValueError): - pass - - # Check if v2 is NaN - if not v2_is_null: - try: - import math - - if math.isnan(v2): - v2_is_null = True - except (TypeError, ValueError): - pass - - # If both are null (None or NaN), they match - if v1_is_null and v2_is_null: - continue - - # If only one is null, they don't match - if v1_is_null or v2_is_null: - return False - - # Direct comparison: handle lists/arrays separately - try: - if v1 != v2: - return False - except (TypeError, ValueError): # pragma: no cover - # If direct comparison fails (e.g., for lists/arrays), try element-wise comparison - try: - if isinstance(v1, list) and isinstance(v2, list): - if v1 != v2: - return False - elif hasattr(v1, "__eq__") and hasattr(v2, "__eq__"): - # For array-like objects, check if they're equal - if not (v1 == v2).all() if hasattr((v1 == v2), "all") else v1 == v2: - return False - else: - return False - except Exception: - return False - - return True - - def conjointly_validation( data_tbl: IntoFrame, expressions: Any, threshold: int, tbl_type: str = "local" ) -> Any: diff --git a/pointblank/validate.py b/pointblank/validate.py index 4417b8757..33b39efa5 100644 --- a/pointblank/validate.py +++ b/pointblank/validate.py @@ -4681,6 +4681,17 @@ def _columns_param() -> None: ): if name in values: params[name] = values[name] + elif at == "col_vals_in_table": + values = vi.values or {} + params["columns"] = values.get("columns", vi.column) + params["ref_table"] = _placeholder( + code="your_reference_table", + note=( + "Step 'col_vals_in_table' checks against a reference table that cannot be " + "serialized; a placeholder was emitted and must be edited manually." + ), + ) + params["ref_column"] = values.get("ref_column") elif at == "conjointly": params["expressions"] = _placeholder( code="[lambda df: df]", diff --git a/tests/test__interrogation.py b/tests/test__interrogation.py index 272cea9af..e5ed3eb2a 100644 --- a/tests/test__interrogation.py +++ b/tests/test__interrogation.py @@ -341,108 +341,6 @@ def test_safe_modify_datetime_eager_pandas(): assert not isinstance(result, datetime.datetime) -class TestTblMatch: - """Tests for the tbl_match() function.""" - - def test_matching_pandas_dataframes(self): - from pointblank._interrogation import tbl_match - - df1 = pd.DataFrame({"a": [1, 2, 3], "b": ["x", "y", "z"]}) - df2 = pd.DataFrame({"a": [1, 2, 3], "b": ["x", "y", "z"]}) - assert tbl_match(df1, df2) is True - - def test_matching_polars_dataframes(self): - from pointblank._interrogation import tbl_match - - df1 = pl.DataFrame({"a": [1, 2, 3], "b": ["x", "y", "z"]}) - df2 = pl.DataFrame({"a": [1, 2, 3], "b": ["x", "y", "z"]}) - assert tbl_match(df1, df2) is True - - def test_different_values(self): - from pointblank._interrogation import tbl_match - - df1 = pd.DataFrame({"a": [1, 2, 3]}) - df2 = pd.DataFrame({"a": [1, 2, 99]}) - assert tbl_match(df1, df2) is False - - def test_different_column_count(self): - from pointblank._interrogation import tbl_match - - df1 = pd.DataFrame({"a": [1], "b": [2]}) - df2 = pd.DataFrame({"a": [1]}) - assert tbl_match(df1, df2) is False - - def test_different_row_count(self): - from pointblank._interrogation import tbl_match - - df1 = pd.DataFrame({"a": [1, 2]}) - df2 = pd.DataFrame({"a": [1]}) - assert tbl_match(df1, df2) is False - - def test_different_column_names(self): - from pointblank._interrogation import tbl_match - - df1 = pd.DataFrame({"a": [1]}) - df2 = pd.DataFrame({"b": [1]}) - assert tbl_match(df1, df2) is False - - def test_different_column_order(self): - from pointblank._interrogation import tbl_match - - df1 = pd.DataFrame({"a": [1], "b": [2]}) - df2 = pd.DataFrame({"b": [2], "a": [1]}) - assert tbl_match(df1, df2) is False - - def test_nan_values_match(self): - from pointblank._interrogation import tbl_match - - df1 = pd.DataFrame({"a": [1.0, float("nan"), 3.0]}) - df2 = pd.DataFrame({"a": [1.0, float("nan"), 3.0]}) - assert tbl_match(df1, df2) is True - - def test_none_values_match(self): - from pointblank._interrogation import tbl_match - - df1 = pl.DataFrame({"a": [1, None, 3]}) - df2 = pl.DataFrame({"a": [1, None, 3]}) - assert tbl_match(df1, df2) is True - - def test_none_vs_value_mismatch(self): - from pointblank._interrogation import tbl_match - - df1 = pl.DataFrame({"a": [1, None, 3]}) - df2 = pl.DataFrame({"a": [1, 2, 3]}) - assert tbl_match(df1, df2) is False - - def test_empty_dataframes_match(self): - from pointblank._interrogation import tbl_match - - df1 = pd.DataFrame({"a": pd.Series([], dtype=int)}) - df2 = pd.DataFrame({"a": pd.Series([], dtype=int)}) - assert tbl_match(df1, df2) is True - - def test_cross_backend_pandas_polars(self): - from pointblank._interrogation import tbl_match - - df_pd = pd.DataFrame({"a": [1, 2, 3]}) - df_pl = pl.DataFrame({"a": [1, 2, 3]}) - assert tbl_match(df_pd, df_pl) is True - - def test_string_columns(self): - from pointblank._interrogation import tbl_match - - df1 = pl.DataFrame({"s": ["hello", "world"]}) - df2 = pl.DataFrame({"s": ["hello", "world"]}) - assert tbl_match(df1, df2) is True - - def test_string_columns_mismatch(self): - from pointblank._interrogation import tbl_match - - df1 = pl.DataFrame({"s": ["hello", "world"]}) - df2 = pl.DataFrame({"s": ["hello", "earth"]}) - assert tbl_match(df1, df2) is False - - class TestInterrogateNe: """Tests for interrogate_ne() null-handling paths.""" @@ -1421,7 +1319,7 @@ def test_ibis_pair_keeps_dtypes(self, swap): detour that would turn a nullable integer column into floats).""" import ibis - from pointblank._interrogation import _coerce_to_common_backend, tbl_match + from pointblank._interrogation import _coerce_to_common_backend duck_float = ibis.duckdb.connect().create_table("f", pl.DataFrame({"k": [1.0, 2.0, None]})) mem_int = ibis.memtable(pl.DataFrame({"k": [1, 2, None]})) @@ -1432,9 +1330,6 @@ def test_ibis_pair_keeps_dtypes(self, swap): dtypes = {result1.schema["k"], result2.schema["k"]} assert dtypes == {pl.Float64, pl.Int64} - # So the dtype difference is caught in both directions - assert tbl_match(*tbls) is False - @pytest.mark.parametrize("data_kind", ["polars", "pandas"]) def test_ibis_comparison_table_matches_data_backend(self, data_kind): import ibis diff --git a/tests/test_plan_serialization.py b/tests/test_plan_serialization.py index 2bd11cdb5..e17c4ea32 100644 --- a/tests/test_plan_serialization.py +++ b/tests/test_plan_serialization.py @@ -168,6 +168,43 @@ def test_to_code_warns_on_specially(small_table): # ─── to_yaml() ─────────────────────────────────────────────────────────────────── +def test_to_code_col_vals_in_table_uses_ref_table_placeholder(): + # The reference table can't be serialized, so it becomes a placeholder (with a warning); the + # step itself, with its columns and options, is kept + import polars as pl + + tbl = pl.DataFrame({"a": [1, 2, 3], "b": ["p", "q", "r"]}) + ref = pl.DataFrame({"x": [1, 2], "y": ["p", "q"]}) + validation = ( + Validate(data=tbl) + .col_vals_in_table(columns="a", ref_table=ref, ref_column="x", na_pass=True) + .col_vals_in_table(columns=["a", "b"], ref_table=ref, ref_column=["x", "y"]) + ) + + with pytest.warns(UserWarning, match="reference table that cannot be serialized"): + code = validation.to_code() + + assert ( + '.col_vals_in_table(columns="a", ref_table=your_reference_table, ref_column="x", ' + "na_pass=True)" in code + ) + + # With the placeholder bound to a table, the generated code gives the same plan + namespace = {} + exec( + code.replace("your_data", "data"), + {"pb": pb, "data": tbl, "your_reference_table": ref}, + namespace, + ) + rebuilt = namespace["validation"].interrogate() + original = validation.interrogate() + assert rebuilt.n_failed(scalar=False) == original.n_failed(scalar=False) == {1: 1, 2: 1} + + with pytest.warns(UserWarning, match="reference table that cannot be serialized"): + yaml_text = validation.to_yaml() + assert "ref_table: your_reference_table" in yaml_text + + def test_to_yaml_roundtrips_via_yaml_interrogate(small_table): validation = ( Validate(data=small_table, tbl_name="small_table", label="My Plan") diff --git a/tests/test_tbl_match_compare.py b/tests/test_tbl_match_compare.py index 63e9e4888..bd7480152 100644 --- a/tests/test_tbl_match_compare.py +++ b/tests/test_tbl_match_compare.py @@ -14,7 +14,6 @@ import pytest import pointblank as pb -from pointblank._interrogation import tbl_match as tbl_match_legacy SOURCE = pl.DataFrame({"id": [1, 2, 3, 4], "x": [1.0, 2.0, 3.0, 4.0], "s": ["a", "b", "c", "d"]}) @@ -27,40 +26,37 @@ def _step(validation: pb.Validate, i: int = 1): # ---------------------------------------------------------------------------------------------- -# Verdict parity with the previous (single test unit) implementation +# Verdicts of the previous (single test unit) implementation # ---------------------------------------------------------------------------------------------- _BASE = {"a": [1, 2, 3, 4], "b": ["w", "x", "y", "z"], "c": [4.0, 5.0, 6.0, 7.0]} PARITY_CASES = { - "identical": (_BASE, _BASE), - "value_differs": (_BASE, {**_BASE, "c": [4.0, 5.5, 6.0, 7.0]}), - "fewer_rows": (_BASE, {k: v[:2] for k, v in _BASE.items()}), - "more_rows": ({k: v[:2] for k, v in _BASE.items()}, _BASE), - "renamed_column": (_BASE, {"a": _BASE["a"], "b": _BASE["b"], "d": _BASE["c"]}), - "column_order": (_BASE, {"a": _BASE["a"], "c": _BASE["c"], "b": _BASE["b"]}), - "column_case": (_BASE, {"A": _BASE["a"], "b": _BASE["b"], "c": _BASE["c"]}), - "extra_column": (_BASE, {**_BASE, "d": [0, 0, 0, 0]}), - "int_vs_float": (_BASE, {**_BASE, "a": [1.0, 2.0, 3.0, 4.0]}), - "nan_vs_null": ( - {"a": [1, 2], "c": [float("nan"), 1.0]}, - {"a": [1, 2], "c": [None, 1.0]}, - ), - "nulls_equal": ({"a": [1, None], "b": [None, "x"]}, {"a": [1, None], "b": [None, "x"]}), - "null_vs_value": ({"a": [1, None]}, {"a": [1, 2]}), - "row_order": (_BASE, {k: list(reversed(v)) for k, v in _BASE.items()}), + "identical": (_BASE, _BASE, True), + "value_differs": (_BASE, {**_BASE, "c": [4.0, 5.5, 6.0, 7.0]}, False), + "fewer_rows": (_BASE, {k: v[:2] for k, v in _BASE.items()}, False), + "more_rows": ({k: v[:2] for k, v in _BASE.items()}, _BASE, False), + "renamed_column": (_BASE, {"a": _BASE["a"], "b": _BASE["b"], "d": _BASE["c"]}, False), + "column_order": (_BASE, {"a": _BASE["a"], "c": _BASE["c"], "b": _BASE["b"]}, False), + "column_case": (_BASE, {"A": _BASE["a"], "b": _BASE["b"], "c": _BASE["c"]}, False), + "extra_column": (_BASE, {**_BASE, "d": [0, 0, 0, 0]}, False), + "int_vs_float": (_BASE, {**_BASE, "a": [1.0, 2.0, 3.0, 4.0]}, False), + "nan_vs_null": ({"a": [1, 2], "c": [float("nan"), 1.0]}, {"a": [1, 2], "c": [None, 1.0]}, True), + "nulls_equal": ({"a": [1, None], "b": [None, "x"]}, {"a": [1, None], "b": [None, "x"]}, True), + "null_vs_value": ({"a": [1, None]}, {"a": [1, 2]}, False), + "row_order": (_BASE, {k: list(reversed(v)) for k, v in _BASE.items()}, False), } @pytest.mark.parametrize("backend", ["polars", "pandas"]) @pytest.mark.parametrize("case", list(PARITY_CASES)) -def test_default_verdict_matches_legacy_implementation(case, backend): - data_dict, compare_dict = PARITY_CASES[case] +def test_default_verdict_matches_previous_implementation(case, backend): + # `expected` is what the previous implementation (removed) gave for each case + data_dict, compare_dict, expected = PARITY_CASES[case] data, comparison = pl.DataFrame(data_dict), pl.DataFrame(compare_dict) if backend == "pandas": data, comparison = data.to_pandas(), comparison.to_pandas() - expected = tbl_match_legacy(data_tbl=data, tbl_compare=comparison) validation = pb.Validate(data=data).tbl_match(tbl_compare=comparison).interrogate() assert validation.all_passed() is expected