Skip to content

Serialize col_vals_in_table() steps and remove the unused legacy tbl_match() #461

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
rich-iannone merged 5 commits into from
Oct 15, 2026
Merged

Serialize col_vals_in_table() steps and remove the unused legacy tbl_match() #461

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
5 commits
Select commit Hold shift + click to select a range
66d85bd
Remove tbl_match from interrogation module
rich-iannone Oct 15, 2026
3be13aa
Support col_vals_in_table step serialization
rich-iannone Oct 15, 2026
b6d4a49
Remove tbl_match tests from interrogation suite
rich-iannone Oct 15, 2026
5707096
Test reference table serialization placeholders
rich-iannone Oct 15, 2026
470ec57
Decouple table match tests from legacy code
rich-iannone Oct 15, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
206 changes: 0 additions & 206 deletions pointblank/_interrogation.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -821,212 +821,6 @@ def convert(tbl: Any, from_backend: str, to_backend: str, which: str) -> Any:
return data_tbl, tbl_compare


def tbl_match(data_tbl: IntoFrame, tbl_compare: IntoFrame) -> bool:
"""
Check if two tables match exactly in schema, row count, and data.

This function performs a comprehensive comparison between two tables,
checking progressively stricter conditions from least to most stringent:

1. Column count match
2. Row count match
3. Schema match (case-insensitive column names, any order)
4. Schema match (case-insensitive column names, correct order)
5. Schema match (case-sensitive column names, correct order)
6. Data match: compares values column-by-column

If the two tables have different backends (e.g., one is Polars and one is Pandas),
the comparison table will be automatically coerced to match the data table's backend
before comparison. This ensures consistent dtype handling.

Parameters
----------
data_tbl
The target table to validate.
tbl_compare
The comparison table to validate against.

Returns
-------
bool
True if tables match completely, False otherwise.
"""
from pointblank.schema import Schema, _check_schema_match
from pointblank.validate import get_column_count, get_row_count

# Coerce to common backend if needed
data_tbl, tbl_compare = _coerce_to_common_backend(data_tbl, tbl_compare)

# Convert both tables to narwhals for compatibility
tbl = _convert_to_narwhals(df=data_tbl)
tbl_cmp = _convert_to_narwhals(df=tbl_compare)

# Stage 1: Check column count (least stringent)
col_count_matching = get_column_count(data=data_tbl) == get_column_count(data=tbl_compare)

if not col_count_matching:
return False

# Stage 2: Check row count
row_count_matching = get_row_count(data=data_tbl) == get_row_count(data=tbl_compare)

if not row_count_matching:
return False

# Stage 3: Check schema match for case-insensitive column names, any order
schema = Schema(tbl=tbl_compare)

col_schema_matching_any_order = _check_schema_match(
data_tbl=data_tbl,
schema=schema,
complete=True,
in_order=False,
case_sensitive_colnames=False,
case_sensitive_dtypes=False,
full_match_dtypes=False,
)

if not col_schema_matching_any_order:
return False

# Stage 4: Check schema match for case-insensitive column names, correct order
col_schema_matching_in_order = _check_schema_match(
data_tbl=data_tbl,
schema=schema,
complete=True,
in_order=True,
case_sensitive_colnames=False,
case_sensitive_dtypes=False,
full_match_dtypes=False,
)

if not col_schema_matching_in_order:
return False

# Stage 5: Check schema match for case-sensitive column names, correct order
col_schema_matching_exact = _check_schema_match(
data_tbl=data_tbl,
schema=schema,
complete=True,
in_order=True,
case_sensitive_colnames=True,
case_sensitive_dtypes=False,
full_match_dtypes=False,
)

if not col_schema_matching_exact:
return False

# Stage 6: Check for exact data by cell across matched columns (most stringent)
# Handle edge case where both tables have zero rows (they match)
if get_row_count(data=data_tbl) == 0:
return True

column_count = get_column_count(data=data_tbl)

# Compare column-by-column
for i in range(column_count):
# Get column name
col_name = tbl.columns[i]

# Get column data from both tables
col_data_1 = tbl.select(col_name)
col_data_2 = tbl_cmp.select(col_name)

# Convert to native format for comparison
# We need to collect if lazy frames
if is_narwhals_lazyframe(col_data_1): # pragma: no cover
col_data_1 = col_data_1.collect()

if is_narwhals_lazyframe(col_data_2): # pragma: no cover
col_data_2 = col_data_2.collect()

# Convert to native and then to lists for comparison
# Native frames could be Polars, Pandas, or Ibis - use Any for dynamic access
col_1_native: Any = col_data_1.to_native()
col_2_native: Any = col_data_2.to_native()

# Extract values as lists for comparison
# Note: We use hasattr for runtime detection but maintain Any typing
values_1: list[Any]
values_2: list[Any]
if hasattr(col_1_native, "to_list"): # pragma: no cover # Polars DataFrame
values_1 = col_1_native[col_name].to_list()
values_2 = col_2_native[col_name].to_list()

elif hasattr(col_1_native, "tolist"): # pragma: no cover # Pandas DataFrame
values_1 = col_1_native[col_name].tolist()
values_2 = col_2_native[col_name].tolist()

elif hasattr(col_1_native, "collect"): # pragma: no cover # Ibis
values_1 = col_1_native[col_name].to_pandas().tolist()
values_2 = col_2_native[col_name].to_pandas().tolist()

else: # pragma: no cover
# Fallback: try direct comparison
values_1 = list(col_1_native[col_name])
values_2 = list(col_2_native[col_name])

# Compare the two lists element by element, handling NaN/None
if len(values_1) != len(values_2): # pragma: no cover
return False

for v1, v2 in zip(values_1, values_2):
# Handle None/NaN comparisons and check both None and NaN
# Note: When Pandas NaN is converted to Polars, it may become None
v1_is_null = v1 is None
v2_is_null = v2 is None

# Check if v1 is NaN
if not v1_is_null:
try:
import math

if math.isnan(v1):
v1_is_null = True
except (TypeError, ValueError):
pass

# Check if v2 is NaN
if not v2_is_null:
try:
import math

if math.isnan(v2):
v2_is_null = True
except (TypeError, ValueError):
pass

# If both are null (None or NaN), they match
if v1_is_null and v2_is_null:
continue

# If only one is null, they don't match
if v1_is_null or v2_is_null:
return False

# Direct comparison: handle lists/arrays separately
try:
if v1 != v2:
return False
except (TypeError, ValueError): # pragma: no cover
# If direct comparison fails (e.g., for lists/arrays), try element-wise comparison
try:
if isinstance(v1, list) and isinstance(v2, list):
if v1 != v2:
return False
elif hasattr(v1, "__eq__") and hasattr(v2, "__eq__"):
# For array-like objects, check if they're equal
if not (v1 == v2).all() if hasattr((v1 == v2), "all") else v1 == v2:
return False
else:
return False
except Exception:
return False

return True


def conjointly_validation(
data_tbl: IntoFrame, expressions: Any, threshold: int, tbl_type: str = "local"
) -> Any:
Expand Down
11 changes: 11 additions & 0 deletions pointblank/validate.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -4681,6 +4681,17 @@ def _columns_param() -> None:
):
if name in values:
params[name] = values[name]
elif at == "col_vals_in_table":
values = vi.values or {}
params["columns"] = values.get("columns", vi.column)
params["ref_table"] = _placeholder(
code="your_reference_table",
note=(
"Step 'col_vals_in_table' checks against a reference table that cannot be "
"serialized; a placeholder was emitted and must be edited manually."
),
)
params["ref_column"] = values.get("ref_column")
elif at == "conjointly":
params["expressions"] = _placeholder(
code="[lambda df: df]",
Expand Down
107 changes: 1 addition & 106 deletions tests/test__interrogation.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -341,108 +341,6 @@ def test_safe_modify_datetime_eager_pandas():
assert not isinstance(result, datetime.datetime)


class TestTblMatch:
"""Tests for the tbl_match() function."""

def test_matching_pandas_dataframes(self):
from pointblank._interrogation import tbl_match

df1 = pd.DataFrame({"a": [1, 2, 3], "b": ["x", "y", "z"]})
df2 = pd.DataFrame({"a": [1, 2, 3], "b": ["x", "y", "z"]})
assert tbl_match(df1, df2) is True

def test_matching_polars_dataframes(self):
from pointblank._interrogation import tbl_match

df1 = pl.DataFrame({"a": [1, 2, 3], "b": ["x", "y", "z"]})
df2 = pl.DataFrame({"a": [1, 2, 3], "b": ["x", "y", "z"]})
assert tbl_match(df1, df2) is True

def test_different_values(self):
from pointblank._interrogation import tbl_match

df1 = pd.DataFrame({"a": [1, 2, 3]})
df2 = pd.DataFrame({"a": [1, 2, 99]})
assert tbl_match(df1, df2) is False

def test_different_column_count(self):
from pointblank._interrogation import tbl_match

df1 = pd.DataFrame({"a": [1], "b": [2]})
df2 = pd.DataFrame({"a": [1]})
assert tbl_match(df1, df2) is False

def test_different_row_count(self):
from pointblank._interrogation import tbl_match

df1 = pd.DataFrame({"a": [1, 2]})
df2 = pd.DataFrame({"a": [1]})
assert tbl_match(df1, df2) is False

def test_different_column_names(self):
from pointblank._interrogation import tbl_match

df1 = pd.DataFrame({"a": [1]})
df2 = pd.DataFrame({"b": [1]})
assert tbl_match(df1, df2) is False

def test_different_column_order(self):
from pointblank._interrogation import tbl_match

df1 = pd.DataFrame({"a": [1], "b": [2]})
df2 = pd.DataFrame({"b": [2], "a": [1]})
assert tbl_match(df1, df2) is False

def test_nan_values_match(self):
from pointblank._interrogation import tbl_match

df1 = pd.DataFrame({"a": [1.0, float("nan"), 3.0]})
df2 = pd.DataFrame({"a": [1.0, float("nan"), 3.0]})
assert tbl_match(df1, df2) is True

def test_none_values_match(self):
from pointblank._interrogation import tbl_match

df1 = pl.DataFrame({"a": [1, None, 3]})
df2 = pl.DataFrame({"a": [1, None, 3]})
assert tbl_match(df1, df2) is True

def test_none_vs_value_mismatch(self):
from pointblank._interrogation import tbl_match

df1 = pl.DataFrame({"a": [1, None, 3]})
df2 = pl.DataFrame({"a": [1, 2, 3]})
assert tbl_match(df1, df2) is False

def test_empty_dataframes_match(self):
from pointblank._interrogation import tbl_match

df1 = pd.DataFrame({"a": pd.Series([], dtype=int)})
df2 = pd.DataFrame({"a": pd.Series([], dtype=int)})
assert tbl_match(df1, df2) is True

def test_cross_backend_pandas_polars(self):
from pointblank._interrogation import tbl_match

df_pd = pd.DataFrame({"a": [1, 2, 3]})
df_pl = pl.DataFrame({"a": [1, 2, 3]})
assert tbl_match(df_pd, df_pl) is True

def test_string_columns(self):
from pointblank._interrogation import tbl_match

df1 = pl.DataFrame({"s": ["hello", "world"]})
df2 = pl.DataFrame({"s": ["hello", "world"]})
assert tbl_match(df1, df2) is True

def test_string_columns_mismatch(self):
from pointblank._interrogation import tbl_match

df1 = pl.DataFrame({"s": ["hello", "world"]})
df2 = pl.DataFrame({"s": ["hello", "earth"]})
assert tbl_match(df1, df2) is False


class TestInterrogateNe:
"""Tests for interrogate_ne() null-handling paths."""

Expand Down Expand Up @@ -1421,7 +1319,7 @@ def test_ibis_pair_keeps_dtypes(self, swap):
detour that would turn a nullable integer column into floats)."""
import ibis

from pointblank._interrogation import _coerce_to_common_backend, tbl_match
from pointblank._interrogation import _coerce_to_common_backend

duck_float = ibis.duckdb.connect().create_table("f", pl.DataFrame({"k": [1.0, 2.0, None]}))
mem_int = ibis.memtable(pl.DataFrame({"k": [1, 2, None]}))
Expand All @@ -1432,9 +1330,6 @@ def test_ibis_pair_keeps_dtypes(self, swap):
dtypes = {result1.schema["k"], result2.schema["k"]}
assert dtypes == {pl.Float64, pl.Int64}

# So the dtype difference is caught in both directions
assert tbl_match(*tbls) is False

@pytest.mark.parametrize("data_kind", ["polars", "pandas"])
def test_ibis_comparison_table_matches_data_backend(self, data_kind):
import ibis
Expand Down
Loading
Loading