diff --git a/pointblank/datascan.py b/pointblank/datascan.py index 11db84748..4e067cd6e 100644 --- a/pointblank/datascan.py +++ b/pointblank/datascan.py @@ -345,68 +345,41 @@ def get_tabular_report(self, *, show_sample_data: bool = False) -> GT: ) if "freqs" in formatted_data.columns: # TODO: don't love this arbitrary check - # Extract HTML freqs: - try: - formatted_data = formatted_data.with_columns( - __freq_true=nw.col("freqs").struct.field("True"), - __freq_false=nw.col("freqs").struct.field("False"), - ) - except Exception: # TODO: should be narrowed if possible - # if no struct implimentation exists, it must be done manually - freq_ser: nw.Series = formatted_data["freqs"] - trues: list[int | None] = [] - falses: list[int | None] = [] - for freq in freq_ser: - try: - trues.append(freq["True"]) - falses.append(freq["False"]) - except (KeyError, TypeError): - trues.append(None) - falses.append(None) - true_ser: nw.Series = nw.new_series( - name="__freq_true", values=trues, backend=_backend + # Build the T/F summary for boolean columns straight from each column's profile. + # The `freqs` dict only has keys for the values present (e.g., `{"True": 3}` for an + # all-`True` column), so a missing key is a count of zero. The rows of `data` follow + # the order of `column_profiles`. + bool_unique_html: list[str | None] = [] + for prof in self.profile.column_profiles: + freqs = next((stat.val for stat in prof.statistics if stat.name == "freqs"), None) + if not isinstance(freqs, dict): + bool_unique_html.append(None) + continue + pct_true, pct_false = _fmt_frac( + [ + freqs.get("True", 0) / self.profile.row_count, + freqs.get("False", 0) / self.profile.row_count, + ] ) - false_ser: nw.Series = nw.new_series( - name="__freq_false", values=falses, backend=_backend + bool_unique_html.append( + f"T{pct_true}" + f"
F{pct_false}" ) - formatted_data = formatted_data.with_columns( - __freq_true=true_ser, __freq_false=false_ser - ) - - ## format pct true values - formatted_data = formatted_data.with_columns( - # for bools, UQs are represented as percentages - __pct_true=nw.col("__freq_true") / self.profile.row_count, - __pct_false=nw.col("__freq_false") / self.profile.row_count, + bool_ser: nw.Series = nw.new_series( + name="__bool_unique_html", + values=bool_unique_html, + dtype=nw.String(), + backend=_backend, ) - for _fmt_col in ("__pct_true", "__pct_false"): - _formatted: list[str | None] = _fmt_frac(formatted_data[_fmt_col]) - formatted = nw.new_series(name=_fmt_col, values=_formatted, backend=_backend) - formatted_data = formatted_data.drop(_fmt_col) - formatted_data = formatted_data.with_columns(formatted.alias(_fmt_col)) formatted_data = ( - formatted_data.with_columns( - __bool_unique_html=nw.concat_str( - nw.lit("T"), - nw.col("__pct_true"), - nw.lit("
F"), - nw.col("__pct_false"), - ), - ) + formatted_data.with_columns(__bool_unique_html=bool_ser) .with_columns( n_unique=nw.when(~nw.col("__bool_unique_html").is_null()) .then(nw.col("__bool_unique_html")) .otherwise(nw.col("n_unique")) ) - .drop( - "__freq_true", - "__freq_false", - "__bool_unique_html", - "freqs", - "__pct_true", - "__pct_false", - ) + .drop("__bool_unique_html", "freqs") ) ## Determine Value Formatting Selectors: diff --git a/tests/test_datascan.py b/tests/test_datascan.py index c4fd67917..7ffcc23c4 100644 --- a/tests/test_datascan.py +++ b/tests/test_datascan.py @@ -216,6 +216,41 @@ def test_col_summary_tbl(df): assert isinstance(col_summary, GT) +def _bool_summary_cells(col_summary: GT) -> list[str]: + tag = "" + return [ + str(v).replace(tag, "").replace("", "") + for v in nw.from_native(col_summary._tbl_data)["n_unique"] + ] + + +@pytest.mark.parametrize( + ("data", "expected"), + [ + ({"flag": [True, True, True]}, ["T1
F0"]), + ({"flag": [True]}, ["T1
F0"]), + ({"flag": [True, None, True]}, ["T0.67
F0"]), + ({"flag": [False, False]}, ["T0
F1"]), + ({"a": [True, True, False], "b": [True, True, True]}, ["T0.67
F0.33", "T1
F0"]), + ({"a": [True, True, True], "b": [True, False, None]}, ["T1
F0", "T0.33
F0.33"]), + ], +) +@pytest.mark.parametrize("backend", ["polars", "pandas", "duckdb"]) +def test_col_summary_tbl_bool_missing_true_or_false(data, expected, backend): + if backend == "pandas": + pytest.importorskip("pandas") + if any(None in vals for vals in data.values()): + pytest.skip("pandas stores booleans with missing values as `object`") + tbl = pl.DataFrame(data).to_pandas() + elif backend == "duckdb": + ibis = pytest.importorskip("ibis") + tbl = ibis.memtable(pl.DataFrame(data).to_arrow()) + else: + tbl = pl.DataFrame(data) + + assert _bool_summary_cells(col_summary_tbl(tbl)) == expected + + def test_col_summary_tbl_polars_categorical_column(): log_levels = pl.Enum(["debug", "info", "warning", "error"])