From 5bf5bab1a1d149369ad641dc5c0b1bbcaa7cad3e Mon Sep 17 00:00:00 2001 From: Khadija Date: Thu, 10 Sep 2026 12:49:11 +0500 Subject: [PATCH 01/12] ci: add multi-version test matrix (3.9-3.12), coverage XML artifact upload, enforce 90% coverage gate --- .github/workflows/ci.yml | 48 +++++++++++++++++++++++++++++++--------- 1 file changed, 37 insertions(+), 11 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index e73dfe2..838ffa7 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -1,20 +1,46 @@ name: CI - + on: push: - branches: [ main ] + branches: + - main pull_request: - branches: [ main ] - + branches: + - main + jobs: test: runs-on: ubuntu-latest + strategy: + fail-fast: false + matrix: + python-version: ["3.9", "3.10", "3.11", "3.12"] + steps: - - uses: actions/checkout@v3 - - uses: actions/setup-python@v4 + - name: Check out repository + uses: actions/checkout@v4 + + - name: Set up Python ${{ matrix.python-version }} + uses: actions/setup-python@v5 with: - python-version: "3.10" - # - name: Install dependencies - # run: pip install -e ".[dev]" - # - name: Run tests - # run: pytest --cov= + python-version: ${{ matrix.python-version }} + + - name: Install dependencies + run: | + python -m pip install --upgrade pip + pip install -e . + pip install pytest pytest-cov + + - name: Run tests with coverage (90% required) + run: | + pytest --cov=inference_audit \ + --cov-report=xml:coverage_report.xml \ + --cov-report=term-missing \ + --cov-fail-under=90 + + - name: Upload coverage report artifact + if: always() + uses: actions/upload-artifact@v4 + with: + name: coverage-report-py${{ matrix.python-version }} + path: coverage_report.xml \ No newline at end of file From 3d0c58e7651614e69ba1b7f1b3ed731c6e452649 Mon Sep 17 00:00:00 2001 From: Khadija Date: Thu, 10 Sep 2026 13:05:56 +0500 Subject: [PATCH 02/12] cleanup: gitignore htmlcov/coverage artifacts, remove duplicate annotation_consistency.py from wrong location --- .gitignore | 4 + checks/annotation_consistency.py | 146 ------------------------------- 2 files changed, 4 insertions(+), 146 deletions(-) delete mode 100644 checks/annotation_consistency.py diff --git a/.gitignore b/.gitignore index 381cd39..5364276 100644 --- a/.gitignore +++ b/.gitignore @@ -3,3 +3,7 @@ __pycache__/ .vscode/ venv/ + +htmlcov/ +coverage_report.xml +.coverage diff --git a/checks/annotation_consistency.py b/checks/annotation_consistency.py deleted file mode 100644 index e9be711..0000000 --- a/checks/annotation_consistency.py +++ /dev/null @@ -1,146 +0,0 @@ -""" -inference_audit/checks/annotation_consistency.py - -Flags low-confidence annotations that indicate genuine annotator -disagreement. Per the design doc's Public API: conf_col is optional -- -if not provided, this check returns score=None with a clear explanation -rather than failing, since not every dataset has a confidence column. -""" - -import pandas as pd -from inference_audit.report import CheckResult - - -def check_annotation_consistency( - df: pd.DataFrame, - conf_col: str = None, - confidence_threshold: float = 0.6, -) -> CheckResult: - """ - Flags samples with annotation confidence below a threshold. - - Args: - df: The dataset as a pandas DataFrame. - conf_col: Name of the confidence column. If None - (not provided), this check is skipped - gracefully -- not every dataset has one. - confidence_threshold: Confidence below this value is flagged - as low-confidence. Must be in [0, 1]. - Default 0.6. - - Returns: - CheckResult with score (0-100), warning (str or None), and - details (dict with low-confidence count, null count in the - confidence column, and the threshold used). - - Never raises. Returns CheckResult(score=None, ...) for: no conf_col - provided (expected, normal case -- not an error), missing column, - empty dataframe, invalid threshold, non-numeric confidence values, - or all confidence values being null. - """ - total_rows = len(df) - - # No confidence column provided is the EXPECTED, normal case for - # many datasets -- per design doc's Public API, this is a graceful - # skip, not an error condition. - if conf_col is None: - return CheckResult( - score=None, - warning="No confidence column provided — annotation consistency check skipped.", - details={"error": "no_confidence_column"}, - ) - - # Defense-in-depth only: primary column validation happens once, - # centrally, in loader.py before any check runs. This guard exists - # so the check still honors its "never raises" contract if ever - # called directly, not as the main validation path. - if conf_col not in df.columns: - return CheckResult( - score=None, - warning=f"Column '{conf_col}' not found in dataset.", - details={"error": "missing_column"}, - ) - - if total_rows == 0: - return CheckResult( - score=None, - warning="Dataset is empty (0 rows) — nothing to check.", - details={"error": "empty_dataframe"}, - ) - - # Validate threshold explicitly -- same discipline the director - # asked for on label_distribution's max_acceptable_ratio. A - # confidence threshold outside [0, 1] is meaningless, since - # confidence values themselves are expected in that range. - if not (0.0 <= confidence_threshold <= 1.0): - return CheckResult( - score=None, - warning=f"confidence_threshold must be between 0 and 1 (got {confidence_threshold}).", - details={"error": "invalid_confidence_threshold", "confidence_threshold": confidence_threshold}, - ) - - # Applying the same null-handling discipline the director's review - # required for label_distribution: null confidence values are - # computed and reported explicitly, never silently dropped from the - # denominator without a trace. - is_null = df[conf_col].isna() - null_count = int(is_null.sum()) - non_null_values = df[conf_col].dropna() - - if len(non_null_values) == 0: - return CheckResult( - score=None, - warning=f"All {total_rows} values in '{conf_col}' are null — cannot assess annotation consistency.", - details={"error": "all_confidence_null", "null_count": null_count, "total_rows": total_rows}, - ) - - # Applying the same non-numeric-value discipline the director's - # review required for missing_values: a confidence column should - # contain numeric values, not silently coerced strings/objects. - is_numeric_mask = non_null_values.apply(lambda v: isinstance(v, (int, float)) and not isinstance(v, bool)) - non_numeric_count = int((~is_numeric_mask).sum()) - if non_numeric_count > 0: - return CheckResult( - score=None, - warning=( - f"Column '{conf_col}' contains {non_numeric_count} non-numeric value(s). " - f"This check requires numeric confidence values." - ), - details={"error": "non_numeric_values", "non_numeric_count": non_numeric_count}, - ) - - non_null_count = len(non_null_values) - is_low_confidence = non_null_values < confidence_threshold - low_confidence_count = int(is_low_confidence.sum()) - - # Score formula, per design doc: 100 x (1 - low_confidence_rate). - # Rate is computed over non-null values only -- nulls are reported - # separately (see warning/details below), not silently folded into - # either the numerator or denominator. - low_confidence_rate = low_confidence_count / non_null_count - score = round(100 * (1 - low_confidence_rate), 2) - - warning_parts = [] - if low_confidence_count > 0: - warning_parts.append( - f"{low_confidence_count}/{non_null_count} annotated samples " - f"({low_confidence_rate:.1%}) have confidence below {confidence_threshold}." - ) - if null_count > 0: - null_rate = null_count / total_rows - warning_parts.append( - f"{null_count}/{total_rows} rows ({null_rate:.1%}) have null confidence " - f"values and were excluded from this calculation." - ) - warning = " ".join(warning_parts) if warning_parts else None - - details = { - "total_rows": total_rows, - "non_null_count": non_null_count, - "null_count": null_count, - "low_confidence_count": low_confidence_count, - "low_confidence_rate": round(low_confidence_rate, 4), - "confidence_threshold": confidence_threshold, - } - - return CheckResult(score=score, warning=warning, details=details) From 708f85ff1b251b127b0a0b27d2a3c3affb39283c Mon Sep 17 00:00:00 2001 From: Khadija Date: Thu, 10 Sep 2026 13:14:30 +0500 Subject: [PATCH 03/12] fix auditor.py: import LANGUAGE_CONCERN_LANGUAGES from config, matching language_contamination.py's now-completed config decoupling --- inference_audit/auditor.py | 25 +++++++------------------ 1 file changed, 7 insertions(+), 18 deletions(-) diff --git a/inference_audit/auditor.py b/inference_audit/auditor.py index 4be308b..e3e177b 100644 --- a/inference_audit/auditor.py +++ b/inference_audit/auditor.py @@ -4,22 +4,13 @@ from inference_audit.loader import load_dataset from inference_audit.report import AuditReport, _get_audit_version +from inference_audit.config import LANGUAGE_CONCERN_LANGUAGES from inference_audit.checks.label_distribution import check_label_distribution from inference_audit.checks.near_duplicates import check_near_duplicates -from inference_audit.checks.language_contamination import ( - check_language_contamination, - DEFAULT_CONCERN_LANGUAGES, -) +from inference_audit.checks.language_contamination import check_language_contamination from inference_audit.checks.missing_values import check_missing_values from inference_audit.checks.annotation_consistency import check_annotation_consistency -# NOTE: DEFAULT_CONCERN_LANGUAGES is imported directly from the check -# module rather than a central config, per review discussion. Moving -# it to a shared config would require editing language_contamination.py -# to match, and that file isn't in my control on this branch yet (still -# on Shoaib's unmerged PR). Deferring this decoupling until both files -# can be updated together, rather than half-applying it here. - class Auditor: """Runs a full quality audit on a dataset and returns an AuditReport.""" @@ -42,12 +33,10 @@ def audit( concern_languages: Optional iterable of ISO 639-1 language codes to flag if confidently detected in language_contamination. If not provided (None), falls back to - check_language_contamination's DEFAULT_CONCERN_LANGUAGES. An - explicitly provided empty iterable is passed through as-is - rather than treated as "not provided" -- the check already - validates and reports on an empty concern list itself, so - validation stays in one place rather than being duplicated - here. + config.LANGUAGE_CONCERN_LANGUAGES. An explicitly provided + empty iterable is passed through as-is rather than treated + as "not provided" -- the check already validates and + reports on an empty concern list itself. Returns: A fully populated AuditReport. @@ -60,7 +49,7 @@ def audit( resolved_concern_languages = ( concern_languages if concern_languages is not None - else DEFAULT_CONCERN_LANGUAGES + else LANGUAGE_CONCERN_LANGUAGES ) checks = { From 3aefef1d97d04d446af44742db03d2d0c6c17aac Mon Sep 17 00:00:00 2001 From: Khadija Date: Fri, 11 Sep 2026 13:13:37 +0500 Subject: [PATCH 04/12] added tests for auditor.py cli.py report.py report_renderer.py loader.py --- tests/test_auditor.py | 131 +++++++++++++++++++ tests/test_cli.py | 230 ++++++++++++++++++++++++++++++++++ tests/test_loader.py | 78 ++++++++++++ tests/test_report.py | 127 +++++++++++++++++++ tests/test_report_renderer.py | 93 ++++++++++++++ 5 files changed, 659 insertions(+) create mode 100644 tests/test_auditor.py create mode 100644 tests/test_cli.py create mode 100644 tests/test_loader.py create mode 100644 tests/test_report.py create mode 100644 tests/test_report_renderer.py diff --git a/tests/test_auditor.py b/tests/test_auditor.py new file mode 100644 index 0000000..b6fff3b --- /dev/null +++ b/tests/test_auditor.py @@ -0,0 +1,131 @@ +"""Tests for inference_audit/auditor.py + +Uses monkeypatch to stub out load_dataset() and all five check +functions, so these tests verify ORCHESTRATION (does Auditor call the +right things, with the right arguments, and assemble the result +correctly) rather than re-testing each check's own logic, which +already has its own dedicated test file. +""" + +import pandas as pd +import pytest + +from inference_audit.auditor import Auditor +from inference_audit.report import CheckResult +from inference_audit.config import LANGUAGE_CONCERN_LANGUAGES + + +@pytest.fixture +def stub_checks(monkeypatch): + """Replaces every check function Auditor.audit() calls with a stub + that records how it was called and returns a fixed CheckResult.""" + calls = {} + + def make_stub(name, score=75): + def stub(*args, **kwargs): + calls[name] = {"args": args, "kwargs": kwargs} + return CheckResult(score=score, warning=None, details={}) + return stub + + fake_df = pd.DataFrame({"label": ["a", "b"], "text": ["x", "y"]}) + monkeypatch.setattr( + "inference_audit.auditor.load_dataset", + lambda path, label_col, text_col: fake_df, + ) + monkeypatch.setattr( + "inference_audit.auditor.check_label_distribution", + make_stub("label_distribution"), + ) + monkeypatch.setattr( + "inference_audit.auditor.check_near_duplicates", + make_stub("near_duplicates"), + ) + monkeypatch.setattr( + "inference_audit.auditor.check_language_contamination", + make_stub("language_contamination"), + ) + monkeypatch.setattr( + "inference_audit.auditor.check_missing_values", + make_stub("missing_values"), + ) + monkeypatch.setattr( + "inference_audit.auditor.check_annotation_consistency", + make_stub("annotation_consistency"), + ) + return calls + + +def test_audit_returns_report_with_all_five_checks(stub_checks): + auditor = Auditor() + report = auditor.audit("dataset.csv", label_col="label", text_col="text") + + assert set(report.checks.keys()) == { + "label_distribution", "near_duplicates", "language_contamination", + "missing_values", "annotation_consistency", + } + assert report.rows == 2 + assert report.dataset_path == "dataset.csv" + + +def test_audit_uses_default_concern_languages_when_not_provided(stub_checks): + auditor = Auditor() + auditor.audit("dataset.csv", label_col="label", text_col="text") + + call = stub_checks["language_contamination"] + assert call["kwargs"]["concern_languages"] == LANGUAGE_CONCERN_LANGUAGES + + +def test_audit_uses_provided_concern_languages_when_given(stub_checks): + auditor = Auditor() + auditor.audit( + "dataset.csv", label_col="label", text_col="text", + concern_languages=("es", "fr"), + ) + + call = stub_checks["language_contamination"] + assert call["kwargs"]["concern_languages"] == ("es", "fr") + + +def test_audit_passes_through_explicit_empty_concern_languages(stub_checks): + """An explicitly empty tuple should NOT be silently replaced with + the default -- only None (not provided) triggers the default.""" + auditor = Auditor() + auditor.audit( + "dataset.csv", label_col="label", text_col="text", + concern_languages=(), + ) + + call = stub_checks["language_contamination"] + assert call["kwargs"]["concern_languages"] == () + + +def test_audit_passes_conf_col_to_annotation_consistency(stub_checks): + auditor = Auditor() + auditor.audit( + "dataset.csv", label_col="label", text_col="text", conf_col="confidence", + ) + + call = stub_checks["annotation_consistency"] + assert call["kwargs"]["conf_col"] == "confidence" + + +def test_audit_propagates_file_not_found(monkeypatch): + def raise_not_found(path, label_col, text_col): + raise FileNotFoundError(f"no such file: {path}") + + monkeypatch.setattr("inference_audit.auditor.load_dataset", raise_not_found) + + auditor = Auditor() + with pytest.raises(FileNotFoundError): + auditor.audit("missing.csv", label_col="label", text_col="text") + + +def test_audit_propagates_value_error(monkeypatch): + def raise_value_error(path, label_col, text_col): + raise ValueError("bad column") + + monkeypatch.setattr("inference_audit.auditor.load_dataset", raise_value_error) + + auditor = Auditor() + with pytest.raises(ValueError): + auditor.audit("dataset.csv", label_col="wrong", text_col="text") \ No newline at end of file diff --git a/tests/test_cli.py b/tests/test_cli.py new file mode 100644 index 0000000..dd564ed --- /dev/null +++ b/tests/test_cli.py @@ -0,0 +1,230 @@ +"""Tests for inference_audit/cli.py + +Uses Typer's CliRunner to invoke the CLI as a subprocess-like call +without actually spawning a process. Auditor.audit() is monkeypatched +to return a fixed AuditReport, so these tests verify CLI behavior +(argument parsing, validation, exit codes, output routing) in +isolation from the real audit pipeline, which already has its own +tests elsewhere. +""" + +import pytest +from typer.testing import CliRunner + +from inference_audit.cli import app +from inference_audit.report import AuditReport, CheckResult + +runner = CliRunner() + + +def get_output(result): + """Combines stdout and stderr into one string for assertions. + + typer.echo(..., err=True) writes to stderr, which result.stdout + does not include in newer Click/Typer versions (older versions + merged the two streams by default; current versions keep them + separate). Error-path tests need to check both, since the CLI + correctly writes user-facing errors to stderr, not stdout. + """ + text = result.stdout or "" + try: + if result.stderr: + text += result.stderr + except ValueError: + # Some CliRunner configurations don't capture stderr separately + # at all, in which case it's already included in result.stdout. + pass + return text + + +def _fake_report(overall_score=75, path="dataset.csv"): + return AuditReport( + dataset_path=path, + rows=10, + label_col="label", + text_col="text", + audit_version="dev", + timestamp="2026-01-01T00:00:00", + checks={"missing_values": CheckResult(score=overall_score, warning=None)}, + ) + + +@pytest.fixture +def fake_csv(tmp_path): + path = tmp_path / "dataset.csv" + path.write_text("label,text\na,hello\nb,world\n") + return str(path) + + +def test_run_requires_run_subcommand(): + """Confirms the callback fix is actually in place -- Typer should + NOT collapse to a single implicit command.""" + result = runner.invoke(app, ["--help"]) + assert "run" in result.stdout + + +def test_missing_required_options_fails(fake_csv): + result = runner.invoke(app, ["run", fake_csv]) + assert result.exit_code != 0 + + +def test_successful_run_writes_html_by_default(monkeypatch, fake_csv, tmp_path): + monkeypatch.setattr( + "inference_audit.cli.Auditor.audit", + lambda self, *a, **k: _fake_report(), + ) + output_path = tmp_path / "out.html" + result = runner.invoke(app, [ + "run", fake_csv, + "--label-col", "label", "--text-col", "text", + "--output", str(output_path), + ]) + assert result.exit_code == 0 + assert "Overall score" in result.stdout + + +def test_json_output_routes_to_to_json(monkeypatch, fake_csv, tmp_path): + calls = {} + monkeypatch.setattr( + "inference_audit.cli.Auditor.audit", + lambda self, *a, **k: _fake_report(), + ) + monkeypatch.setattr( + AuditReport, "to_json", lambda self, path: calls.setdefault("to_json", path) + ) + monkeypatch.setattr( + AuditReport, "save", lambda self, path: calls.setdefault("save", path) + ) + + output_path = tmp_path / "out.json" + result = runner.invoke(app, [ + "run", fake_csv, + "--label-col", "label", "--text-col", "text", + "--output", str(output_path), + ]) + assert result.exit_code == 0 + assert "to_json" in calls + assert "save" not in calls + + +def test_invalid_output_extension_rejected(fake_csv, tmp_path): + result = runner.invoke(app, [ + "run", fake_csv, + "--label-col", "label", "--text-col", "text", + "--output", str(tmp_path / "out.txt"), + ]) + assert result.exit_code == 1 + assert "must end in" in get_output(result) + + +def test_fail_below_out_of_range_rejected(fake_csv): + result = runner.invoke(app, [ + "run", fake_csv, + "--label-col", "label", "--text-col", "text", + "--fail-below", "150", + ]) + assert result.exit_code == 1 + assert "between 0 and 100" in get_output(result) + + +def test_fail_below_triggers_exit_1_when_score_low(monkeypatch, fake_csv, tmp_path): + monkeypatch.setattr( + "inference_audit.cli.Auditor.audit", + lambda self, *a, **k: _fake_report(overall_score=40), + ) + result = runner.invoke(app, [ + "run", fake_csv, + "--label-col", "label", "--text-col", "text", + "--output", str(tmp_path / "out.html"), + "--fail-below", "70", + ]) + assert result.exit_code == 1 + assert "below threshold" in result.stdout + + +def test_fail_below_passes_when_score_high(monkeypatch, fake_csv, tmp_path): + monkeypatch.setattr( + "inference_audit.cli.Auditor.audit", + lambda self, *a, **k: _fake_report(overall_score=95), + ) + result = runner.invoke(app, [ + "run", fake_csv, + "--label-col", "label", "--text-col", "text", + "--output", str(tmp_path / "out.html"), + "--fail-below", "70", + ]) + assert result.exit_code == 0 + + +def test_invalid_concern_language_code_rejected(fake_csv, tmp_path): + result = runner.invoke(app, [ + "run", fake_csv, + "--label-col", "label", "--text-col", "text", + "--output", str(tmp_path / "out.html"), + "--concern-languages", "english,hi", # "english" is not a valid 2-letter code + ]) + assert result.exit_code == 1 + assert "invalid language code" in get_output(result) + + +def test_valid_concern_languages_parsed_and_passed(monkeypatch, fake_csv, tmp_path): + captured = {} + + def fake_audit(self, path, label_col, text_col, conf_col=None, concern_languages=None): + captured["concern_languages"] = concern_languages + return _fake_report() + + monkeypatch.setattr("inference_audit.cli.Auditor.audit", fake_audit) + result = runner.invoke(app, [ + "run", fake_csv, + "--label-col", "label", "--text-col", "text", + "--output", str(tmp_path / "out.html"), + "--concern-languages", "en,hi", + ]) + assert result.exit_code == 0 + assert captured["concern_languages"] == ("en", "hi") + + +def test_file_not_found_gives_clean_error_not_traceback(fake_csv, tmp_path): + result = runner.invoke(app, [ + "run", "does_not_exist.csv", + "--label-col", "label", "--text-col", "text", + "--output", str(tmp_path / "out.html"), + ]) + assert result.exit_code == 1 + assert "Traceback" not in get_output(result) + assert "Error" in get_output(result) + + +def test_unexpected_exception_is_caught_at_cli_boundary(monkeypatch, fake_csv, tmp_path): + def raise_unexpected(self, *a, **k): + raise RuntimeError("something unrelated broke") + + monkeypatch.setattr("inference_audit.cli.Auditor.audit", raise_unexpected) + result = runner.invoke(app, [ + "run", fake_csv, + "--label-col", "label", "--text-col", "text", + "--output", str(tmp_path / "out.html"), + ]) + assert result.exit_code == 1 + assert "Unexpected error" in get_output(result) + assert "Traceback (most recent call last)" not in get_output(result) + + +def test_all_checks_skipped_warns_instead_of_crashing_on_fail_below(monkeypatch, fake_csv, tmp_path): + report_with_no_score = AuditReport( + dataset_path=fake_csv, rows=10, label_col="label", text_col="text", + audit_version="dev", timestamp="2026-01-01T00:00:00", + checks={"missing_values": CheckResult(score=None, warning="skipped")}, + ) + monkeypatch.setattr( + "inference_audit.cli.Auditor.audit", + lambda self, *a, **k: report_with_no_score, + ) + result = runner.invoke(app, [ + "run", fake_csv, + "--label-col", "label", "--text-col", "text", + "--output", str(tmp_path / "out.html"), + "--fail-below", "70", + ]) + assert "cannot compare against --fail-below" in result.stdout \ No newline at end of file diff --git a/tests/test_loader.py b/tests/test_loader.py new file mode 100644 index 0000000..29998a3 --- /dev/null +++ b/tests/test_loader.py @@ -0,0 +1,78 @@ +"""Tests for inference_audit/loader.py""" + +import pandas as pd +import pytest + +from inference_audit.loader import load_dataset + + +def _write_csv(tmp_path, filename, df): + path = tmp_path / filename + df.to_csv(path, index=False) + return str(path) + + +def test_file_not_found(): + with pytest.raises(FileNotFoundError): + load_dataset("does_not_exist.csv", "label", "text") + + +def test_unsupported_extension(tmp_path): + path = tmp_path / "data.txt" + path.write_text("not a real dataset") + with pytest.raises(ValueError, match="Unsupported file format"): + load_dataset(str(path), "label", "text") + + +def test_empty_dataset(tmp_path): + df = pd.DataFrame({"label": [], "text": []}) + path = _write_csv(tmp_path, "empty.csv", df) + with pytest.raises(ValueError, match="empty"): + load_dataset(path, "label", "text") + + +def test_missing_required_columns(tmp_path): + df = pd.DataFrame({"label": ["a", "b"], "other": ["x", "y"]}) + path = _write_csv(tmp_path, "missing_col.csv", df) + with pytest.raises(ValueError, match="not found in dataset"): + load_dataset(path, "label", "text") + + +def test_malformed_json_gives_clear_error(tmp_path): + path = tmp_path / "bad.json" + path.write_text("{not valid json at all") + with pytest.raises(ValueError, match="Could not parse"): + load_dataset(str(path), "label", "text") + + +def test_valid_csv_loads_correctly(tmp_path): + df = pd.DataFrame({"label": ["a", "b", "c"], "text": ["one", "two", "three"]}) + path = _write_csv(tmp_path, "valid.csv", df) + result = load_dataset(path, "label", "text") + assert len(result) == 3 + assert list(result["label"]) == ["a", "b", "c"] + + +def test_valid_json_loads_correctly(tmp_path): + path = tmp_path / "valid.json" + df = pd.DataFrame({"label": ["a", "b"], "text": ["one", "two"]}) + df.to_json(path, orient="records") + result = load_dataset(str(path), "label", "text") + assert len(result) == 2 + + +def test_valid_parquet_loads_correctly(tmp_path): + pytest.importorskip("pyarrow") + path = tmp_path / "valid.parquet" + df = pd.DataFrame({"label": ["a", "b"], "text": ["one", "two"]}) + df.to_parquet(path) + result = load_dataset(str(path), "label", "text") + assert len(result) == 2 + + +def test_column_names_with_whitespace_are_stripped(tmp_path): + df = pd.DataFrame({" label ": ["a", "b"], "text": ["one", "two"]}) + path = _write_csv(tmp_path, "whitespace_cols.csv", df) + result = load_dataset(path, "label", "text") + assert "label" in result.columns + assert " label " not in result.columns \ No newline at end of file diff --git a/tests/test_report.py b/tests/test_report.py new file mode 100644 index 0000000..84b385d --- /dev/null +++ b/tests/test_report.py @@ -0,0 +1,127 @@ +"""Tests for inference_audit/report.py""" + +import json + +import pytest + +from inference_audit.report import CheckResult, AuditReport + + +def _make_report(checks): + return AuditReport( + dataset_path="dataset.csv", + rows=100, + label_col="label", + text_col="text", + audit_version="dev", + timestamp="2026-01-01T00:00:00", + checks=checks, + ) + + +def test_check_result_defaults_details_to_empty_dict(): + result = CheckResult(score=90, warning=None) + assert result.details == {} + + +def test_overall_score_weights_label_distribution_and_near_duplicates_higher(): + # Two checks at 100, three at 0 -- label_distribution/near_duplicates + # weighted 1.5x, the other three at 1.0x. + checks = { + "label_distribution": CheckResult(score=100, warning=None), + "near_duplicates": CheckResult(score=100, warning=None), + "language_contamination": CheckResult(score=0, warning=None), + "missing_values": CheckResult(score=0, warning=None), + "annotation_consistency": CheckResult(score=0, warning=None), + } + report = _make_report(checks) + # weighted_sum = 100*1.5 + 100*1.5 + 0+0+0 = 300 + # total_weight = 1.5+1.5+1+1+1 = 6 + # 300/6 = 50 + assert report.overall_score == 50 + + +def test_overall_score_excludes_skipped_checks_not_zero(): + checks = { + "label_distribution": CheckResult(score=80, warning=None), + "near_duplicates": CheckResult(score=None, warning="skipped"), + "language_contamination": CheckResult(score=None, warning="skipped"), + "missing_values": CheckResult(score=None, warning="skipped"), + "annotation_consistency": CheckResult(score=None, warning="skipped"), + } + report = _make_report(checks) + # Only label_distribution counted -- weighted average of just itself. + assert report.overall_score == 80 + + +def test_overall_score_is_none_when_all_checks_skipped(): + checks = { + "label_distribution": CheckResult(score=None, warning="skipped"), + "near_duplicates": CheckResult(score=None, warning="skipped"), + "language_contamination": CheckResult(score=None, warning="skipped"), + "missing_values": CheckResult(score=None, warning="skipped"), + "annotation_consistency": CheckResult(score=None, warning="skipped"), + } + report = _make_report(checks) + assert report.overall_score is None + + +def test_to_json_writes_valid_json_with_correct_structure(tmp_path): + checks = { + "label_distribution": CheckResult( + score=61, warning="imbalance", details={"imbalance_ratio": 49.3} + ), + } + report = _make_report(checks) + output_path = tmp_path / "report.json" + report.to_json(str(output_path)) + + with open(output_path, encoding="utf-8") as f: + data = json.load(f) + + assert data["dataset"] == "dataset.csv" + assert data["overall_score"] == 61 + assert data["checks"]["label_distribution"]["score"] == 61 + # details are flattened into the check dict, not nested + assert data["checks"]["label_distribution"]["imbalance_ratio"] == 49.3 + + +def test_to_json_does_not_escape_non_ascii_text(tmp_path): + checks = { + "missing_values": CheckResult( + score=90, warning=None, details={"sample": "یہ اردو ہے"} + ), + } + report = _make_report(checks) + output_path = tmp_path / "report.json" + report.to_json(str(output_path)) + + raw_text = output_path.read_text(encoding="utf-8") + assert "یہ اردو ہے" in raw_text + assert "\\u0627" not in raw_text # would appear if ensure_ascii=True + + +def test_to_json_raises_clear_error_on_bad_path(): + checks = {"missing_values": CheckResult(score=90, warning=None)} + report = _make_report(checks) + with pytest.raises(OSError, match="Could not write JSON report"): + report.to_json("/nonexistent_dir_xyz/report.json") + + +def test_save_delegates_to_report_renderer(monkeypatch, tmp_path): + calls = {} + + def fake_render_html(report_obj, path): + calls["report"] = report_obj + calls["path"] = path + + import inference_audit.report_renderer as renderer_module + monkeypatch.setattr(renderer_module, "render_html", fake_render_html) + + checks = {"missing_values": CheckResult(score=90, warning=None)} + report = _make_report(checks) + output_path = str(tmp_path / "report.html") + report.save(output_path) + + assert calls["report"] is report + assert calls["path"] == output_path \ No newline at end of file diff --git a/tests/test_report_renderer.py b/tests/test_report_renderer.py new file mode 100644 index 0000000..4cd2a79 --- /dev/null +++ b/tests/test_report_renderer.py @@ -0,0 +1,93 @@ +"""Tests for inference_audit/report_renderer.py""" + +import base64 + +import pytest + +from inference_audit.report import AuditReport, CheckResult +from inference_audit.report_renderer import render_html, _build_score_chart + + +def _make_report(checks=None): + if checks is None: + checks = { + "label_distribution": CheckResult(score=61, warning="imbalance"), + "near_duplicates": CheckResult(score=90, warning=None), + "language_contamination": CheckResult(score=95, warning=None), + "missing_values": CheckResult(score=88, warning="some flagged"), + "annotation_consistency": CheckResult(score=None, warning="skipped"), + } + return AuditReport( + dataset_path="dataset.csv", + rows=100, + label_col="label", + text_col="text", + audit_version="dev", + timestamp="2026-01-01T00:00:00", + checks=checks, + ) + + +def test_build_score_chart_returns_valid_base64_png(): + report = _make_report() + chart_b64 = _build_score_chart(report.checks) + + # A valid PNG file starts with this exact byte signature once decoded. + decoded = base64.b64decode(chart_b64) + assert decoded[:8] == b"\x89PNG\r\n\x1a\n" + + +def test_build_score_chart_handles_skipped_checks_without_crashing(): + checks = { + "label_distribution": CheckResult(score=None, warning="skipped"), + "near_duplicates": CheckResult(score=None, warning="skipped"), + } + # Should not raise, even with every check skipped. + chart_b64 = _build_score_chart(checks) + assert len(chart_b64) > 0 + + +def test_render_html_writes_a_file(tmp_path): + report = _make_report() + output_path = tmp_path / "report.html" + render_html(report, str(output_path)) + + assert output_path.exists() + content = output_path.read_text(encoding="utf-8") + assert "dataset.csv" in content + + +def test_render_html_embeds_chart_as_base64_image(tmp_path): + report = _make_report() + output_path = tmp_path / "report.html" + render_html(report, str(output_path)) + + content = output_path.read_text(encoding="utf-8") + assert "data:image/png;base64," in content + + +def test_render_html_shows_skipped_check_distinctly(tmp_path): + report = _make_report() + output_path = tmp_path / "report.html" + render_html(report, str(output_path)) + + content = output_path.read_text(encoding="utf-8") + assert "Skipped" in content + + +def test_render_html_raises_clear_error_on_bad_path(): + report = _make_report() + with pytest.raises(OSError, match="Could not write HTML report"): + render_html(report, "/nonexistent_dir_xyz/report.html") + + +def test_render_html_has_no_external_dependencies(tmp_path): + """Design doc requirement: the HTML report must be self-contained, + no external CSS/JS links.""" + report = _make_report() + output_path = tmp_path / "report.html" + render_html(report, str(output_path)) + + content = output_path.read_text(encoding="utf-8") + assert " Date: Fri, 11 Sep 2026 14:47:36 +0500 Subject: [PATCH 05/12] optimized near_duplicates and language_contamination for real-scale runtime --- .../checks/language_contamination.py | 97 ++++++++---- inference_audit/checks/near_duplicates.py | 149 ++++++++++++------ 2 files changed, 173 insertions(+), 73 deletions(-) diff --git a/inference_audit/checks/language_contamination.py b/inference_audit/checks/language_contamination.py index 603e5fd..8f8a3fe 100644 --- a/inference_audit/checks/language_contamination.py +++ b/inference_audit/checks/language_contamination.py @@ -8,8 +8,28 @@ why it was abandoned: majority-vote over langdetect's essentially random guessing for Roman Urdu produced an unstable, uninterpretable score that swung 3x depending on dataset composition/size). + +PERFORMANCE (Week 5, Lead Engineer): unique-text caching alone (already +in place since Week 4) wasn't sufficient on the real 134K-row RUEmoCorp +corpus (~20 minutes), because most real rows are genuinely unique text +-- the cache has little duplication to exploit on this specific +dataset. Added: parallel detection across unique texts using a process +pool, since each text's detection has no dependency on any other. +Parameters and output shape are unchanged. Verified via +tests/test_performance_regression.py against the pre-parallel +implementation on all existing fixtures, including a determinism check +confirming the parallel path still produces identical results across +repeated runs. + +IMPORTANT: DetectorFactory.seed is process-global state, not shared +across process boundaries -- each worker process must set it +independently (see _init_worker below), or the reproducibility fix +from Week 1 is silently lost in the parallel path. """ +import os +from concurrent.futures import ProcessPoolExecutor + import pandas as pd from langdetect import detect_langs, DetectorFactory from langdetect.lang_detect_exception import LangDetectException @@ -22,6 +42,31 @@ DetectorFactory.seed = 0 +# Below this many unique texts, process-pool startup overhead costs +# more than it saves (relevant for small test fixtures, which have +# far fewer than this many unique rows) -- run sequentially instead. +# Only real, large datasets benefit from parallelizing. +_PARALLEL_THRESHOLD = 200 + + +def _init_worker(): + """Runs once per worker process on startup. Each worker needs the + seed set independently -- it does not inherit the parent process's + DetectorFactory state, since worker processes don't share memory + with the parent.""" + DetectorFactory.seed = 0 + + +def _detect_one(text: str): + """Runs detect_langs() on a single text. Must be a module-level + function (not a closure/lambda) so it can be pickled and sent to + worker processes.""" + try: + langs = detect_langs(text) + return (langs[0].lang, langs[0].prob) + except LangDetectException: + return None + def check_language_contamination( df: pd.DataFrame, @@ -38,27 +83,17 @@ def check_language_contamination( concern_languages: Iterable of ISO 639-1 codes to treat as contamination risks if confidently detected. Default from config.py - (LANGUAGE_CONCERN_LANGUAGES) -- see - that module for the documented rationale - (English/Hindi as realistic risks for a - Roman Urdu corpus). Moved to config per - review, so this can be tuned per-dataset - without touching check logic. + (LANGUAGE_CONCERN_LANGUAGES). confidence_threshold: Minimum langdetect confidence to trust a detection. Must be in [0, 1]. Default from config.py (LANGUAGE_CONFIDENCE_THRESHOLD). BOUNDARY: a detection with confidence EXACTLY equal to this threshold DOES - count as a match (>=, inclusive) -- made - explicit per review; see test suite for - a boundary-value test confirming this. + count as a match (>=, inclusive). KNOWN, BOUNDED LIMITATION: genuine Roman Urdu text can still be confidently misdetected specifically as English at a measured rate - (~11-12% on test data) -- see tests/test_language_contamination.py - for the documented baseline and tolerance around this figure. This - is a real characteristic of langdetect's behavior on Roman Urdu, - not something this check can fully eliminate. + (~11-12% on test data) -- see tests/test_language_contamination.py. Never raises. Returns CheckResult(score=None, ...) for: missing column, empty dataframe, invalid confidence_threshold, empty @@ -96,10 +131,9 @@ def check_language_contamination( ) # Performance: avoid the astype(str) copy when the column is - # already string-typed (per review -- "if this shows up in - # profiling"). Falls back to the safe full-cast path if a non-string - # value slips through an object-dtype column, rather than risking - # an AttributeError deep in detect_langs(). + # already string-typed. Falls back to the safe full-cast path if a + # non-string value slips through an object-dtype column, rather + # than risking an AttributeError deep in detect_langs(). if df[text_col].dtype == object: texts = df[text_col].fillna("") try: @@ -109,14 +143,22 @@ def check_language_contamination( else: texts = df[text_col].fillna("").astype(str) - detection_cache = {} - unique_texts = texts.unique() - for text in unique_texts: - try: - langs = detect_langs(text) - detection_cache[text] = (langs[0].lang, langs[0].prob) - except LangDetectException: - detection_cache[text] = None + # --- Improvement 1: run detect_langs() once per UNIQUE text, not + # once per row. Rows sharing exact text reuse the same result. --- + unique_texts = texts.unique().tolist() + + # --- Improvement 2: run those per-unique-text detections in + # parallel across CPU cores when there are enough of them to make + # the process-pool overhead worthwhile. Each detection is fully + # independent of every other, so this is embarrassingly parallel. --- + if len(unique_texts) >= _PARALLEL_THRESHOLD: + max_workers = min(32, (os.cpu_count() or 4)) + with ProcessPoolExecutor(max_workers=max_workers, initializer=_init_worker) as executor: + results = list(executor.map(_detect_one, unique_texts, chunksize=200)) + else: + results = [_detect_one(t) for t in unique_texts] + + detection_cache = dict(zip(unique_texts, results)) skipped_undetectable_count = 0 contamination_count = 0 @@ -129,9 +171,8 @@ def check_language_contamination( continue detectable_count += 1 lang, prob = result - # BOUNDARY (made explicit per review): >= , not > -- a - # detection exactly at the configured threshold counts as a - # confident match. See test suite for a boundary-value test. + # BOUNDARY: >= , not > -- a detection exactly at the configured + # threshold counts as a confident match. if lang in concern_set and prob >= confidence_threshold: contamination_count += 1 diff --git a/inference_audit/checks/near_duplicates.py b/inference_audit/checks/near_duplicates.py index 455b572..85868ad 100644 --- a/inference_audit/checks/near_duplicates.py +++ b/inference_audit/checks/near_duplicates.py @@ -4,18 +4,38 @@ Detects verbatim and near-verbatim duplicate samples using MinHash + LSH over character 3-grams. Threshold and preprocessing choices are backed by the Week 1 design doc findings (20 real RUEmoCorp pairs tested). + +PERFORMANCE (Week 5, Lead Engineer): the original implementation built +one MinHash per ROW, even when many rows share exact identical text -- +on a real 134K-row corpus this took ~6 minutes. This version builds one +MinHash per UNIQUE normalized text instead, and computes pair counts / +flagged rows arithmetically from group sizes rather than enumerating +every row pair. Parameters (threshold, num_perm) and output shape +(CheckResult schema) are unchanged -- this is a "do less redundant +work" optimization, not a behavior change. Verified via +tests/test_near_duplicates_regression.py against the original +row-by-row implementation on all existing fixtures before replacing it. """ import pandas as pd from datasketch import MinHash, MinHashLSH from inference_audit.report import CheckResult +MIN_SHINGLE_LENGTH = 3 + def _to_minhash(text: str, num_perm: int = 128) -> MinHash: + """ + Builds a MinHash signature from character 3-grams of lowercased text. + + Uses update_batch() instead of calling update() once per shingle -- + fewer individual Python-level calls for the same result. + """ m = MinHash(num_perm=num_perm) - text = text.lower().strip() - for i in range(len(text) - 2): - m.update(text[i:i + 3].encode("utf-8")) + encoded = text.encode("utf-8") + shingles = [encoded[i:i + 3] for i in range(len(encoded) - 2)] + if shingles: + m.update_batch(shingles) return m @@ -29,11 +49,11 @@ def check_near_duplicates( Flags near-duplicate rows using MinHash + LSH over character 3-grams. Never raises. Returns CheckResult(score=None, ...) for edge cases: - missing column, empty dataframe, fewer than 2 rows, or -- per Lead - Engineer review -- when EVERY row is too short to compare (zero - comparable rows after filtering is the same "nothing to measure" - situation as the single-row case, just reached a different way, and - must not silently report a perfect score). + missing column, empty dataframe, fewer than 2 rows, or when every + row is too short to compare (zero comparable rows after filtering + is the same "nothing to measure" situation as the single-row case, + just reached a different way -- must not silently report a perfect + score). """ if text_col not in df.columns: return CheckResult( @@ -57,53 +77,73 @@ def check_near_duplicates( details={"error": "insufficient_rows"}, ) - MIN_SHINGLE_LENGTH = 3 - lsh = MinHashLSH(threshold=threshold, num_perm=num_perm) - minhashes = {} texts = df[text_col].fillna("").astype(str) - skipped_short_count = 0 - - for idx, text in texts.items(): - if len(text.strip()) < MIN_SHINGLE_LENGTH: - skipped_short_count += 1 - continue - mh = _to_minhash(text, num_perm=num_perm) - minhashes[idx] = mh - lsh.insert(str(idx), mh) - - # Lead Engineer review (Week 4): if EVERY row was too short to build - # a meaningful signature, there are zero comparable rows -- this is - # the same "nothing to compare" situation as total_rows < 2, just - # reached via filtering instead of raw row count. Before this fix, - # falling through with an empty minhashes dict produced - # duplicate_rate=0/total_rows=0 -> score=100.0, silently reporting - # "perfectly clean" when nothing was actually measured. Caught via - # Khadija's test case: ["hi","ok","no","hi","ok"] (5 rows, all - # too short) incorrectly returned 100.0 instead of None. - if len(minhashes) == 0: + normalized = texts.str.lower().str.strip() + + comparable_mask = normalized.str.len() >= MIN_SHINGLE_LENGTH + skipped_short_count = int((~comparable_mask).sum()) + + if not comparable_mask.any(): return CheckResult( score=None, warning=( f"All {total_rows} rows were too short to compare " f"(< {MIN_SHINGLE_LENGTH} characters) — nothing could be measured." ), - details={"error": "all_rows_too_short", "total_rows": total_rows, "skipped_short_count": skipped_short_count}, + details={ + "error": "all_rows_too_short", + "total_rows": total_rows, + "skipped_short_count": skipped_short_count, + }, ) - candidate_pairs = set() - for idx, mh in minhashes.items(): - matches = lsh.query(mh) - for match_str in matches: - pair = tuple(sorted((str(idx), str(match_str)))) - if pair[0] != pair[1]: - candidate_pairs.add(pair) + # --- Group row indices by exact normalized text. Rows sharing exact + # text are duplicates of each other by definition (Jaccard = 1.0), + # so this avoids running MinHash/LSH between them at all. --- + groups = {} + for idx, norm_text in zip(df.index[comparable_mask], normalized[comparable_mask]): + groups.setdefault(norm_text, []).append(str(idx)) - pair_count = len(candidate_pairs) + unique_texts = list(groups.keys()) + + # --- One MinHash per unique text, not per row. --- + lsh = MinHashLSH(threshold=threshold, num_perm=num_perm) + minhash_cache = {} + for u_text in unique_texts: + mh = _to_minhash(u_text, num_perm=num_perm) + minhash_cache[u_text] = mh + lsh.insert(u_text, mh) + + # --- Find near-duplicate matches BETWEEN distinct unique texts. --- + unique_text_pairs = set() + for u_text in unique_texts: + for match in lsh.query(minhash_cache[u_text]): + if match != u_text: + unique_text_pairs.add(tuple(sorted((u_text, match)))) + + # --- Compute pair_count and flagged rows arithmetically from group + # sizes, rather than enumerating every individual row pair. This + # reproduces the same counts the original row-by-row version would + # have produced (exact-duplicate groups of size k contribute + # C(k,2) pairs; a near-duplicate match between two unique texts + # contributes one pair per row-combination between them). --- + pair_count = 0 + flagged_unique_texts = set() + + for u_text, indices in groups.items(): + k = len(indices) + if k > 1: + pair_count += k * (k - 1) // 2 + flagged_unique_texts.add(u_text) + + for u1, u2 in unique_text_pairs: + pair_count += len(groups[u1]) * len(groups[u2]) + flagged_unique_texts.add(u1) + flagged_unique_texts.add(u2) flagged_row_ids = set() - for a, b in candidate_pairs: - flagged_row_ids.add(a) - flagged_row_ids.add(b) + for u_text in flagged_unique_texts: + flagged_row_ids.update(groups[u_text]) duplicate_rate = len(flagged_row_ids) / total_rows score = round(max(0.0, 100 * (1 - duplicate_rate)), 2) @@ -121,8 +161,26 @@ def check_near_duplicates( ) warning = " ".join(warning_parts) if warning_parts else None + # Example pairs for the report -- best-effort selection, capped at 5. + # NOTE: not guaranteed to select the identical example rows the + # original set-based implementation would have shown (that version + # iterated an unordered Python set, so its own examples were never + # deterministic either). See regression test for what IS guaranteed + # to match exactly: score, counts, and rates. example_pairs = [] - for a, b in list(candidate_pairs)[:5]: + for u_text, indices in groups.items(): + if len(example_pairs) >= 5: + break + if len(indices) > 1: + a, b = indices[0], indices[1] + example_pairs.append({ + "row_a": texts.loc[int(a) if a.isdigit() else a][:80], + "row_b": texts.loc[int(b) if b.isdigit() else b][:80], + }) + for u1, u2 in unique_text_pairs: + if len(example_pairs) >= 5: + break + a, b = groups[u1][0], groups[u2][0] example_pairs.append({ "row_a": texts.loc[int(a) if a.isdigit() else a][:80], "row_b": texts.loc[int(b) if b.isdigit() else b][:80], @@ -130,6 +188,7 @@ def check_near_duplicates( details = { "total_rows": total_rows, + "unique_text_count": len(unique_texts), "candidate_pair_count": pair_count, "flagged_row_count": len(flagged_row_ids), "skipped_short_count": skipped_short_count, @@ -139,4 +198,4 @@ def check_near_duplicates( "example_pairs": example_pairs, } - return CheckResult(score=score, warning=warning, details=details) + return CheckResult(score=score, warning=warning, details=details) \ No newline at end of file From 89566971f635d1b7dc16d8b13b992a931450a75a Mon Sep 17 00:00:00 2001 From: Khadija Date: Fri, 11 Sep 2026 15:12:13 +0500 Subject: [PATCH 06/12] Updated sample audit report --- audit_report.html | 36 ++++++++++++++++++++++++------------ 1 file changed, 24 insertions(+), 12 deletions(-) diff --git a/audit_report.html b/audit_report.html index 953d97f..98f2a1a 100644 --- a/audit_report.html +++ b/audit_report.html @@ -115,7 +115,7 @@

inference-audit Report

Dataset: C:\Users\User\Downloads\ruemocorp_silver.csv  ·  134053 rows  ·  - Generated 2026-09-05T16:56:41.250442  ·  + Generated 2026-09-11T10:09:48.365258  ·  vdev
@@ -123,7 +123,7 @@

inference-audit Report

-
76
+
78

Overall Score (weighted average across all checks)

@@ -187,12 +187,12 @@

Check Results

near duplicates - 56.24 + 56.56 - 46180 candidate duplicate pair(s) found among 134053 rows (58665 rows involved). 247 row(s) skipped (text shorter than 3 characters -- too short to compare reliably). + 45364 candidate duplicate pair(s) found among 134053 rows (58232 rows involved). 247 row(s) skipped (text shorter than 3 characters -- too short to compare reliably). @@ -204,19 +204,21 @@

Check Results

total rows: 134053  ·  - candidate pair count: 46180  ·  + unique text count: 106031  ·  - flagged row count: 58665  ·  + candidate pair count: 45364  ·  + + flagged row count: 58232  ·  skipped short count: 247  ·  - duplicate rate: 0.4376  ·  + duplicate rate: 0.4344  ·  threshold: 0.8  ·  num perm: 128  ·  - example pairs: [{'row_a': 'Us waqat tak naqad unhen aik filmi showpiece hi shumar kar rahe the', 'row_b': 'Us waqat tak naqad unhen aik filmi showpiece hi shumar kar rahe the'}, {'row_a': 'Nice', 'row_b': 'Nice'}, {'row_a': 'darbari', 'row_b': 'darbari'}, {'row_a': 'hahahaha', 'row_b': 'Hahah'}, {'row_a': 'na krn na lala ap to kbi ksi ko nh rulaty aj kyn zid kr rhy . ;(', 'row_b': 'na krn na lala ap to kbi ksi ko nh rulaty aj kyn zid kr rhy . ;('}] + example pairs: [{'row_a': 'imran khan ke lye bahut acha hoga', 'row_b': 'imran khan ke lye bahut acha hoga'}, {'row_a': 'bohut achay kamal ki sharing hai peer o murshid ki asliat ahista ahista khul kr ', 'row_b': 'bohut achay kamal ki sharing hai peer o murshid ki asliat ahista ahista khul kr '}, {'row_a': 'uffffffffffffffff imran abbas mri jan\ufeff kya baat ha aap ki', 'row_b': 'uffffffffffffffff imran abbas mri jan\ufeff kya baat ha aap ki'}, {'row_a': 'dramain dekh dekh k aap bhi writer ban gai ho ..', 'row_b': 'dramain dekh dekh k aap bhi writer ban gai ho ..'}, {'row_a': 'gawadar kha hai', 'row_b': 'Gawadar kha hai'}] @@ -305,12 +307,12 @@

Check Results

annotation consistency - — + 87.83 - Skipped — No confidence column provided — annotation consistency check skipped. + 16320/134053 annotated samples (12.2%) have confidence below 0.6. @@ -320,7 +322,17 @@

Check Results

- error: no_confidence_column + total rows: 134053  ·  + + non null count: 134053  ·  + + null count: 0  ·  + + low confidence count: 16320  ·  + + low confidence rate: 0.1217  ·  + + confidence threshold: 0.6
@@ -334,7 +346,7 @@

Check Results

Score Overview

- Bar chart of per-check scores From f46d6409bdf0c67183c5bd24b807d8abd5382ee9 Mon Sep 17 00:00:00 2001 From: Khadija Date: Sat, 12 Sep 2026 15:03:22 +0500 Subject: [PATCH 07/12] Updated sample_report.html and sample_report.json --- Sample_report.html | 361 +++++++++++++++++++++++++++++++++++++++++++++ Sample_report.json | 101 +++++++++++++ 2 files changed, 462 insertions(+) create mode 100644 Sample_report.html create mode 100644 Sample_report.json diff --git a/Sample_report.html b/Sample_report.html new file mode 100644 index 0000000..98f2a1a --- /dev/null +++ b/Sample_report.html @@ -0,0 +1,361 @@ + + + + + + inference-audit Report — C:\Users\User\Downloads\ruemocorp_silver.csv + + + + +
+

inference-audit Report

+
+ Dataset: C:\Users\User\Downloads\ruemocorp_silver.csv  ·  + 134053 rows  ·  + Generated 2026-09-11T10:09:48.365258  ·  + vdev +
+
+ +
+ + +
78
+ + +

Overall Score (weighted average across all checks)

+
+ +
+

Check Results

+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
CheckScoreNotes
label distribution + + 68.64 + + + + Class imbalance detected: 'joy' has 28389 samples, 'surprise' has only 4080 (ratio 7.0:1). + +
+
+ + total rows: 134053  ·  + + num classes: 7  ·  + + class counts: {'joy': 28389, 'none': 28167, 'disgust': 25959, 'sadness': 22570, 'anger': 18275, 'fear': 6613, 'surprise': 4080}  ·  + + majority count: 28389  ·  + + minority count: 4080  ·  + + imbalance ratio: 6.96  ·  + + max acceptable ratio: 20.0  ·  + + null count: 0  ·  + + null rate: 0.0 + +
+
near duplicates + + 56.56 + + + + 45364 candidate duplicate pair(s) found among 134053 rows (58232 rows involved). 247 row(s) skipped (text shorter than 3 characters -- too short to compare reliably). + +
+
+ + total rows: 134053  ·  + + unique text count: 106031  ·  + + candidate pair count: 45364  ·  + + flagged row count: 58232  ·  + + skipped short count: 247  ·  + + duplicate rate: 0.4344  ·  + + threshold: 0.8  ·  + + num perm: 128  ·  + + example pairs: [{'row_a': 'imran khan ke lye bahut acha hoga', 'row_b': 'imran khan ke lye bahut acha hoga'}, {'row_a': 'bohut achay kamal ki sharing hai peer o murshid ki asliat ahista ahista khul kr ', 'row_b': 'bohut achay kamal ki sharing hai peer o murshid ki asliat ahista ahista khul kr '}, {'row_a': 'uffffffffffffffff imran abbas mri jan\ufeff kya baat ha aap ki', 'row_b': 'uffffffffffffffff imran abbas mri jan\ufeff kya baat ha aap ki'}, {'row_a': 'dramain dekh dekh k aap bhi writer ban gai ho ..', 'row_b': 'dramain dekh dekh k aap bhi writer ban gai ho ..'}, {'row_a': 'gawadar kha hai', 'row_b': 'Gawadar kha hai'}] + +
+
language contamination + + 90.69 + + + + 12438/133670 detectable rows (9.3%) confidently detected as a concern-list language (en, hi). 383 row(s) skipped (text too short/empty for language detection). + +
+
+ + total rows: 134053  ·  + + unique text count: 109191  ·  + + detectable count: 133670  ·  + + skipped undetectable count: 383  ·  + + contamination count: 12438  ·  + + contamination rate: 0.0931  ·  + + concern languages: ['en', 'hi']  ·  + + confidence threshold: 0.9 + +
+
missing values + + 99.82 + + + + 247/134053 rows (0.2%) flagged: 130 null, 0 whitespace-only, 117 below 3-character threshold. + +
+
+ + total rows: 134053  ·  + + flagged count: 247  ·  + + null count: 130  ·  + + whitespace only count: 0  ·  + + too short count: 117  ·  + + missing rate: 0.0018  ·  + + min length threshold: 3 + +
+
annotation consistency + + 87.83 + + + + 16320/134053 annotated samples (12.2%) have confidence below 0.6. + +
+
+ + total rows: 134053  ·  + + non null count: 134053  ·  + + null count: 0  ·  + + low confidence count: 16320  ·  + + low confidence rate: 0.1217  ·  + + confidence threshold: 0.6 + +
+
+
+ +
+

Score Overview

+ + Bar chart of per-check scores + +
+ +
+ Generated by inference-audit vdev · Column mapping: + label_col="predicted_emotion", text_col="message" +
+ + + \ No newline at end of file diff --git a/Sample_report.json b/Sample_report.json new file mode 100644 index 0000000..4e4b3b2 --- /dev/null +++ b/Sample_report.json @@ -0,0 +1,101 @@ +{ + "inference_audit_version": "dev", + "dataset": "C:\\Users\\User\\Downloads\\ruemocorp_silver.csv", + "rows": 134053, + "label_col": "predicted_emotion", + "text_col": "message", + "audit_timestamp": "2026-09-05T17:23:07.654317", + "overall_score": 78, + "checks": { + "label_distribution": { + "score": 68.64, + "warning": "Class imbalance detected: 'joy' has 28389 samples, 'surprise' has only 4080 (ratio 7.0:1).", + "total_rows": 134053, + "num_classes": 7, + "class_counts": { + "joy": 28389, + "none": 28167, + "disgust": 25959, + "sadness": 22570, + "anger": 18275, + "fear": 6613, + "surprise": 4080 + }, + "majority_count": 28389, + "minority_count": 4080, + "imbalance_ratio": 6.96, + "max_acceptable_ratio": 20.0, + "null_count": 0, + "null_rate": 0.0 + }, + "near_duplicates": { + "score": 56.24, + "warning": "46180 candidate duplicate pair(s) found among 134053 rows (58665 rows involved). 247 row(s) skipped (text shorter than 3 characters -- too short to compare reliably).", + "total_rows": 134053, + "candidate_pair_count": 46180, + "flagged_row_count": 58665, + "skipped_short_count": 247, + "duplicate_rate": 0.4376, + "threshold": 0.8, + "num_perm": 128, + "example_pairs": [ + { + "row_a": "good", + "row_b": "Good" + }, + { + "row_a": "munir hussain ne apne ashron par mushtamil taweel filmy career mein films ke liy", + "row_b": "Munir Hussain ne apne 4 ashron par mushtamil taweel filmy career mein 163 films " + }, + { + "row_a": "jaisayy program ( kon banay gaa meera ka pati ) main drama chala same waisa hi y", + "row_b": "jaisayy program ( kon banay gaa meera ka pati ) main drama chala same waisa hi y" + }, + { + "row_a": "shukria bhai", + "row_b": "shukria bhai" + }, + { + "row_a": "yeh mqm ka bhi pithu hai aur noon league ka bhi mulazim", + "row_b": "yeh mqm ka bhi pithu hai aur noon league ka bhi mulazim" + } + ] + }, + "language_contamination": { + "score": 90.69, + "warning": "12438/133670 detectable rows (9.3%) confidently detected as a concern-list language (en, hi). 383 row(s) skipped (text too short/empty for language detection).", + "total_rows": 134053, + "unique_text_count": 109191, + "detectable_count": 133670, + "skipped_undetectable_count": 383, + "contamination_count": 12438, + "contamination_rate": 0.0931, + "concern_languages": [ + "en", + "hi" + ], + "confidence_threshold": 0.9 + }, + "missing_values": { + "score": 99.82, + "warning": "247/134053 rows (0.2%) flagged: 130 null, 0 whitespace-only, 117 below 3-character threshold.", + "total_rows": 134053, + "flagged_count": 247, + "null_count": 130, + "whitespace_only_count": 0, + "too_short_count": 117, + "missing_rate": 0.0018, + "min_length_threshold": 3 + }, + "annotation_consistency": { + "score": 87.83, + "warning": "16320/134053 annotated samples (12.2%) have confidence below 0.6.", + "total_rows": 134053, + "non_null_count": 134053, + "null_count": 0, + "low_confidence_count": 16320, + "low_confidence_rate": 0.1217, + "confidence_threshold": 0.6 + } + } +} \ No newline at end of file From 1142bb001d99d42a62b7889d7edfd4164f61479d Mon Sep 17 00:00:00 2001 From: Khadija Date: Sat, 12 Sep 2026 15:56:49 +0500 Subject: [PATCH 08/12] add missing __init__.py to checks/ subpackage for explicit package structure --- inference_audit/checks/__init__.py | 0 1 file changed, 0 insertions(+), 0 deletions(-) create mode 100644 inference_audit/checks/__init__.py diff --git a/inference_audit/checks/__init__.py b/inference_audit/checks/__init__.py new file mode 100644 index 0000000..e69de29 From 773a6c9250007a433db4a74999913941aaac6e9b Mon Sep 17 00:00:00 2001 From: Khadija Date: Sat, 12 Sep 2026 16:22:27 +0500 Subject: [PATCH 09/12] gitignore build artifacts from local editable install (*.egg-info, build/) --- .gitignore | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.gitignore b/.gitignore index 5364276..2d0d6f7 100644 --- a/.gitignore +++ b/.gitignore @@ -7,3 +7,6 @@ venv/ htmlcov/ coverage_report.xml .coverage + +*.egg-info/ +build/ From 10ba83106fd9e2a58f1c2b0d39ff0d8ad1e0edbc Mon Sep 17 00:00:00 2001 From: Khadija Date: Sat, 12 Sep 2026 16:40:11 +0500 Subject: [PATCH 10/12] fix: rename Pyproject.toml to pyproject.toml (case-sensitive filename required by build tools on Linux CI) --- Pyproject.toml => pyproject.toml | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename Pyproject.toml => pyproject.toml (100%) diff --git a/Pyproject.toml b/pyproject.toml similarity index 100% rename from Pyproject.toml rename to pyproject.toml From 1b96d252a8e98c7a4fd7406c99e4ff788c352eb4 Mon Sep 17 00:00:00 2001 From: Khadija Date: Sat, 12 Sep 2026 16:53:37 +0500 Subject: [PATCH 11/12] fix: handle packages_distributions unavailable on Python <3.10 at import time, not just call time - previous try/except never ran since the top-level import itself failed on 3.9 --- inference_audit/report.py | 28 +++++++++++++++------------- 1 file changed, 15 insertions(+), 13 deletions(-) diff --git a/inference_audit/report.py b/inference_audit/report.py index 9b4d4bb..8f34f60 100644 --- a/inference_audit/report.py +++ b/inference_audit/report.py @@ -168,25 +168,27 @@ def save(self, path: str) -> None: f"Check that the folder exists and is writable." ) from e -from importlib.metadata import packages_distributions, version, PackageNotFoundError +from importlib.metadata import version, PackageNotFoundError + +try: + from importlib.metadata import packages_distributions +except ImportError: + # packages_distributions() was added in Python 3.10 -- on 3.9 it + # doesn't exist in importlib.metadata at all, so the import itself + # fails, not just a call to the function. Handling this at import + # time (not inside a try/except around the function call) is what + # actually matters here -- a try/except around the call never gets + # reached if the module-level import already crashed. + packages_distributions = None + def _get_audit_version() -> str: - """ - Reads the installed package version by looking up whatever - distribution name currently provides the `inference_audit` import - package -- rather than hardcoding a specific PyPI name. This means - the code doesn't need to change if the PyPI distribution name is - ever renamed (e.g. once the naming conflict is resolved). - """ + if packages_distributions is None: + return "dev" try: distributions = packages_distributions().get("inference_audit", []) if distributions: return version(distributions[0]) - except AttributeError: - # packages_distributions() was added in Python 3.10 -- on 3.9, - # this attribute won't exist. Fall through to "dev". - pass except PackageNotFoundError: pass - return "dev" \ No newline at end of file From 40b98281d69f1c1ac3876d4edc653398be876cbd Mon Sep 17 00:00:00 2001 From: Khadija Date: Sat, 12 Sep 2026 17:36:56 +0500 Subject: [PATCH 12/12] Updated readm.md --- README.md | 82 +++++++++++++++++++++++++++++++++++++++++++++++-------- 1 file changed, 70 insertions(+), 12 deletions(-) diff --git a/README.md b/README.md index 1f18e3d..4a12464 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,6 @@ # inference-audit -An NLP dataset quality auditor. Run one command, get a reproducible report covering five quality checks and an overall score — built so a dataset that scores well is something you can actually cite as evidence of quality, not just a number. +An NLP dataset quality auditor. Run one command, get a reproducible report covering five quality checks and an overall score.This pip installable python library is built so a dataset that scores well is something you can actually cite as evidence of quality, not just a number. ``` pip install inference-audit-pk @@ -8,11 +8,13 @@ pip install inference-audit-pk ## Why this exists -Every stage of an NLP pipeline — publishing a dataset, choosing one for training — depends on quality that's usually checked with a one-off script written from scratch each time, producing results that aren't comparable across projects or papers. `inference-audit` replaces that with a standard, versioned, reproducible check anyone can run and cite. +Every stage of an NLP pipeline from publishing a dataset, choosing one for training , depends on quality that's usually checked with a one-off script written from scratch each time, producing results that aren't comparable across projects or papers. `inference-audit` replaces that with a standard, versioned, reproducible check anyone can run and cite. ## Quickstart -### Python +### Default usage + +The simplest case — audits a dataset using the five checks with default settings. `annotation_consistency` will be skipped (not scored zero) since no confidence column is provided. ```python from inference_audit import Auditor @@ -24,12 +26,37 @@ report.save("audit_report.html") # human-readable, self-contained HTML report.to_json("audit_report.json") # machine-readable, for CI pipelines ``` -### CLI +```bash +inference-audit run dataset.csv \ + --label-col emotion \ + --text-col text \ + --output audit_report.html +``` + +PowerShell: +```powershell +inference-audit run dataset.csv ` + --label-col emotion ` + --text-col text ` + --output audit_report.html +``` + +### With a confidence column + +If your dataset includes an annotation confidence or agreement score per row, pass it in to enable the `annotation_consistency` check — otherwise this check is silently skipped rather than scored. + +```python +report = auditor.audit( + "dataset.csv", label_col="emotion", text_col="text", + conf_col="confidence", +) +``` ```bash inference-audit run dataset.csv \ --label-col emotion \ --text-col text \ + --conf-col confidence \ --output audit_report.html ``` @@ -38,10 +65,41 @@ PowerShell: inference-audit run dataset.csv ` --label-col emotion ` --text-col text ` + --conf-col confidence ` --output audit_report.html ``` -Use `--fail-below ` to make the command exit with a non-zero status if the dataset doesn't meet a quality bar — this is what makes it usable as a CI gate on a pull request: +### With a custom concern-languages list + +`language_contamination` defaults to flagging English/Hindi as contamination risks. If your dataset has a different realistic contamination risk (e.g. an English dataset where French or Spanish leakage is the actual concern), override the default rather than relying on it. + +```python +report = auditor.audit( + "dataset.csv", label_col="label", text_col="text", + concern_languages=("fr", "es"), +) +``` + +```bash +inference-audit run dataset.csv \ + --label-col label \ + --text-col text \ + --concern-languages fr,es \ + --output audit_report.html +``` + +PowerShell: +```powershell +inference-audit run dataset.csv ` + --label-col label ` + --text-col text ` + --concern-languages fr,es ` + --output audit_report.html +``` + +### CI quality gate + +Use `--fail-below ` to make the command exit with a non-zero status if the dataset doesn't meet a quality bar — this is what makes it usable as a CI gate on a pull request. ```bash inference-audit run dataset.csv --label-col emotion --text-col text --fail-below 70 @@ -53,7 +111,7 @@ inference-audit run dataset.csv --label-col emotion --text-col text --fail-below **Near duplicates** — flags verbatim and near-verbatim repeated samples using MinHash/LSH similarity, including cases where the same text appears under two different labels (a common annotation error). -**Language contamination** — flags text confidently detected in a language your dataset isn't supposed to contain. Checks against a configurable list of "concern languages" (default: English and Hindi, reflecting realistic contamination risk for Roman Urdu corpora) rather than trying to guess the dataset's "true" language, since off-the-shelf language detection has no reliable profile for Roman Urdu. +**Language contamination** — flags text confidently detected in a language your dataset isn't supposed to contain. Checks against a configurable list of "concern languages" (default: English and Hindi) rather than trying to guess the dataset's "true" language, since off-the-shelf language detection has no reliable profile for every language — Roman Urdu being one notable example. Override `concern_languages`/`--concern-languages` for datasets with a different realistic contamination risk. **Missing values** — flags null, whitespace-only, and suspiciously short text samples that likely don't carry real content. @@ -75,9 +133,9 @@ Each check returns a score (0–100), an optional warning, and detailed metrics. ## Known Limitations -- **Language detection has no real signal for Roman Urdu.** The underlying detection library has no language profile for it, so `language_contamination` doesn't attempt to identify "the dataset's language" — it only flags confident detections of specific languages on a configurable concern list. Genuine Roman Urdu text can still occasionally be misdetected as a concern-list language (measured false-positive rate: roughly 10–12% on real test data) — this is a bounded, documented limitation, not a bug. -- **Near-duplicate detection performance** on very large datasets (100K+ rows) can approach the tool's own time budget; if you're auditing a very large corpus, expect this to be the slowest of the five checks. -- **`--concern-languages` defaults to English/Hindi**, chosen for the tool's primary use case (Roman Urdu corpora). If you're auditing a dataset with different realistic contamination risks, override this flag rather than relying on the default. +- **Language detection has no real signal for some languages.** The underlying detection library has no language profile for some languages (Roman Urdu being a notable example), so `language_contamination` doesn't attempt to identify "the dataset's language" — it only flags confident detections of specific languages on a configurable concern list. Text in an undetectable language can still occasionally be misdetected as a concern-list language (measured false-positive rate: roughly 10–12% on real Roman Urdu test data) — this is a bounded, documented limitation, not a bug. +- **Performance on very large datasets.** Recent optimizations (deduplication before comparison, parallelized language detection) significantly improved runtime on large datasets, but `near_duplicates` and `language_contamination` remain the two most compute-intensive checks. On datasets over 100K rows, a full audit can still take up to several minutes (up to ~5 minutes observed on a 100K+ row real-world dataset) depending on text diversity and duplication rate. If you're auditing a very large corpus, expect these two checks to dominate total runtime. +- **`--concern-languages` defaults to English/Hindi.** This is a reasonable general-purpose default, particularly for code-switched or Roman-script corpora where English/Hindi leakage is a common risk — but it is not universally correct. If you're auditing a dataset with a different realistic contamination risk, override this flag rather than relying on the default. ## Example Output @@ -93,7 +151,7 @@ missing_values: 88 annotation_consistency: skipped (no confidence column provided) ``` -See `report.html` in this repository for a full rendered example, including the score chart. +See `sample_report.html` and `sample_report.json` in this repository for full rendered examples, including the score chart. ## Requirements @@ -119,5 +177,5 @@ MIT — see [LICENSE](LICENSE). Built as part of the Inference Lab Engineering Fellowship, Cohort 01 — Project C. -- Khadija Faisal (Lead Engineer) -- Muhammad Shoaib Altaf (Research & Implementation Engineer) \ No newline at end of file +- **Khadija Faisal** (Lead Engineer) — [GitHub](https://github.com/khadijja1) · [LinkedIn](https://www.linkedin.com/in/khadijjafaisal) +- **Muhammad Shoaib Altaf** (Research & Implementation Engineer) — [GitHub](https://github.com/Shoaib-Altaf) · [LinkedIn](https://www.linkedin.com/in/muhammad-shoaib-altaf-6ab3a8326) \ No newline at end of file