From 9e593efbac16bac7bcf98147e693714eb7060e24 Mon Sep 17 00:00:00 2001 From: Khadija Faisal Date: Sat, 26 Sep 2026 14:51:29 +0500 Subject: [PATCH 1/3] Add badges to README for project visibility Added badges for PyPI version, Python version, CI status, and license to README. --- README.md | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index 4a12464..87c110f 100644 --- a/README.md +++ b/README.md @@ -1,3 +1,7 @@ +[![PyPI](https://img.shields.io/pypi/v/inference-audit-pk)](https://pypi.org/project/inference-audit-pk/) +[![Python](https://img.shields.io/pypi/pyversions/inference-audit-pk)](https://pypi.org/project/inference-audit-pk/) +[![CI](https://github.com/Inference-LAB/inference-audit/actions/workflows/ci.yml/badge.svg)](https://github.com/Inference-LAB/inference-audit/actions) +[![License: MIT](https://img.shields.io/badge/License-MIT-yellow.svg)](LICENSE) # inference-audit An NLP dataset quality auditor. Run one command, get a reproducible report covering five quality checks and an overall score.This pip installable python library is built so a dataset that scores well is something you can actually cite as evidence of quality, not just a number. @@ -178,4 +182,4 @@ MIT — see [LICENSE](LICENSE). Built as part of the Inference Lab Engineering Fellowship, Cohort 01 — Project C. - **Khadija Faisal** (Lead Engineer) — [GitHub](https://github.com/khadijja1) · [LinkedIn](https://www.linkedin.com/in/khadijjafaisal) -- **Muhammad Shoaib Altaf** (Research & Implementation Engineer) — [GitHub](https://github.com/Shoaib-Altaf) · [LinkedIn](https://www.linkedin.com/in/muhammad-shoaib-altaf-6ab3a8326) \ No newline at end of file +- **Muhammad Shoaib Altaf** (Research & Implementation Engineer) — [GitHub](https://github.com/Shoaib-Altaf) · [LinkedIn](https://www.linkedin.com/in/muhammad-shoaib-altaf-6ab3a8326) From 72992cc9cd022a011cef79e21d46f3eb12fd3b2d Mon Sep 17 00:00:00 2001 From: Khadija Date: Sat, 26 Sep 2026 16:30:58 +0500 Subject: [PATCH 2/3] fix: clear error for Windows multiprocessing guard issue; add missing 3.12 classifier; update README examples (0.1.1) --- CHANGELOG.md | 14 ++++++ README.md | 47 +++++++++++++------ .../checks/language_contamination.py | 15 +++++- pyproject.toml | 3 +- 4 files changed, 62 insertions(+), 17 deletions(-) create mode 100644 CHANGELOG.md diff --git a/CHANGELOG.md b/CHANGELOG.md new file mode 100644 index 0000000..6ecc044 --- /dev/null +++ b/CHANGELOG.md @@ -0,0 +1,14 @@ +# Changelog + +## [0.1.1] - 2026-09-26 + +### Fixed +- Clearer error message when `Auditor.audit()` is called from a top-level + script on Windows without an `if __name__ == "__main__":` guard, instead + of an unreadable recursive multiprocessing crash. +- Added missing `Programming Language :: Python :: 3.12` classifier + (CI already tested against 3.12; PyPI metadata was incomplete). + +### Documentation +- README Python examples now include the required `__main__` guard, with + an explanation of why it's needed. \ No newline at end of file diff --git a/README.md b/README.md index 87c110f..16fb053 100644 --- a/README.md +++ b/README.md @@ -1,10 +1,11 @@ +# inference-audit + [![PyPI](https://img.shields.io/pypi/v/inference-audit-pk)](https://pypi.org/project/inference-audit-pk/) [![Python](https://img.shields.io/pypi/pyversions/inference-audit-pk)](https://pypi.org/project/inference-audit-pk/) [![CI](https://github.com/Inference-LAB/inference-audit/actions/workflows/ci.yml/badge.svg)](https://github.com/Inference-LAB/inference-audit/actions) [![License: MIT](https://img.shields.io/badge/License-MIT-yellow.svg)](LICENSE) -# inference-audit -An NLP dataset quality auditor. Run one command, get a reproducible report covering five quality checks and an overall score.This pip installable python library is built so a dataset that scores well is something you can actually cite as evidence of quality, not just a number. +An NLP dataset quality auditor. Run one command, get a reproducible report covering five quality checks and an overall score. This pip installable python library is built so a dataset that scores well is something you can actually cite as evidence of quality, not just a number. ``` pip install inference-audit-pk @@ -14,6 +15,10 @@ pip install inference-audit-pk Every stage of an NLP pipeline from publishing a dataset, choosing one for training , depends on quality that's usually checked with a one-off script written from scratch each time, producing results that aren't comparable across projects or papers. `inference-audit` replaces that with a standard, versioned, reproducible check anyone can run and cite. +## A note on the Python examples below + +Every Python example in this README is wrapped in `if __name__ == "__main__":`. This is required on Windows, not a style choice: `language_contamination` uses a process pool internally for performance, and Windows has no `fork()`, so it re-imports your entry script to create each worker process. Without the guard, that re-import re-triggers your top-level code, causing a runtime error. If you're calling `Auditor.audit()` from inside a function (e.g. in a larger application) rather than a bare top-level script, the guard is not needed there — it only matters for code sitting directly at module level in the file you run. + ## Quickstart ### Default usage @@ -23,11 +28,12 @@ The simplest case — audits a dataset using the five checks with default settin ```python from inference_audit import Auditor -auditor = Auditor() -report = auditor.audit("dataset.csv", label_col="emotion", text_col="text") +if __name__ == "__main__": + auditor = Auditor() + report = auditor.audit("dataset.csv", label_col="emotion", text_col="text") -report.save("audit_report.html") # human-readable, self-contained HTML -report.to_json("audit_report.json") # machine-readable, for CI pipelines + report.save("audit_report.html") # human-readable, self-contained HTML + report.to_json("audit_report.json") # machine-readable, for CI pipelines ``` ```bash @@ -50,10 +56,15 @@ inference-audit run dataset.csv ` If your dataset includes an annotation confidence or agreement score per row, pass it in to enable the `annotation_consistency` check — otherwise this check is silently skipped rather than scored. ```python -report = auditor.audit( - "dataset.csv", label_col="emotion", text_col="text", - conf_col="confidence", -) +from inference_audit import Auditor + +if __name__ == "__main__": + auditor = Auditor() + report = auditor.audit( + "dataset.csv", label_col="emotion", text_col="text", + conf_col="confidence", + ) + report.save("audit_report.html") ``` ```bash @@ -78,10 +89,15 @@ inference-audit run dataset.csv ` `language_contamination` defaults to flagging English/Hindi as contamination risks. If your dataset has a different realistic contamination risk (e.g. an English dataset where French or Spanish leakage is the actual concern), override the default rather than relying on it. ```python -report = auditor.audit( - "dataset.csv", label_col="label", text_col="text", - concern_languages=("fr", "es"), -) +from inference_audit import Auditor + +if __name__ == "__main__": + auditor = Auditor() + report = auditor.audit( + "dataset.csv", label_col="label", text_col="text", + concern_languages=("fr", "es"), + ) + report.save("audit_report.html") ``` ```bash @@ -135,11 +151,14 @@ Each check returns a score (0–100), an optional warning, and detailed metrics. | `--concern-languages` | No | Comma-separated ISO 639-1 codes to flag if detected (default: `en,hi`) | | `--fail-below` | No | Exit with code 1 if the overall score is below this value (0–100) | +The CLI does not require the `if __name__ == "__main__":` guard — that's already handled internally. + ## Known Limitations - **Language detection has no real signal for some languages.** The underlying detection library has no language profile for some languages (Roman Urdu being a notable example), so `language_contamination` doesn't attempt to identify "the dataset's language" — it only flags confident detections of specific languages on a configurable concern list. Text in an undetectable language can still occasionally be misdetected as a concern-list language (measured false-positive rate: roughly 10–12% on real Roman Urdu test data) — this is a bounded, documented limitation, not a bug. - **Performance on very large datasets.** Recent optimizations (deduplication before comparison, parallelized language detection) significantly improved runtime on large datasets, but `near_duplicates` and `language_contamination` remain the two most compute-intensive checks. On datasets over 100K rows, a full audit can still take up to several minutes (up to ~5 minutes observed on a 100K+ row real-world dataset) depending on text diversity and duplication rate. If you're auditing a very large corpus, expect these two checks to dominate total runtime. - **`--concern-languages` defaults to English/Hindi.** This is a reasonable general-purpose default, particularly for code-switched or Roman-script corpora where English/Hindi leakage is a common risk — but it is not universally correct. If you're auditing a dataset with a different realistic contamination risk, override this flag rather than relying on the default. +- **On Windows, the Python API requires an `if __name__ == "__main__":` guard** when called from a top-level script, due to how `language_contamination`'s process pool is created without `fork()` support. See the note above the Quickstart section. The CLI is unaffected. ## Example Output diff --git a/inference_audit/checks/language_contamination.py b/inference_audit/checks/language_contamination.py index 8f8a3fe..38cff46 100644 --- a/inference_audit/checks/language_contamination.py +++ b/inference_audit/checks/language_contamination.py @@ -153,8 +153,19 @@ def check_language_contamination( # independent of every other, so this is embarrassingly parallel. --- if len(unique_texts) >= _PARALLEL_THRESHOLD: max_workers = min(32, (os.cpu_count() or 4)) - with ProcessPoolExecutor(max_workers=max_workers, initializer=_init_worker) as executor: - results = list(executor.map(_detect_one, unique_texts, chunksize=200)) + try: + with ProcessPoolExecutor(max_workers=max_workers, initializer=_init_worker) as executor: + results = list(executor.map(_detect_one, unique_texts, chunksize=200)) + except RuntimeError as e: + if "bootstrapping phase" in str(e): + raise RuntimeError( + "language_contamination uses multiprocessing for performance on " + "large datasets. On Windows, calling Auditor.audit() from a " + "top-level script requires wrapping your code in " + "`if __name__ == '__main__':`. See the README's Quickstart " + "section for a working example." + ) from e + raise else: results = [_detect_one(t) for t in unique_texts] diff --git a/pyproject.toml b/pyproject.toml index 70b1a86..17a7f94 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta" [project] name = "inference-audit-pk" -version = "0.1.0" +version = "0.1.1" description = "NLP dataset quality auditor: label distribution, near-duplicates, language contamination, missing values, and annotation consistency checks in one command." readme = "README.md" requires-python = ">=3.9" @@ -30,6 +30,7 @@ classifiers = [ "Programming Language :: Python :: 3.9", "Programming Language :: Python :: 3.10", "Programming Language :: Python :: 3.11", + "Programming Language :: Python :: 3.12", "Operating System :: OS Independent", "Topic :: Scientific/Engineering :: Artificial Intelligence", ] From 568528eb884f6058882acf1a6e0ca2c229e3251d Mon Sep 17 00:00:00 2001 From: Khadija Date: Sat, 26 Sep 2026 16:43:52 +0500 Subject: [PATCH 3/3] fix: clear error for Windows multiprocessing guard issue; add missing 3.12 classifier; update README examples (0.1.1) --- CHANGELOG.md | 2 +- README.md | 2 +- inference_audit/checks/language_contamination.py | 3 ++- pyproject.toml | 1 + 4 files changed, 5 insertions(+), 3 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 6ecc044..dbe5a60 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,6 +1,6 @@ # Changelog -## [0.1.1] - 2026-09-26 +## [0.1.1] - 2026-09-XX ### Fixed - Clearer error message when `Auditor.audit()` is called from a top-level diff --git a/README.md b/README.md index 16fb053..82c43cc 100644 --- a/README.md +++ b/README.md @@ -198,7 +198,7 @@ MIT — see [LICENSE](LICENSE). ## Authors -Built as part of the Inference Lab Engineering Fellowship, Cohort 01 — Project C. +Built as part of the INFERENCE Lab Engineering Fellowship, Cohort 01 — Project C. - **Khadija Faisal** (Lead Engineer) — [GitHub](https://github.com/khadijja1) · [LinkedIn](https://www.linkedin.com/in/khadijjafaisal) - **Muhammad Shoaib Altaf** (Research & Implementation Engineer) — [GitHub](https://github.com/Shoaib-Altaf) · [LinkedIn](https://www.linkedin.com/in/muhammad-shoaib-altaf-6ab3a8326) diff --git a/inference_audit/checks/language_contamination.py b/inference_audit/checks/language_contamination.py index 38cff46..30bd2af 100644 --- a/inference_audit/checks/language_contamination.py +++ b/inference_audit/checks/language_contamination.py @@ -150,7 +150,8 @@ def check_language_contamination( # --- Improvement 2: run those per-unique-text detections in # parallel across CPU cores when there are enough of them to make # the process-pool overhead worthwhile. Each detection is fully - # independent of every other, so this is embarrassingly parallel. --- + # independent of every other, so this is embarrassingly parallel. + # clear error for Windows multiprocessing guard issue --- if len(unique_texts) >= _PARALLEL_THRESHOLD: max_workers = min(32, (os.cpu_count() or 4)) try: diff --git a/pyproject.toml b/pyproject.toml index 17a7f94..62eaf1e 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -10,6 +10,7 @@ readme = "README.md" requires-python = ">=3.9" license = {file = "LICENSE"} authors = [ + {name = "INFERENCE Lab"}, {name = "Khadija Faisal"}, {name = "Muhammad Shoaib Altaf"}, ]