From 075ae5ac966a3d32b784bcc5c68563fb6b7da355 Mon Sep 17 00:00:00 2001 From: Shivam Lalakiya <50960482+shivamlalakiya@users.noreply.github.com> Date: Fri, 25 Sep 2026 12:00:09 -0500 Subject: [PATCH] Add map_columns for renaming user-supplied CRM export headers A no-code upload flow lets a user choose which of their own column headers means "donor ID" or "gift date" without ever showing the library the original names. map_columns applies that mapping and checks the result has every column the next ingest step needs, raising one error that lists every missing column instead of failing on the first one a caller happens to touch. --- CHANGELOG.md | 6 +++ docs/reference/index.md | 1 + philanthropy/ingest/__init__.py | 6 +++ philanthropy/ingest/_map_columns.py | 79 +++++++++++++++++++++++++++++ tests/test_map_columns.py | 64 +++++++++++++++++++++++ 5 files changed, 156 insertions(+) create mode 100644 philanthropy/ingest/_map_columns.py create mode 100644 tests/test_map_columns.py diff --git a/CHANGELOG.md b/CHANGELOG.md index 8ce458d..fd926a0 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,6 +5,12 @@ Format: [Keep a Changelog](https://keepachangelog.com/en/1.1.0/) ## [Unreleased] +### Added +- `philanthropy.ingest.map_columns(df, mapping, *, required=...)`: renames a + user-supplied export's headers to canonical names and raises one + `ValueError` listing every still-missing required column, for callers + building a column-mapping UI over an arbitrary CRM export. + ## [0.8.0] - 2026-09-24 The first release with a Raiser's Edge on-ramp and `as_of` scoring cutoffs on the diff --git a/docs/reference/index.md b/docs/reference/index.md index e31eaf9..02db716 100755 --- a/docs/reference/index.md +++ b/docs/reference/index.md @@ -55,6 +55,7 @@ Everything reachable from `philanthropy.__all__` is listed below. A symbol not l | `constituent_events_to_features`, `read_constituent_events` | `ingest` | Tracks the UniSchema `ConstituentEvent` schema, which is versioned upstream. | | `civicrm_contributions_to_features`, `read_civicrm_contributions` | `ingest` | Tracks CiviCRM's contribution export labels and APIv4 field names, which move with the CRM. | | `raisers_edge_gifts_to_features`, `read_raisers_edge_gifts`, `DEFAULT_EXCLUDED_GIFT_TYPES` | `ingest` | Tracks Raiser's Edge export labels and the RE NXT gift-type vocabulary; the excluded-type default will grow as real exports arrive. | +| `map_columns` | `ingest` | The one-error-per-missing-column message shape may still change. | | `plot_affinity_distribution`, `plot_retention_waterfall` | `visualisation` | Chart composition is presentation, not contract. | | `fetch_kdd98_donors` | `datasets` | Returns the raw upstream columns untyped; may gain as-of date parsing as the real-data leakage replication in #124 lands. | | `make_donor_panel` | `datasets` | The returned dict may gain keys (pledges, appeals, soft credits) as more of the library needs panel-shaped fixtures; existing keys and their columns will not change silently. | diff --git a/philanthropy/ingest/__init__.py b/philanthropy/ingest/__init__.py index 98af90b..673cb86 100644 --- a/philanthropy/ingest/__init__.py +++ b/philanthropy/ingest/__init__.py @@ -16,6 +16,10 @@ ``raisers_edge_gifts_to_features`` aggregates it, dropping the commitment rows (pledges, matching gift pledges, recurring gift templates) first so a pledged dollar is not counted both as the promise and as the payments against it. + +``map_columns`` renames a user-supplied export's headers to the canonical +names a bridge above expects, raising one error listing every column still +missing after the rename. """ from ._civicrm import ( @@ -26,6 +30,7 @@ constituent_events_to_features, read_constituent_events, ) +from ._map_columns import map_columns from ._raisers_edge import ( DEFAULT_EXCLUDED_GIFT_TYPES, raisers_edge_gifts_to_features, @@ -36,6 +41,7 @@ "DEFAULT_EXCLUDED_GIFT_TYPES", "civicrm_contributions_to_features", "constituent_events_to_features", + "map_columns", "raisers_edge_gifts_to_features", "read_civicrm_contributions", "read_constituent_events", diff --git a/philanthropy/ingest/_map_columns.py b/philanthropy/ingest/_map_columns.py new file mode 100644 index 0000000..eb0cccc --- /dev/null +++ b/philanthropy/ingest/_map_columns.py @@ -0,0 +1,79 @@ +""" +philanthropy.ingest._map_columns +================================= +Rename a user-supplied CRM export's headers to the canonical names a +PhilanthroPy ingest function expects. + +A no-code upload flow lets a user pick, per file, which of their own column +headers means "donor ID" or "gift date"; the library never sees the user's +original header names, only the mapping the user chose. :func:`map_columns` +applies that mapping and checks the result actually has every column the next +step requires, so a missing or mistyped mapping fails loudly at the upload +step instead of surfacing as a cryptic ``KeyError`` deep inside a feature +function. +""" + +from __future__ import annotations + +from typing import Mapping, Sequence + +import pandas as pd + +__all__ = ["map_columns"] + + +def map_columns( + df: pd.DataFrame, + mapping: Mapping[str, str], + *, + required: Sequence[str] = (), +) -> pd.DataFrame: + """Rename a DataFrame's columns per a user-supplied mapping. + + Parameters + ---------- + df : pandas.DataFrame + The uploaded export, with whatever headers the source system wrote. + mapping : mapping of str to str + User header -> canonical name, e.g. ``{"Constituent ID": + "contact_id", "Gift Date": "activity_date"}``. Keys not present in + ``df.columns`` are ignored, so a mapping built from a superset of + known headers is safe to reuse across files. + required : sequence of str, default=() + Canonical column names that must be present after renaming. + + Returns + ------- + mapped : pandas.DataFrame + ``df`` with the mapped columns renamed. Unmapped columns are kept + as-is. + + Raises + ------ + ValueError + If, after renaming, any name in ``required`` is missing. The message + lists every missing column at once, so a user fixing the mapping in a + UI does not have to resubmit once per missing field. + + Examples + -------- + >>> import pandas as pd + >>> df = pd.DataFrame({"CnID": [1], "Gift Date": ["2025-01-01"]}) + >>> map_columns( + ... df, + ... {"CnID": "contact_id", "Gift Date": "activity_date"}, + ... required=["contact_id", "activity_date"], + ... ).columns.tolist() + ['contact_id', 'activity_date'] + >>> map_columns(df, {"CnID": "contact_id"}, required=["contact_id", "amount"]) + Traceback (most recent call last): + ... + ValueError: missing required column(s) after mapping: amount + """ + renamed = df.rename(columns=dict(mapping)) + missing = [col for col in required if col not in renamed.columns] + if missing: + raise ValueError( + "missing required column(s) after mapping: " + ", ".join(missing) + ) + return renamed diff --git a/tests/test_map_columns.py b/tests/test_map_columns.py new file mode 100644 index 0000000..737d936 --- /dev/null +++ b/tests/test_map_columns.py @@ -0,0 +1,64 @@ +""" +tests/test_map_columns.py +Tests for philanthropy.ingest.map_columns. +""" + +import pandas as pd +import pytest + +from philanthropy.ingest import map_columns + + +def test_renames_mapped_columns_and_keeps_unmapped(): + df = pd.DataFrame({"CnID": [1, 2], "Gift Date": ["2025-01-01", "2025-02-01"], "Note": ["a", "b"]}) + + out = map_columns(df, {"CnID": "contact_id", "Gift Date": "activity_date"}) + + assert list(out.columns) == ["contact_id", "activity_date", "Note"] + + +def test_ignores_mapping_keys_absent_from_df(): + df = pd.DataFrame({"CnID": [1]}) + + out = map_columns(df, {"CnID": "contact_id", "Hours": "hours"}) + + assert list(out.columns) == ["contact_id"] + + +def test_passes_when_all_required_present_after_mapping(): + df = pd.DataFrame({"CnID": [1], "Gift Date": ["2025-01-01"]}) + + out = map_columns( + df, + {"CnID": "contact_id", "Gift Date": "activity_date"}, + required=["contact_id", "activity_date"], + ) + + assert list(out.columns) == ["contact_id", "activity_date"] + + +def test_raises_one_error_listing_every_missing_required_column(): + df = pd.DataFrame({"CnID": [1]}) + + with pytest.raises(ValueError, match="activity_date, amount"): + map_columns( + df, + {"CnID": "contact_id"}, + required=["contact_id", "activity_date", "amount"], + ) + + +def test_no_required_columns_never_raises(): + df = pd.DataFrame({"CnID": [1]}) + + out = map_columns(df, {"CnID": "contact_id"}) + + assert list(out.columns) == ["contact_id"] + + +def test_does_not_mutate_input_dataframe(): + df = pd.DataFrame({"CnID": [1]}) + + map_columns(df, {"CnID": "contact_id"}) + + assert list(df.columns) == ["CnID"]