From 4e0db5c2d8f5c41c82fc6e355da6ba46966c7333 Mon Sep 17 00:00:00 2001 From: Shivam Lalakiya <50960482+shivamlalakiya@users.noreply.github.com> Date: Fri, 25 Sep 2026 13:12:32 -0500 Subject: [PATCH] Add read_gifts as one entry point over the gift bridge presets CiviCRM, Raiser's Edge and NPSP each ship their own read__... loader and _..._to_features aggregator pair. That's fine when a caller only ever works with one CRM's export, but anyone supporting more than one CRM (a consultant with clients on different systems, a tool that takes "whichever export you have") ends up importing three differently named function pairs to do the same read-then-aggregate operation. read_gifts(path_or_df, source=...) looks that pair up in a small registry (GIFT_SOURCES) instead, and runs it: read_gifts("gifts.csv", source="npsp") is exactly equivalent to npsp_opportunities_to_features(read_npsp_opportunities("gifts.csv")). Extra keyword arguments pass straight through to the matched aggregator, so exclude_stages, exclude_gift_types and statuses all still work. Kept the aggregator itself where it already lives in _civicrm.py rather than moving it to a new neutral module: it's the lower-risk option, and "extract" is satisfied by exposing all three presets uniformly through one function instead of relocating logic that raisers_edge and npsp already import cleanly. cli.py's own three-way --source dispatch is left untouched since it does its own per-step error handling that read_gifts would have to either duplicate or change. Tested with make ci (2148 passed, 8 skipped, 98.58% coverage) and make riskcov (98% over the risk tier, floor 93%). No existing test file changed. --- CHANGELOG.md | 7 ++ docs/reference/index.md | 1 + philanthropy/ingest/__init__.py | 7 ++ philanthropy/ingest/_read_gifts.py | 104 +++++++++++++++++++++++++ tests/test_read_gifts.py | 121 +++++++++++++++++++++++++++++ 5 files changed, 240 insertions(+) create mode 100644 philanthropy/ingest/_read_gifts.py create mode 100644 tests/test_read_gifts.py diff --git a/CHANGELOG.md b/CHANGELOG.md index 060303c..1dd9b77 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -25,6 +25,13 @@ Format: [Keep a Changelog](https://keepachangelog.com/en/1.1.0/) `_hours_12m` / `_amount_12m` when those columns are present), cut at `as_of` so nothing dated after the cutoff is counted. A new activity type never needs new model code; it just yields its own columns. +- `philanthropy.ingest.read_gifts(path_or_df, *, source=...)`: reads (if given + a path) and aggregates a gift export in one call, looking up the CiviCRM, + Raiser's Edge or NPSP reader-and-aggregator pair by name from the new + `GIFT_SOURCES` registry, for a caller working with more than one CRM export + format. Keyword arguments other than `source` pass straight through to the + matched aggregator, so `exclude_stages`, `exclude_gift_types` and `statuses` + all still work. ## [0.8.0] - 2026-09-24 diff --git a/docs/reference/index.md b/docs/reference/index.md index f97571a..dae27be 100755 --- a/docs/reference/index.md +++ b/docs/reference/index.md @@ -58,6 +58,7 @@ Everything reachable from `philanthropy.__all__` is listed below. A symbol not l | `npsp_opportunities_to_features`, `read_npsp_opportunities`, `DEFAULT_EXCLUDED_STAGES` | `ingest` | Tracks NPSP Opportunity export labels and a per-org-configurable stage vocabulary; the excluded-stage default will grow as real exports arrive. | | `map_columns` | `ingest` | The one-error-per-missing-column message shape may still change. | | `activities_to_features` | `ingest` | The activity-type feature set (`_count_12m`, `_distinct`, ...) may grow as more source types are onboarded. | +| `read_gifts`, `GIFT_SOURCES` | `ingest` | A thin preset registry over the CiviCRM, Raiser's Edge and NPSP bridges above; it inherits their tier and grows a new preset name as they do. | | `plot_affinity_distribution`, `plot_retention_waterfall` | `visualisation` | Chart composition is presentation, not contract. | | `fetch_kdd98_donors` | `datasets` | Returns the raw upstream columns untyped; may gain as-of date parsing as the real-data leakage replication in #124 lands. | | `make_donor_panel` | `datasets` | The returned dict may gain keys (pledges, appeals, soft credits) as more of the library needs panel-shaped fixtures; existing keys and their columns will not change silently. | diff --git a/philanthropy/ingest/__init__.py b/philanthropy/ingest/__init__.py index 4862102..0b4d1c7 100644 --- a/philanthropy/ingest/__init__.py +++ b/philanthropy/ingest/__init__.py @@ -30,6 +30,10 @@ (event attendance, volunteer shifts, email clicks, ...) into per-donor, per-type engagement features, generalising the pattern above to an open-ended set of activity types discovered from the data itself. + +``read_gifts(path_or_df, source=...)`` looks up the CiviCRM, Raiser's Edge or +NPSP reader-and-aggregator pair by name, for a caller working with more than +one CRM export format. """ from ._activities import activities_to_features @@ -52,10 +56,12 @@ raisers_edge_gifts_to_features, read_raisers_edge_gifts, ) +from ._read_gifts import GIFT_SOURCES, read_gifts __all__ = [ "DEFAULT_EXCLUDED_GIFT_TYPES", "DEFAULT_EXCLUDED_STAGES", + "GIFT_SOURCES", "activities_to_features", "civicrm_contributions_to_features", "constituent_events_to_features", @@ -64,6 +70,7 @@ "raisers_edge_gifts_to_features", "read_civicrm_contributions", "read_constituent_events", + "read_gifts", "read_npsp_opportunities", "read_raisers_edge_gifts", ] diff --git a/philanthropy/ingest/_read_gifts.py b/philanthropy/ingest/_read_gifts.py new file mode 100644 index 0000000..0216a80 --- /dev/null +++ b/philanthropy/ingest/_read_gifts.py @@ -0,0 +1,104 @@ +""" +philanthropy.ingest._read_gifts +================================ +One call over the CiviCRM, Raiser's Edge and NPSP gift bridges. + +Each bridge module pairs its own ``read__...`` loader with a +``_..._to_features`` aggregator, because each CRM's export needs its +own header aliases and commitment-versus-payment exclusion filter before the +rows can be handed to the shared aggregator in +:mod:`philanthropy.ingest._civicrm`. A caller working with more than one CRM +export format otherwise has to import three differently-named pairs for what +is, from the outside, the same read-then-aggregate operation. +:func:`read_gifts` is that pair looked up by a ``source`` name instead. +""" + +from __future__ import annotations + +from pathlib import Path +from typing import Any, Callable, Iterable, Mapping, Tuple, Union + +import pandas as pd + +from ._civicrm import civicrm_contributions_to_features, read_civicrm_contributions +from ._npsp import npsp_opportunities_to_features, read_npsp_opportunities +from ._raisers_edge import raisers_edge_gifts_to_features, read_raisers_edge_gifts + +__all__ = ["GIFT_SOURCES", "read_gifts"] + +#: Valid ``source`` names for :func:`read_gifts`, in the order the CLI's +#: `--source` choices already list them. +GIFT_SOURCES: Tuple[str, ...] = ("civicrm", "raisers_edge", "npsp") + +# (reader, aggregator) pair per source, the same shape as the preset dispatch +# in cli.py's _cmd_features. +_REGISTRY: "dict[str, tuple[Callable[[Union[str, Path]], pd.DataFrame], Callable[..., pd.DataFrame]]]" = { + "civicrm": (read_civicrm_contributions, civicrm_contributions_to_features), + "raisers_edge": (read_raisers_edge_gifts, raisers_edge_gifts_to_features), + "npsp": (read_npsp_opportunities, npsp_opportunities_to_features), +} + + +def read_gifts( + path_or_df: Union[str, Path, pd.DataFrame, Iterable[Mapping]], + *, + source: str, + **kwargs: Any, +) -> pd.DataFrame: + """Read (if given a path) and aggregate a CRM gift export in one call. + + Looks ``source`` up in a small preset registry mapping each of + ``"civicrm"``, ``"raisers_edge"`` and ``"npsp"`` to that CRM's + ``read__...`` loader and ``_..._to_features`` aggregator, + then runs the pair. ``read_gifts("gifts.csv", source="npsp")`` is + equivalent to + ``npsp_opportunities_to_features(read_npsp_opportunities("gifts.csv"))``. + + Parameters + ---------- + path_or_df : str, pathlib.Path, DataFrame, or iterable of mapping + A gift export CSV file, a directory of them, or gift rows already in + memory (a DataFrame or an iterable of mappings, e.g. an API result). + A path is only meaningful for a single source's own export format; + already-in-memory rows are handed straight to the aggregator. + source : str + Which CRM the export came from. One of :data:`GIFT_SOURCES`. + **kwargs + Passed through to the source's aggregator, e.g. ``statuses=`` for + ``"civicrm"``, ``exclude_gift_types=`` for ``"raisers_edge"``, + ``exclude_stages=`` for ``"npsp"``, or the ``reference_date=`` every + preset accepts. + + Returns + ------- + features : pandas.DataFrame + One row per donor, indexed by ``contact_id``, identical to calling + the source's own reader and aggregator directly. + + Raises + ------ + ValueError + If ``source`` is not one of :data:`GIFT_SOURCES`. + + Examples + -------- + >>> rows = [ + ... {"Constituent ID": "88", "Gift Date": "2025-01-10", + ... "Gift Amount": "1200.00", "Gift Type": "Pledge"}, + ... {"Constituent ID": "88", "Gift Date": "2025-02-10", + ... "Gift Amount": "100.00", "Gift Type": "Pay-Cash"}, + ... ] + >>> feats = read_gifts(rows, source="raisers_edge") + >>> float(feats.loc["88", "total_gift_amount"]) # the pledge is excluded + 100.0 + """ + try: + read, to_features = _REGISTRY[source] + except KeyError: + raise ValueError( + f"Unknown gift source {source!r}; expected one of {GIFT_SOURCES}." + ) from None + + if isinstance(path_or_df, (str, Path)): + path_or_df = read(path_or_df) + return to_features(path_or_df, **kwargs) diff --git a/tests/test_read_gifts.py b/tests/test_read_gifts.py new file mode 100644 index 0000000..634b965 --- /dev/null +++ b/tests/test_read_gifts.py @@ -0,0 +1,121 @@ +""" +tests/test_read_gifts.py +Tests for philanthropy.ingest.read_gifts, the preset-registry entry point +over the CiviCRM, Raiser's Edge and NPSP gift bridges. + +The point of this module is that `read_gifts(x, source=name)` is exactly +equivalent to calling that source's own reader-and-aggregator pair directly; +each preset's own filtering behaviour is already covered by +tests/test_civicrm.py, tests/test_raisers_edge.py and tests/test_npsp.py. +""" + +import pandas as pd +import pytest + +from philanthropy.ingest import ( + GIFT_SOURCES, + civicrm_contributions_to_features, + npsp_opportunities_to_features, + raisers_edge_gifts_to_features, + read_civicrm_contributions, + read_gifts, + read_npsp_opportunities, + read_raisers_edge_gifts, +) + + +def _write_csv(tmp_path, header, *rows): + path = tmp_path / "gifts.csv" + path.write_text("\n".join((header,) + rows) + "\n") + return path + + +# --------------------------------------------------------------------------- # +# Path input: read_gifts matches the two-step reader + aggregator call +# --------------------------------------------------------------------------- # +def test_civicrm_path_matches_direct_call(tmp_path): + path = _write_csv( + tmp_path, + "Contact ID,Contribution Date,Total Amount,Contribution Status", + "101,2025-01-15,250.00,Completed", + ) + via_registry = read_gifts(path, source="civicrm") + direct = civicrm_contributions_to_features(read_civicrm_contributions(path)) + pd.testing.assert_frame_equal(via_registry, direct) + + +def test_raisers_edge_path_matches_direct_call(tmp_path): + path = _write_csv( + tmp_path, + "Constituent ID,Gift Date,Gift Amount,Gift Type", + "88,2025-01-10,1200.00,Pledge", + "88,2025-02-10,100.00,Pay-Cash", + ) + via_registry = read_gifts(path, source="raisers_edge") + direct = raisers_edge_gifts_to_features(read_raisers_edge_gifts(path)) + pd.testing.assert_frame_equal(via_registry, direct) + + +def test_npsp_path_matches_direct_call(tmp_path): + path = _write_csv( + tmp_path, + "Account ID,Close Date,Amount,Stage", + "88,2025-01-10,100.00,Pledged", + "88,2025-02-10,100.00,Closed Won", + ) + via_registry = read_gifts(path, source="npsp") + direct = npsp_opportunities_to_features(read_npsp_opportunities(path)) + pd.testing.assert_frame_equal(via_registry, direct) + + +# --------------------------------------------------------------------------- # +# In-memory input: no file read, straight to the aggregator +# --------------------------------------------------------------------------- # +def test_dataframe_input_skips_reading_and_goes_straight_to_the_aggregator(): + df = pd.DataFrame( + [{"Account ID": "7", "Close Date": "2025-05-01", + "Amount": "300.00", "Stage": "Closed Won"}] + ) + via_registry = read_gifts(df, source="npsp") + direct = npsp_opportunities_to_features(df) + pd.testing.assert_frame_equal(via_registry, direct) + + +def test_iterable_of_mappings_input(): + rows = [ + {"Contact ID": "101", "Contribution Date": "2025-01-15", + "Total Amount": "250.00", "Contribution Status": "Completed"}, + ] + via_registry = read_gifts(rows, source="civicrm") + direct = civicrm_contributions_to_features(rows) + pd.testing.assert_frame_equal(via_registry, direct) + + +# --------------------------------------------------------------------------- # +# Unknown source +# --------------------------------------------------------------------------- # +def test_unknown_source_raises_a_clear_error(): + with pytest.raises(ValueError, match="salesforce_classic"): + read_gifts([], source="salesforce_classic") + + +def test_gift_sources_lists_the_three_presets(): + assert set(GIFT_SOURCES) == {"civicrm", "raisers_edge", "npsp"} + + +# --------------------------------------------------------------------------- # +# kwarg passthrough to the underlying aggregator +# --------------------------------------------------------------------------- # +def test_source_specific_kwarg_reaches_the_underlying_aggregator(): + rows = [ + {"Account ID": "88", "Close Date": "2025-01-10", "Amount": "100.00", + "Stage": "Pledged"}, + {"Account ID": "88", "Close Date": "2025-02-10", "Amount": "100.00", + "Stage": "Closed Won"}, + ] + # Default excludes Pledged: only the Closed Won row counts. + default = read_gifts(rows, source="npsp") + assert float(default.loc["88", "total_gift_amount"]) == 100.0 + # exclude_stages=None disables the filter and sums both rows. + unfiltered = read_gifts(rows, source="npsp", exclude_stages=None) + assert float(unfiltered.loc["88", "total_gift_amount"]) == 200.0