From 070a518f38820045c13cac33a4600331e86f3720 Mon Sep 17 00:00:00 2001 From: Shivam Lalakiya <50960482+shivamlalakiya@users.noreply.github.com> Date: Fri, 25 Sep 2026 12:31:33 -0500 Subject: [PATCH] Add a Salesforce NPSP Opportunity export reader Mirrors the existing Raiser's Edge bridge: read_npsp_opportunities loads an Opportunity export CSV (or a folder of them) and npsp_opportunities_to_features rolls it up into the donor-level feature table used by train/score, delegating the actual aggregation to the shared CiviCRM roll-up. NPSP's Recurring Donations feature creates one Opportunity per instalment, starting it in the Pledged stage and only moving it to a closed/won stage (Closed Won, or a site's own Posted) once the money is actually received. Depending on an org's instalment settings, the Pledged row and the Closed Won row recording its receipt can both exist for the same amount and close date, so summing every row's Amount double-counts that instalment. Pledged rows are dropped by default (DEFAULT_EXCLUDED_STAGES), same as Raiser's Edge drops its Pledge and Recurring Gift rows. Header aliases accept a Salesforce report export's column labels (Account Name, Close Date, Amount, Stage), the raw Opportunity API field names (AccountId, CloseDate, StageName), and NPSP's own Data Import template headers (Donation Date, Donation Amount, Donation Stage). Wired into the CLI as `philanthropy features --source npsp`. Added tests/test_npsp.py at the same depth as tests/test_raisers_edge.py plus two CLI integration tests, a CHANGELOG entry, a paragraph in docs/how-to/use_the_cli.md, and a Tier 2 (Beta) row in docs/reference/index.md. --- CHANGELOG.md | 7 + docs/how-to/use_the_cli.md | 4 +- docs/reference/index.md | 1 + philanthropy/cli.py | 12 +- philanthropy/ingest/__init__.py | 13 ++ philanthropy/ingest/_npsp.py | 295 +++++++++++++++++++++++++++++++ tests/test_cli.py | 36 ++++ tests/test_npsp.py | 302 ++++++++++++++++++++++++++++++++ 8 files changed, 664 insertions(+), 6 deletions(-) create mode 100644 philanthropy/ingest/_npsp.py create mode 100644 tests/test_npsp.py diff --git a/CHANGELOG.md b/CHANGELOG.md index fd926a0..c48035b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -10,6 +10,13 @@ Format: [Keep a Changelog](https://keepachangelog.com/en/1.1.0/) user-supplied export's headers to canonical names and raises one `ValueError` listing every still-missing required column, for callers building a column-mapping UI over an arbitrary CRM export. +- `philanthropy.ingest.read_npsp_opportunities` and + `npsp_opportunities_to_features`: a Salesforce Nonprofit Success Pack (NPSP) + Opportunity export bridge, alongside the existing CiviCRM and Raiser's Edge + ones. Drops `Pledged` Recurring Donation instalment rows by default + (`DEFAULT_EXCLUDED_STAGES`) so an instalment isn't counted both as the + pledge and again once it closes `Won`. Wired into the CLI as + `philanthropy features --source npsp`. ## [0.8.0] - 2026-09-24 diff --git a/docs/how-to/use_the_cli.md b/docs/how-to/use_the_cli.md index 8a42a66..32a0b0f 100644 --- a/docs/how-to/use_the_cli.md +++ b/docs/how-to/use_the_cli.md @@ -17,7 +17,7 @@ Four subcommands: `features`, `train`, `score`, `validate`. philanthropy features --source raisers_edge --data gifts.csv --out features.csv ``` -`--source` accepts `raisers_edge` (Blackbaud Raiser's Edge and RE NXT) or `civicrm`. `--data` takes a single CSV or a directory of them, walked recursively, which is the shape of a folder of monthly exports. Omit `--out` and the CSV goes to stdout. +`--source` accepts `raisers_edge` (Blackbaud Raiser's Edge and RE NXT), `npsp` (Salesforce Nonprofit Success Pack), or `civicrm`. `--data` takes a single CSV or a directory of them, walked recursively, which is the shape of a folder of monthly exports. Omit `--out` and the CSV goes to stdout. The output has one row per donor and these columns: @@ -27,7 +27,7 @@ Header spelling is normalised for you, so a desktop Export (`Constituent ID`, `G !!! warning "A pledge is not a payment, and `features` knows the difference" - In Raiser's Edge a pledge and the money paid against it are **separate gift records**, and a recurring gift row is a template rather than a sum ever received. Adding up the amount column double-counts every committed dollar. `features` drops the commitment rows (`Pledge`, `Matching Gift Pledge`, `Recurring Gift`) and the ledger corrections, and keeps the payments (`Pay-Cash`, `PledgePayment`, `RecurringGiftPayment`, ...). Export the **Gift Type** field or it cannot do this, and it will warn you. The excluded set is the `exclude_gift_types` parameter of `philanthropy.ingest.raisers_edge_gifts_to_features` if your site spells its types differently. For CiviCRM the equivalent traps are test-mode rows and non-`Completed` contributions, and they are dropped the same way. + In Raiser's Edge a pledge and the money paid against it are **separate gift records**, and a recurring gift row is a template rather than a sum ever received. Adding up the amount column double-counts every committed dollar. `features` drops the commitment rows (`Pledge`, `Matching Gift Pledge`, `Recurring Gift`) and the ledger corrections, and keeps the payments (`Pay-Cash`, `PledgePayment`, `RecurringGiftPayment`, ...). Export the **Gift Type** field or it cannot do this, and it will warn you. The excluded set is the `exclude_gift_types` parameter of `philanthropy.ingest.raisers_edge_gifts_to_features` if your site spells its types differently. NPSP writes the same split through Opportunity stage instead of a separate record: a Recurring Donation instalment is created `Pledged` and only moved to `Closed Won` (or a site's own `Posted`) once received, and depending on the org's instalment settings both rows can exist for the same money. `features` drops `Pledged` rows for `npsp`; override with the `exclude_stages` parameter of `philanthropy.ingest.npsp_opportunities_to_features` if your org's stages differ. For CiviCRM the equivalent traps are test-mode rows and non-`Completed` contributions, and they are dropped the same way. !!! note "`features` does not invent a label" diff --git a/docs/reference/index.md b/docs/reference/index.md index 02db716..582a8d7 100755 --- a/docs/reference/index.md +++ b/docs/reference/index.md @@ -55,6 +55,7 @@ Everything reachable from `philanthropy.__all__` is listed below. A symbol not l | `constituent_events_to_features`, `read_constituent_events` | `ingest` | Tracks the UniSchema `ConstituentEvent` schema, which is versioned upstream. | | `civicrm_contributions_to_features`, `read_civicrm_contributions` | `ingest` | Tracks CiviCRM's contribution export labels and APIv4 field names, which move with the CRM. | | `raisers_edge_gifts_to_features`, `read_raisers_edge_gifts`, `DEFAULT_EXCLUDED_GIFT_TYPES` | `ingest` | Tracks Raiser's Edge export labels and the RE NXT gift-type vocabulary; the excluded-type default will grow as real exports arrive. | +| `npsp_opportunities_to_features`, `read_npsp_opportunities`, `DEFAULT_EXCLUDED_STAGES` | `ingest` | Tracks NPSP Opportunity export labels and a per-org-configurable stage vocabulary; the excluded-stage default will grow as real exports arrive. | | `map_columns` | `ingest` | The one-error-per-missing-column message shape may still change. | | `plot_affinity_distribution`, `plot_retention_waterfall` | `visualisation` | Chart composition is presentation, not contract. | | `fetch_kdd98_donors` | `datasets` | Returns the raw upstream columns untyped; may gain as-of date parsing as the real-data leakage replication in #124 lands. | diff --git a/philanthropy/cli.py b/philanthropy/cli.py index b0d483b..d375b99 100644 --- a/philanthropy/cli.py +++ b/philanthropy/cli.py @@ -33,7 +33,7 @@ # Gift-export readers `features` can front. Each name maps to a # (reader, aggregator) pair in _cmd_features. -_FEATURE_SOURCES = ("civicrm", "raisers_edge") +_FEATURE_SOURCES = ("civicrm", "raisers_edge", "npsp") # The donor-level columns `features` emits, in order. Named here so # `philanthropy features --help` answers "what do I pass to --features?" @@ -203,6 +203,9 @@ def _cmd_features(args: argparse.Namespace) -> None: if args.source == "raisers_edge": read = ingest.read_raisers_edge_gifts to_features = ingest.raisers_edge_gifts_to_features + elif args.source == "npsp": + read = ingest.read_npsp_opportunities + to_features = ingest.npsp_opportunities_to_features else: read = ingest.read_civicrm_contributions to_features = ingest.civicrm_contributions_to_features @@ -248,9 +251,10 @@ def _build_parser() -> argparse.ArgumentParser: "the models consume, then feed that to `train` and `score`. " "Emitted columns, in order: " + _FEATURE_COLUMNS + ". Commitment " "rows (pledges, recurring gift templates) are dropped for " - "raisers_edge, and test-mode and non-Completed rows for civicrm, " - "so a committed dollar is not counted twice. No label is produced: " - "`train --target` needs a column you define yourself." + "raisers_edge, Pledged instalment rows for npsp, and test-mode " + "and non-Completed rows for civicrm, so a committed dollar is " + "not counted twice. No label is produced: `train --target` " + "needs a column you define yourself." ), ) features.add_argument( diff --git a/philanthropy/ingest/__init__.py b/philanthropy/ingest/__init__.py index 673cb86..23a6cef 100644 --- a/philanthropy/ingest/__init__.py +++ b/philanthropy/ingest/__init__.py @@ -17,6 +17,11 @@ (pledges, matching gift pledges, recurring gift templates) first so a pledged dollar is not counted both as the promise and as the payments against it. +NPSP: ``read_npsp_opportunities`` loads a Salesforce Nonprofit Success Pack +Opportunity export CSV; ``npsp_opportunities_to_features`` aggregates it, +dropping ``Pledged`` instalment rows first so a Recurring Donation instalment +is not counted both in its ``Pledged`` stage and again once it closes ``Won``. + ``map_columns`` renames a user-supplied export's headers to the canonical names a bridge above expects, raising one error listing every column still missing after the rename. @@ -31,6 +36,11 @@ read_constituent_events, ) from ._map_columns import map_columns +from ._npsp import ( + DEFAULT_EXCLUDED_STAGES, + npsp_opportunities_to_features, + read_npsp_opportunities, +) from ._raisers_edge import ( DEFAULT_EXCLUDED_GIFT_TYPES, raisers_edge_gifts_to_features, @@ -39,11 +49,14 @@ __all__ = [ "DEFAULT_EXCLUDED_GIFT_TYPES", + "DEFAULT_EXCLUDED_STAGES", "civicrm_contributions_to_features", "constituent_events_to_features", "map_columns", + "npsp_opportunities_to_features", "raisers_edge_gifts_to_features", "read_civicrm_contributions", "read_constituent_events", + "read_npsp_opportunities", "read_raisers_edge_gifts", ] diff --git a/philanthropy/ingest/_npsp.py b/philanthropy/ingest/_npsp.py new file mode 100644 index 0000000..651160c --- /dev/null +++ b/philanthropy/ingest/_npsp.py @@ -0,0 +1,295 @@ +""" +philanthropy.ingest._npsp +========================== +Bridge from a Salesforce Nonprofit Success Pack (NPSP) Opportunity export to a +PhilanthroPy donor-level feature table. + +`NPSP `_ is the +Salesforce managed package most nonprofits running on Salesforce build their +donor database on, and the ``Opportunity`` object is where a gift lives: a +donation is an Opportunity record, keyed to the donor's ``Account`` (NPSP's +default Household Account model) or its ``Primary Contact``, with ``Amount``, +``CloseDate`` and ``StageName``. An export out of a report or the Data Loader +therefore carries the report column label (``Account Name``, ``Amount``, +``Close Date``, ``Stage``), the raw API field name (``AccountId``, +``CloseDate``, ``StageName``), or NPSP's own Data Import template header +(``Donation Amount``, ``Donation Date``, ``Donation Stage``). All three are +accepted here and normalised onto the canonical ``contact_id`` / +``receive_date`` / ``total_amount`` names. + +:func:`read_npsp_opportunities` loads the CSV(s); +:func:`npsp_opportunities_to_features` drops the ``Pledged`` rows and hands the +remaining rows to :func:`~philanthropy.ingest.civicrm_contributions_to_features`, +which already knows how to roll a gift log up into the one-row-per-donor frame +the estimators consume. + +**The trap this exists to prevent is the same commitment-versus-payment split +Raiser's Edge writes as separate gift records, expressed instead through +Opportunity stage.** NPSP's Recurring Donations feature creates one +Opportunity per instalment; the upcoming instalment is created with +``StageName`` ``Pledged`` (the "we expect this money" stage NPSP selects +automatically), and is only moved to a closed/won stage such as ``Closed Won`` +or a site's own ``Posted`` once the gift is actually received. Depending on +the org's "Installment Opportunity Auto-Creation" setting, the ``Pledged`` +Opportunity for an instalment and the ``Closed Won`` Opportunity recording its +receipt can both exist, on the same close date, for the same amount, so +summing every Opportunity's ``Amount`` naively counts that instalment twice: +once as the promise, again as the money. ``Pledged`` is excluded by default; +closed/won rows (``Closed Won``, ``Posted``, ...) are kept, because those are +the money. + +NPSP's stage vocabulary is org-configured (custom sales processes can rename +or add stages), so the excluded set is a documented default parameter rather +than a constant: see :data:`DEFAULT_EXCLUDED_STAGES`. + +Sources: Salesforce, *Standard NPSP Data Import Fields* +(https://help.salesforce.com/s/articleView?id=sfdo.npsp_standard_di_fields.htm), +which lists the ``Donation Amount`` / ``Donation Date`` / ``Donation Stage`` / +``Donation Record Type Name`` headers NPSP's own Data Import tool uses; +*NPSP Logic for Creating Opportunity Contact Roles* +(https://help.salesforce.com/s/articleView?id=sfdo.NPSP_Logic_for_Creating_OCRs.htm), +which documents the Opportunity's Primary Contact; and the Trailhead modules +"Managing Recurring Donations with Nonprofit Success Pack" +(https://trailhead.salesforce.com/content/learn/modules/donation-management-basics-with-nonprofit-success-pack/create-recurring-donations), +which walks through an instalment Opportunity moving from ``Pledged`` to +``Closed Won``/``Posted``, and "Customize Sales Processes and Paths for +Nonprofit Success" +(https://trailhead.salesforce.com/content/learn/modules/opportunity-settings-in-nonprofit-success-pack/understand-and-customize-sales-process-and-path-npsp), +which documents that the stage list is a per-org sales process. +""" + +from __future__ import annotations + +import re +import warnings +from pathlib import Path +from typing import Iterable, Mapping, Optional, Sequence, Union + +import pandas as pd + +from ._civicrm import ( + _NON_ALNUM, + _REQUIRED, + _canonical, + _empty_feature_frame, + _normalise_headers, + _to_frame, + civicrm_contributions_to_features, + read_civicrm_contributions, +) + +__all__ = [ + "DEFAULT_EXCLUDED_STAGES", + "npsp_opportunities_to_features", + "read_npsp_opportunities", +] + +#: Opportunity stages excluded from the roll-up by default: the stage NPSP +#: selects for a Recurring Donation instalment that has not been received yet. +#: Matching ignores case, spacing and punctuation, so this one spelling also +#: matches ``"pledged"`` and ``" PLEDGED "``. Closed/won rows (``Closed +#: Won``, a site's own ``Posted``, ...) are *not* in this set and are what the +#: features are built from. +DEFAULT_EXCLUDED_STAGES = ( + "Pledged", +) + +# NPSP header normalisation, applied before the CiviCRM bridge's own. Case and +# punctuation are already collapsed ("Close Date" -> close_date), so this maps +# only the residue onto the canonical names. Anything absent here falls +# through to the CiviCRM canonicaliser, which already handles the labels the +# two systems happen to share ("Amount", "Email", "First Name"). +_HEADER_ALIASES = { + # Donor key: NPSP's default Household Account model keys a gift to the + # Account; "Primary Contact" is the Opportunity's contact-level donor. + # Report label, then raw API field name, for each. + "account_id": "contact_id", + "accountid": "contact_id", + "account_name": "contact_id", + "primary_contact": "contact_id", + "npsp_primary_contact_c": "contact_id", + # Date: report label "Close Date", API field CloseDate, Data Import + # template header "Donation Date". + "close_date": "receive_date", + "closedate": "receive_date", + "donation_date": "receive_date", + # Amount: Data Import template header. "Amount" itself is already handled + # by the CiviCRM canonicaliser this falls through to. + "donation_amount": "total_amount", + # Stage: the column this module exists to read. + "stage": "gift_type", + "stagename": "gift_type", + "donation_stage": "gift_type", + # Record Type: NPSP's Opportunity classification (Donation, Grant, + # In-Kind, Major Gift, ...), the NPSP analogue of Raiser's Edge's Fund. + "record_type": "financial_type", + "recordtype_name": "financial_type", + "donation_record_type_name": "financial_type", +} + +# Stage-matching key: strip everything but letters and digits so "Closed Won" +# and "closed_won" collapse to one value, matching the Raiser's Edge bridge's +# gift-type matching. +_NON_ALNUM_ALL = re.compile(r"[^a-z0-9]+") + + +def npsp_opportunities_to_features( + opportunities: Union[Iterable[Mapping], pd.DataFrame], + *, + reference_date: Optional[Union[str, pd.Timestamp]] = None, + exclude_stages: Optional[Sequence[str]] = DEFAULT_EXCLUDED_STAGES, +) -> pd.DataFrame: + """Aggregate an NPSP Opportunity export into donor-level features. + + ``Pledged`` instalment rows are dropped first, then the surviving rows are + handed to :func:`civicrm_contributions_to_features`, which produces the + donor frame. NPSP's stage vocabulary carries no test-mode flag, so none is + applied. + + Parameters + ---------- + opportunities : iterable of mapping, or DataFrame + NPSP Opportunity export rows, under a report's column labels + (``Account Name`` or ``Primary Contact``, ``Close Date``, ``Amount``, + ``Stage``), the raw API field names (``AccountId``, ``CloseDate``, + ``StageName``), or NPSP's Data Import template headers (``Donation + Date``, ``Donation Amount``, ``Donation Stage``). ``contact_id``, + ``receive_date`` and ``total_amount`` are required after + normalisation; ``gift_type``, ``financial_type``, ``email``, + ``first_name`` and ``last_name`` are used when present. + reference_date : str or datetime-like, optional + Anchor for the recency features. If ``None``, the latest Opportunity + date in the batch is used, which keeps the aggregation free of "now" + leakage. + exclude_stages : sequence of str or None, default=:data:`DEFAULT_EXCLUDED_STAGES` + Stages to drop before aggregating, matched against ``gift_type`` + (the normalised Stage column) ignoring case, spacing and punctuation. + ``None`` or an empty sequence disables the filter and sums **every** + row, which double-counts an instalment recorded in both its + ``Pledged`` and closed/won stages. Rows whose stage is blank are kept + either way: an unlabelled row cannot be shown to be a pledge. + + Returns + ------- + features : pandas.DataFrame + One row per donor, indexed by ``contact_id``, with the same columns + :func:`civicrm_contributions_to_features` emits. + ``distinct_financial_types`` counts distinct Record Types here. + + Raises + ------ + KeyError + If ``contact_id``, ``receive_date`` or ``total_amount`` is absent + after normalisation. + + Warns + ----- + UserWarning + If ``exclude_stages`` was requested but the export carries no stage + column. Silently summing a ``Pledged`` instalment together with the + ``Closed Won`` row recording its receipt is exactly the error this + bridge exists to prevent, so it is worth a warning rather than a quiet + wrong total. + + Notes + ----- + Nothing here deduplicates on an Opportunity id, so concatenating two + exports that overlap in time will double-count the overlap; export + disjoint date ranges. + + Examples + -------- + >>> rows = [ + ... {"Account ID": "88", "Close Date": "2025-01-10", + ... "Amount": "100.00", "Stage": "Pledged"}, + ... {"Account ID": "88", "Close Date": "2025-02-10", + ... "Amount": "100.00", "Stage": "Closed Won"}, + ... {"Account ID": "88", "Close Date": "2025-03-10", + ... "Amount": "100.00", "Stage": "Closed Won"}, + ... ] + >>> feats = npsp_opportunities_to_features(rows) + >>> float(feats.loc["88", "total_gift_amount"]) # not 300.0 + 200.0 + >>> int(feats.loc["88", "gift_count"]) + 2 + """ + df = _normalise_headers(_to_frame(opportunities), _canonical_npsp) + if df.empty: + return _empty_feature_frame() + + missing = [col for col in _REQUIRED if col not in df.columns] + if missing: + raise KeyError( + f"NPSP Opportunity export is missing {missing}. Export the " + f"'Account ID' (or 'Primary Contact'), 'Close Date' and 'Amount' " + f"fields (Salesforce API: AccountId, CloseDate, Amount); got " + f"{sorted(df.columns)}." + ) + + if exclude_stages: + if "gift_type" in df.columns: + unwanted = {_stage_key(s) for s in exclude_stages} + keys = df["gift_type"].map(_stage_key) + df = df[~keys.isin(unwanted)] + else: + warnings.warn( + f"NPSP Opportunity export has no Stage column, so " + f"exclude_stages={tuple(exclude_stages)!r} could not be " + f"applied: a Pledged instalment (if any) is summed alongside " + f"the Closed Won row recording its receipt, which counts " + f"every pledged dollar twice. Add the 'Stage' field to the " + f"export.", + stacklevel=2, + ) + + if df.empty: + return _empty_feature_frame() + return civicrm_contributions_to_features( + df, reference_date=reference_date, statuses=None + ) + + +def read_npsp_opportunities(path: Union[str, Path]) -> pd.DataFrame: + """Read NPSP Opportunity export CSV(s) into one frame. + + Accepts a single ``.csv`` or a directory of them, walked recursively and + concatenated in sorted relative-path order; symlinks are not followed. + Every column is read as text and nothing is filtered: **Pledged rows are + still present**, and it is :func:`npsp_opportunities_to_features` that + drops them. + + Reading a gift export is CRM-agnostic once the headers are normalised, so + this delegates to :func:`read_civicrm_contributions` rather than repeating + its BOM handling and path hardening. NPSP headers are normalised on the + way into :func:`npsp_opportunities_to_features`, not here. + + Parameters + ---------- + path : str or pathlib.Path + CSV file, or a directory of them. + + Returns + ------- + opportunities : pandas.DataFrame + The export as written, with text values. + + Raises + ------ + FileNotFoundError + If ``path`` does not exist. + """ + return read_civicrm_contributions(path) + + +# --------------------------------------------------------------------------- # +# Internals +# --------------------------------------------------------------------------- # +def _canonical_npsp(header: str) -> str: + key = _NON_ALNUM.sub("_", str(header).strip().lower()).strip("_") + return _HEADER_ALIASES.get(key) or _canonical(header) + + +def _stage_key(value: object) -> str: + if value is None or (isinstance(value, float) and pd.isna(value)): + return "" + return _NON_ALNUM_ALL.sub("", str(value).strip().lower()) diff --git a/tests/test_cli.py b/tests/test_cli.py index f17fafc..36e1ba2 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -192,6 +192,42 @@ def test_cli_features_writes_to_stdout_by_default(tmp_path, capsys): assert len(out.strip().splitlines()) == 3 +_NPSP_HEADER = "Account ID,Close Date,Amount,Stage\n" + + +def _make_opportunity_export(tmp_path, name="opportunities.csv", n_donors=60): + rows = [_NPSP_HEADER] + for i in range(n_donors): + rows.append(f"{i},2025-01-10,{100 + i}.00,Pledged\n") + rows.append(f"{i},2025-01-10,{100 + i}.00,Closed Won\n") + rows.append(f"{i},2025-02-10,{50 + i}.00,Closed Won\n") + path = tmp_path / name + path.write_text("".join(rows)) + return path + + +def test_cli_features_npsp_drops_the_pledged_rows(tmp_path, capsys): + data = _make_opportunity_export(tmp_path, n_donors=3) + out_path = tmp_path / "features.csv" + main(["features", "--source", "npsp", "--data", str(data), + "--out", str(out_path)]) + feats = pd.read_csv(out_path) + assert len(feats) == 3 + # Donor 0: the two Closed Won rows (100 + 50), not the duplicate 100 pledge. + row = feats.loc[feats["contact_id"] == 0].iloc[0] + assert row["total_gift_amount"] == 150.0 + assert row["gift_count"] == 2 + assert "Wrote 3 donor rows" in capsys.readouterr().out + + +def test_cli_features_npsp_missing_export_field_exits_with_the_field_names(tmp_path): + path = tmp_path / "opportunities.csv" + path.write_text("Account ID,Stage\n1,Closed Won\n") + with pytest.raises(SystemExit) as excinfo: + main(["features", "--source", "npsp", "--data", str(path)]) + assert "Close Date" in str(excinfo.value) + + def test_cli_features_civicrm_source(tmp_path): path = tmp_path / "contributions.csv" path.write_text( diff --git a/tests/test_npsp.py b/tests/test_npsp.py new file mode 100644 index 0000000..2d25245 --- /dev/null +++ b/tests/test_npsp.py @@ -0,0 +1,302 @@ +""" +tests/test_npsp.py +Tests for the philanthropy.ingest Salesforce NPSP Opportunity bridge. + +The point of the module is one filter: NPSP's Recurring Donations feature can +carry both a ``Pledged`` Opportunity for an instalment and the ``Closed Won`` +Opportunity recording its receipt, so a naive sum counts the instalment +twice. Most of what follows checks that the ``Pledged`` rows leave and the +closed/won rows stay, in each spelling NPSP writes the export's headers in. +""" + +import warnings + +import pandas as pd +import pytest + +from philanthropy.ingest import ( + DEFAULT_EXCLUDED_STAGES, + npsp_opportunities_to_features, + read_npsp_opportunities, +) +from philanthropy.ingest._civicrm import _FEATURE_DTYPES + +# Header labels as a Salesforce report export writes them. +_HEADER = "Account ID,Close Date,Amount,Stage,Record Type,Email,First Name,Last Name" + + +def _row(contact, date, amount, stage, *, record_type="Donation", + email="", first="", last=""): + return f"{contact},{date},{amount},{stage},{record_type},{email},{first},{last}" + + +def _export(*rows): + return "\n".join((_HEADER,) + rows) + "\n" + + +@pytest.fixture +def opportunities(): + """One donor with a Recurring Donation instalment that carries both a + Pledged and a Closed Won row for the same $100, plus one already-received + instalment; one donor with only a still-open Pledged instalment.""" + return [ + {"Account ID": "88", "Close Date": "2025-01-10", "Amount": "100.00", + "Stage": "Pledged", "Record Type": "Donation", + "Email": "ada@amc.edu", "First Name": "Ada", "Last Name": "Lovelace"}, + {"Account ID": "88", "Close Date": "2025-01-10", "Amount": "100.00", + "Stage": "Closed Won", "Record Type": "Donation", + "Email": "ada@amc.edu", "First Name": "Ada", "Last Name": "Lovelace"}, + {"Account ID": "88", "Close Date": "2025-02-10", "Amount": "100.00", + "Stage": "Closed Won", "Record Type": "Recurring Donation", + "Email": "ada@amc.edu", "First Name": "Ada", "Last Name": "Lovelace"}, + {"Account ID": "91", "Close Date": "2025-03-01", "Amount": "50.00", + "Stage": "Pledged", "Record Type": "Donation", + "Email": "grace@amc.edu", "First Name": "Grace", "Last Name": "Hopper"}, + ] + + +# --------------------------------------------------------------------------- # +# The pledged-versus-closed-won filter +# --------------------------------------------------------------------------- # +def test_pledged_instalment_is_excluded_and_closed_won_is_not(opportunities): + feats = npsp_opportunities_to_features(opportunities) + # 100 (the pledge) + 100 + 100 would be 300; only the closed/won rows count. + assert float(feats.loc["88", "total_gift_amount"]) == 200.0 + assert int(feats.loc["88", "gift_count"]) == 2 + + +def test_every_row_pledged_returns_a_typed_empty_frame_for_that_donor(opportunities): + feats = npsp_opportunities_to_features(opportunities) + assert "91" not in feats.index + + +def test_every_row_filtered_out_returns_a_typed_empty_frame(): + rows = [{"Account ID": "1", "Close Date": "2025-01-01", + "Amount": "500.00", "Stage": "Pledged"}] + feats = npsp_opportunities_to_features(rows) + assert feats.empty + assert list(feats.columns) == list(_FEATURE_DTYPES) + assert feats.index.name == "contact_id" + + +@pytest.mark.parametrize("stage", ["Pledged"]) +def test_every_default_excluded_stage_is_dropped(stage): + rows = [{"Account ID": "1", "Close Date": "2025-01-01", + "Amount": "500.00", "Stage": stage}, + {"Account ID": "1", "Close Date": "2025-01-02", + "Amount": "10.00", "Stage": "Closed Won"}] + feats = npsp_opportunities_to_features(rows) + assert float(feats.loc["1", "total_gift_amount"]) == 10.0 + + +@pytest.mark.parametrize("stage", ["Closed Won", "Posted", "Prospecting", "Awarded"]) +def test_closed_and_other_open_stages_are_kept(stage): + rows = [{"Account ID": "1", "Close Date": "2025-01-01", + "Amount": "10.00", "Stage": stage}] + feats = npsp_opportunities_to_features(rows) + assert float(feats.loc["1", "total_gift_amount"]) == 10.0 + + +@pytest.mark.parametrize("stage", ["PLEDGED", "pledged", " Pledged "]) +def test_exclusion_matching_ignores_case_and_spacing(stage): + rows = [{"Account ID": "1", "Close Date": "2025-01-01", + "Amount": "500.00", "Stage": stage}, + {"Account ID": "1", "Close Date": "2025-01-02", + "Amount": "10.00", "Stage": "Closed Won"}] + feats = npsp_opportunities_to_features(rows) + assert float(feats.loc["1", "total_gift_amount"]) == 10.0 + + +def test_a_stage_only_containing_pledged_as_a_substring_is_not_excluded(): + """'Not Pledged Yet' must not be swept up by the 'Pledged' default: the + matching key is compared for equality, not membership.""" + rows = [{"Account ID": "1", "Close Date": "2025-01-01", + "Amount": "25.00", "Stage": "Not Pledged Yet"}] + feats = npsp_opportunities_to_features(rows) + assert float(feats.loc["1", "total_gift_amount"]) == 25.0 + + +def test_blank_stage_is_kept(): + """An unlabelled row cannot be shown to be a pledge, so it stays.""" + rows = [{"Account ID": "1", "Close Date": "2025-01-01", + "Amount": "10.00", "Stage": ""}, + {"Account ID": "1", "Close Date": "2025-01-02", + "Amount": "5.00", "Stage": None}] + feats = npsp_opportunities_to_features(rows) + assert float(feats.loc["1", "total_gift_amount"]) == 15.0 + assert int(feats.loc["1", "gift_count"]) == 2 + + +def test_custom_exclude_set_replaces_the_default(opportunities): + feats = npsp_opportunities_to_features(opportunities, exclude_stages=("Closed Won",)) + # Pledged is no longer excluded; the two Closed Won rows are. + assert float(feats.loc["88", "total_gift_amount"]) == 100.0 + + +def test_exclude_none_counts_every_row(opportunities): + feats = npsp_opportunities_to_features(opportunities, exclude_stages=None) + assert float(feats.loc["88", "total_gift_amount"]) == 300.0 + assert int(feats.loc["88", "gift_count"]) == 3 + + +def test_empty_exclude_sequence_counts_every_row(opportunities): + feats = npsp_opportunities_to_features(opportunities, exclude_stages=()) + assert float(feats.loc["88", "total_gift_amount"]) == 300.0 + + +def test_missing_stage_column_warns(): + rows = [{"Account ID": "1", "Close Date": "2025-01-01", "Amount": "10.00"}] + with pytest.warns(UserWarning, match="no Stage column"): + feats = npsp_opportunities_to_features(rows) + assert float(feats.loc["1", "total_gift_amount"]) == 10.0 + + +def test_exclude_none_does_not_warn_about_a_missing_stage_column(): + rows = [{"Account ID": "1", "Close Date": "2025-01-01", "Amount": "10.00"}] + with warnings.catch_warnings(): + warnings.simplefilter("error") + npsp_opportunities_to_features(rows, exclude_stages=None) + + +def test_default_excluded_set_is_documented_and_non_empty(): + assert "Pledged" in DEFAULT_EXCLUDED_STAGES + + +# --------------------------------------------------------------------------- # +# Header dialects +# --------------------------------------------------------------------------- # +def test_raw_api_field_names_are_accepted(): + """The Opportunity API returns AccountId / CloseDate / Amount / StageName.""" + rows = [{"AccountId": "7", "CloseDate": "2025-05-01", "Amount": "300.00", + "StageName": "Closed Won"}, + {"AccountId": "7", "CloseDate": "2025-05-02", "Amount": "999.00", + "StageName": "Pledged"}] + feats = npsp_opportunities_to_features(rows) + assert float(feats.loc["7", "total_gift_amount"]) == 300.0 + + +def test_data_import_template_headers_are_accepted(): + """NPSP's own Data Import tool spells them Donation Date / Donation + Amount / Donation Stage.""" + rows = [{"Account ID": "7", "Donation Date": "2025-05-01", + "Donation Amount": "300.00", "Donation Stage": "Closed Won"}] + feats = npsp_opportunities_to_features(rows) + assert float(feats.loc["7", "total_gift_amount"]) == 300.0 + + +def test_primary_contact_is_accepted_as_the_donor_key(): + rows = [{"Primary Contact": "7", "Close Date": "2025-05-01", + "Amount": "300.00", "Stage": "Closed Won"}] + feats = npsp_opportunities_to_features(rows) + assert float(feats.loc["7", "total_gift_amount"]) == 300.0 + + +def test_export_labels_and_api_names_agree(opportunities): + api = [ + {"AccountId": o["Account ID"], "CloseDate": o["Close Date"], + "Amount": o["Amount"], "StageName": o["Stage"], + "RecordType.Name": o["Record Type"], "email": o["Email"], + "first_name": o["First Name"], "last_name": o["Last Name"]} + for o in opportunities + ] + pd.testing.assert_frame_equal( + npsp_opportunities_to_features(opportunities), + npsp_opportunities_to_features(api), + ) + + +def test_currency_formatted_amounts_parse(): + rows = [{"Account ID": "1", "Close Date": "2025-01-01", + "Amount": "$1,250.00", "Stage": "Closed Won"}] + feats = npsp_opportunities_to_features(rows) + assert float(feats.loc["1", "total_gift_amount"]) == 1250.0 + + +# --------------------------------------------------------------------------- # +# Schema, identity and recency +# --------------------------------------------------------------------------- # +def test_schema_and_dtypes(opportunities): + feats = npsp_opportunities_to_features(opportunities) + assert list(feats.columns) == list(_FEATURE_DTYPES) + assert feats.index.name == "contact_id" + + +def test_record_type_feeds_distinct_financial_types(opportunities): + feats = npsp_opportunities_to_features(opportunities, exclude_stages=None) + # Donor 88's three rows sit on Donation, Donation and Recurring Donation. + assert int(feats.loc["88", "distinct_financial_types"]) == 2 + + +def test_carries_identity_fields(opportunities): + feats = npsp_opportunities_to_features(opportunities) + assert feats.loc["88", "constituent_email"] == "ada@amc.edu" + assert feats.loc["88", "first_name"] == "Ada" + assert feats.loc["88", "last_name"] == "Lovelace" + + +def test_reference_date_shifts_recency(opportunities): + feats = npsp_opportunities_to_features(opportunities, reference_date="2025-12-31") + assert int(feats.loc["88", "recency_days"]) == 324 + + +def test_recency_anchors_on_the_batch_by_default(opportunities): + feats = npsp_opportunities_to_features(opportunities) + # Latest surviving row is donor 88's 2025-02-10 Closed Won instalment. + assert int(feats.loc["88", "recency_days"]) == 0 + + +def test_empty_input_returns_a_typed_empty_frame(): + feats = npsp_opportunities_to_features([]) + assert feats.empty + assert list(feats.columns) == list(_FEATURE_DTYPES) + + +def test_missing_required_column_names_the_npsp_fields(): + rows = [{"Account ID": "1", "Stage": "Closed Won"}] + with pytest.raises(KeyError) as excinfo: + npsp_opportunities_to_features(rows) + message = str(excinfo.value) + assert "NPSP" in message + assert "Close Date" in message + # A CiviCRM-worded error here would read as "you used the wrong reader". + assert "CiviCRM" not in message + + +# --------------------------------------------------------------------------- # +# Reading +# --------------------------------------------------------------------------- # +def test_read_single_csv(tmp_path): + path = tmp_path / "opportunities.csv" + path.write_text(_export(_row("88", "2025-01-10", "100.00", "Pledged"), + _row("88", "2025-02-10", "100.00", "Closed Won"))) + raw = read_npsp_opportunities(path) + assert len(raw) == 2 + # Reading is lossless: the pledged row is still there. + feats = npsp_opportunities_to_features(raw) + assert float(feats.loc["88", "total_gift_amount"]) == 100.0 + + +def test_read_directory_concatenates(tmp_path): + (tmp_path / "jan.csv").write_text( + _export(_row("88", "2025-01-10", "100.00", "Closed Won")) + ) + (tmp_path / "feb.csv").write_text( + _export(_row("88", "2025-02-10", "250.00", "Closed Won")) + ) + feats = npsp_opportunities_to_features(read_npsp_opportunities(tmp_path)) + assert float(feats.loc["88", "total_gift_amount"]) == 350.0 + + +def test_read_missing_path_raises(tmp_path): + with pytest.raises(FileNotFoundError): + read_npsp_opportunities(tmp_path / "nope.csv") + + +def test_numeric_looking_account_ids_stay_text(tmp_path): + """dtype=str on the read keeps 0088 from becoming 88, and a blank in the + column from turning the whole thing into floats.""" + path = tmp_path / "opportunities.csv" + path.write_text(_export(_row("0088", "2025-01-10", "100.00", "Closed Won"), + _row("", "2025-01-11", "50.00", "Closed Won"))) + feats = npsp_opportunities_to_features(read_npsp_opportunities(path)) + assert list(feats.index) == ["0088"]