From 67319bfbcdf97ef659eb0960e74b51fe509e000c Mon Sep 17 00:00:00 2001 From: Shivam Lalakiya <50960482+shivamlalakiya@users.noreply.github.com> Date: Fri, 25 Sep 2026 14:06:04 -0500 Subject: [PATCH 1/2] Add score_upgrade_prospects to train and score the leadership-upgrade model score_upgrade_prospects (philanthropy.models) is the fit-and-score entry point over build_upgrade_snapshots: it trains a MajorGiftClassifier on every fully-resolved historical fiscal year, validates it with a walk-forward FiscalYearGroupedSplitter fold, and scores today's band-qualifying donors with a model refit on all history. The scored output carries an affinity score, rank, decile, a per-donor top-reasons heuristic built from global permutation importance, and a suggested-ask column left NaN (no ask-amount label exists yet to train one honestly). The report includes training-row counts, a low-data warning under ~500 rows, the activities_to_features id-match warning, and a top-N upgrade-rate lift over the naive "highest FY total" rule. build_upgrade_snapshots itself is refactored (no behaviour change) to split its per-year feature logic into two private helpers so the new function can build the same features for an unlabelled "current" row without duplicating the target/leakage logic. Wired into the CLI: `philanthropy train --task upgrade` reads a raw gift export via --source and calls score_upgrade_prospects directly, writing a scored CSV and printing the report; `philanthropy features` gained repeated --activity TYPE=PATH and --as-of flags to fold engagement data into the feature table the same way. --- CHANGELOG.md | 17 + docs/how-to/use_the_cli.md | 22 ++ docs/reference/index.md | 2 + philanthropy/cli.py | 164 ++++++++- philanthropy/ingest/_upgrade_snapshots.py | 147 +++++--- philanthropy/models/__init__.py | 2 + philanthropy/models/_upgrade.py | 424 ++++++++++++++++++++++ tests/test_cli.py | 183 ++++++++++ tests/test_score_upgrade_prospects.py | 271 ++++++++++++++ tests/test_sklearn_compliance.py | 5 +- 10 files changed, 1183 insertions(+), 54 deletions(-) create mode 100644 philanthropy/models/_upgrade.py create mode 100644 tests/test_score_upgrade_prospects.py diff --git a/CHANGELOG.md b/CHANGELOG.md index 4916455..18d4da3 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -36,6 +36,23 @@ Format: [Keep a Changelog](https://keepachangelog.com/en/1.1.0/) Everything but `target` is computed from data through the end of T; the output's `fiscal_year` and donor-id columns feed directly into `FiscalYearGroupedSplitter`. +- `philanthropy.models.score_upgrade_prospects(gifts, *, activities=None, + donors=None, threshold=1000.0, band=(100.0, 999.0), fiscal_year_start=7, + as_of=None, random_state=None)`: the fit-and-score entry point over + `build_upgrade_snapshots`. Trains a `MajorGiftClassifier` on every + fully-resolved historical fiscal year, validated with a walk-forward + `FiscalYearGroupedSplitter` fold, then scores today's band-qualifying + donors (cut at `as_of`, never at a future fiscal-year end) with a model + refit on all history. Returns a `(scores, report)` pair: `scores` has + `affinity_score`, `rank`, `decile`, a per-donor `top_reasons` heuristic + built from global permutation importance, and a `suggested_ask` left + `NaN` (no ask-amount label exists yet to train one honestly); `report` + carries training-row counts, a low-data warning under ~500 rows, the + `activities_to_features` id-match warning, and a top-N upgrade-rate lift + over the naive "highest FY total" rule. Wired into the CLI as + `philanthropy train --task upgrade`; `philanthropy features` gained + repeated `--activity TYPE=PATH` and `--as-of` flags to fold engagement + data into the feature table the same way. ## [0.8.0] - 2026-09-24 diff --git a/docs/how-to/use_the_cli.md b/docs/how-to/use_the_cli.md index 32a0b0f..daf7009 100644 --- a/docs/how-to/use_the_cli.md +++ b/docs/how-to/use_the_cli.md @@ -33,6 +33,15 @@ Header spelling is normalised for you, so a desktop Export (`Constituent ID`, `G `train` needs a `--target` column and a gift export contains no such column. Deciding who counts as a major donor, who lapsed, or who is a planned-giving prospect is yours to define; `features` gets you the predictors, not the answer key. +### Fold in engagement data + +Repeat `--activity TYPE=PATH` to add `philanthropy.ingest.activities_to_features` columns (event attendance, volunteer hours, ...) alongside the gift features. Each file is tagged with its own type and concatenated into one activity log; `--as-of` sets the cutoff (default: the latest `activity_date` across every file given). + +```bash +philanthropy features --source raisers_edge --data gifts.csv \ + --activity event=events.csv --activity volunteer=shifts.csv \ + --as-of 2025-06-30 --out features.csv +``` ## Train a model from a labelled CSV @@ -49,6 +58,19 @@ philanthropy train \ `--model` accepts `DonorPropensityModel` (the default), `MajorGiftClassifier`, `LapsePredictor`, or `PlannedGivingIntentScorer`. `--random-state` defaults to `0`, so a rerun on the same CSV gives the same model. +### `--task upgrade`: train and score the leadership-upgrade model in one call + +`--task upgrade` is a different shape of `train`: `--data` is a **raw** gift export (not a pre-built features CSV), read via `--source` the same way `features` does, and it calls `philanthropy.models.score_upgrade_prospects` directly rather than fitting `--model` on `--features`/`--target`. That one call trains on every fully-resolved historical fiscal year and scores today's band-qualifying donors, so `--out` here is a **scored CSV**, not a saved model bundle, and a validation report prints to stdout the way `validate`'s metrics do. + +```bash +philanthropy train --task upgrade --source raisers_edge --data gifts.csv \ + --threshold 1000 --band 100 999 --fiscal-year-start 7 \ + --activity event=events.csv --donors donors.csv \ + --out upgrade_scores.csv --random-state 0 +``` + +`--threshold`, `--band`, `--fiscal-year-start`, `--activity` and `--as-of` mirror `score_upgrade_prospects`'s own parameters; `--donors` is an optional CSV of static donor attributes with a `donor_id` column. See the function's docstring for the scored columns (`affinity_score`, `rank`, `decile`, `top_reasons`, `suggested_ask`) and the report fields printed after the CSV is written. + ## Score a prospect list `score` reuses the feature list stored in the bundle, so you do not repeat `--features`. Omit `--out` and the CSV goes to stdout, which pipes. diff --git a/docs/reference/index.md b/docs/reference/index.md index 367857c..86afba3 100755 --- a/docs/reference/index.md +++ b/docs/reference/index.md @@ -59,6 +59,7 @@ Everything reachable from `philanthropy.__all__` is listed below. A symbol not l | `map_columns` | `ingest` | The one-error-per-missing-column message shape may still change. | | `activities_to_features` | `ingest` | The activity-type feature set (`_count_12m`, `_distinct`, ...) may grow as more source types are onboarded. | | `build_upgrade_snapshots` | `ingest` | The gift-derived feature set (trend, consecutive years given, ...) is a minimal starting recipe and is likely to be refined. | +| `score_upgrade_prospects` | `models` | The top-reasons heuristic (z-score within the scored population weighted by global permutation importance) and the top-N/lift report shape are a starting recipe over `build_upgrade_snapshots`, likely to be refined; `suggested_ask` is left `NaN` pending a real ask-amount training signal. | | `plot_affinity_distribution`, `plot_retention_waterfall` | `visualisation` | Chart composition is presentation, not contract. | | `fetch_kdd98_donors` | `datasets` | Returns the raw upstream columns untyped; may gain as-of date parsing as the real-data leakage replication in #124 lands. | | `make_donor_panel` | `datasets` | The returned dict may gain keys (pledges, appeals, soft credits) as more of the library needs panel-shaped fixtures; existing keys and their columns will not change silently. | @@ -88,6 +89,7 @@ Every domain method returns a number on its own scale. None of them are calibrat | `GratefulPatientFeaturizer.transform` | `(n, 4)` float | Unbounded counts and weighted sums, all ≥ 0 | | `AskAmountRecommender.ask_ladder` | `(n, 3)` float | **Dollars**, not a score: conservative / target / stretch | | `MovesManagementClassifier.action_priority` | `dict` | Not an array: `stage`, `confidence` (0–1), `portfolio_summary` | +| `score_upgrade_prospects` | `DataFrame` | `affinity_score` 0–100 (from `MajorGiftClassifier`), plus `rank`, `decile`; `suggested_ask` is `NaN` (see the stability-tier note) | | `FinancialForecastModel.predict_revenue_forecast` | `(horizon,)` float | **Dollars per future period**, length is `horizon`, not `len(X)` | | `GiftIntervalCalibrator.predict_gift_interval` | `GiftInterval` | Not a score: two `(n,)` dollar bounds, plus the `attained_level` they certify, which is `r / (n + 1)` and not the requested `1 - alpha` | diff --git a/philanthropy/cli.py b/philanthropy/cli.py index d375b99..2dacc84 100644 --- a/philanthropy/cli.py +++ b/philanthropy/cli.py @@ -13,11 +13,18 @@ `features` rolls a raw CRM gift export up into the donor-level table the models consume, so the path from an export to a scored CSV needs no Python. It does not invent a label: `train` still needs a `--target` column, and constructing one -from your own definition of a major donor stays your job. +from your own definition of a major donor stays your job. `features` also +accepts repeated `--activity TYPE=PATH` flags to fold engagement data in +alongside the gift features (see `activities_to_features`). `train` saves a self-describing bundle (the fitted model, the feature list, and the scikit-learn / philanthropy versions used); `score` and `validate` reuse the feature list stored in that bundle unless you override it with `--features`. +`train --task upgrade` is a different shape entirely: it reads a raw gift +export (not a pre-built features CSV) and calls +`philanthropy.models.score_upgrade_prospects` directly, which trains on +history and scores today's prospects in one call, so `--out` there is a +scored CSV, not a saved model bundle. """ from __future__ import annotations @@ -87,6 +94,71 @@ def _split_features(features: Optional[str]) -> Optional[List[str]]: return [f.strip() for f in features.split(",") if f.strip()] +def _read_activities(activity_specs: Sequence[str]) -> pd.DataFrame: + """Read every ``--activity TYPE=PATH`` flag into one long activity table. + + Each file is tagged with its own ``TYPE`` (overwriting any + ``activity_type`` column it already has) before the files are + concatenated, so a donor's engagement across sources lands in one table + keyed like ``activities_to_features`` expects. + """ + from .utils._validation import ensure_local_path + + frames = [] + for spec in activity_specs: + if "=" not in spec: + raise SystemExit(f"--activity must be TYPE=PATH, got {spec!r}.") + activity_type, path = spec.split("=", 1) + ensure_local_path(path, "activity") + try: + df = pd.read_csv(path) + except FileNotFoundError: + raise SystemExit(f"Data file not found: {path}") + df["activity_type"] = activity_type + frames.append(df) + return pd.concat(frames, ignore_index=True) + + +def _read_raw_gifts(source: str, path: str) -> pd.DataFrame: + """Read a raw CRM gift export and normalise it to ``donor_id`` / + ``gift_date`` / ``gift_amount``, the shape ``score_upgrade_prospects`` + (via ``build_upgrade_snapshots``) needs. + + Reuses each source's own header-canonicalisation function (the same one + its ``*_to_features`` aggregator calls internally) rather than + re-deriving the CRM's column aliases here; renaming its three canonical + columns to the gift-log names is the only new logic. + """ + from . import ingest + from .ingest._civicrm import _normalise_headers + from .utils._validation import ensure_local_path + + ensure_local_path(path, "data") + try: + if source == "raisers_edge": + from .ingest._raisers_edge import _canonical_raisers_edge + + raw = _normalise_headers(ingest.read_raisers_edge_gifts(path), _canonical_raisers_edge) + elif source == "npsp": + from .ingest._npsp import _canonical_npsp + + raw = _normalise_headers(ingest.read_npsp_opportunities(path), _canonical_npsp) + else: + raw = ingest.read_civicrm_contributions(path) + except FileNotFoundError: + raise SystemExit(f"Data file not found: {path}") + + missing = [c for c in ("contact_id", "receive_date", "total_amount") if c not in raw.columns] + if missing: + raise SystemExit( + f"Column(s) {missing} not found in {path} after normalisation. " + f"Available: {list(raw.columns)}." + ) + return raw.rename( + columns={"contact_id": "donor_id", "receive_date": "gift_date", "total_amount": "gift_amount"} + ) + + def _load_bundle(path: str) -> Dict[str, Any]: from .utils import load_model @@ -132,11 +204,17 @@ def _neutralise_csv_injection(df: pd.DataFrame) -> pd.DataFrame: def _cmd_train(args: argparse.Namespace) -> None: + if args.task == "upgrade": + _cmd_train_upgrade(args) + return + features = _split_features(args.features) # Falsy, not `is None`: "--features ' , '" parses to [] and used to reach # fit() with a zero-column matrix. if not features: raise SystemExit("train requires --features (comma-separated column names).") + if not args.target: + raise SystemExit("train requires --target.") df = _read_csv(args.data) _require_columns(df, features + [args.target], args.data) @@ -149,6 +227,46 @@ def _cmd_train(args: argparse.Namespace) -> None: print(f"Trained {args.model} on {len(df)} rows; saved to {args.out}") +def _cmd_train_upgrade(args: argparse.Namespace) -> None: + if not args.source: + raise SystemExit("train --task upgrade requires --source.") + + from .models import score_upgrade_prospects + + gifts = _read_raw_gifts(args.source, args.data) + activities = _read_activities(args.activity) if args.activity else None + + donors = None + if args.donors: + donors = _read_csv(args.donors) + if "donor_id" not in donors.columns: + raise SystemExit( + f"--donors file must have a 'donor_id' column; got {list(donors.columns)}." + ) + donors = donors.set_index("donor_id") + + scores, report = score_upgrade_prospects( + gifts, + activities=activities, + donors=donors, + threshold=args.threshold, + band=tuple(args.band), + fiscal_year_start=args.fiscal_year_start, + as_of=args.as_of, + random_state=args.random_state, + ) + + out = _neutralise_csv_injection(scores.reset_index()) + if args.out: + out.to_csv(args.out, index=False) + print(f"Wrote {len(out)} scored rows to {args.out}") + else: + out.to_csv(sys.stdout, index=False) + + for key, value in report.items(): + print(f"{key}: {value}") + + def _cmd_score(args: argparse.Namespace) -> None: bundle = _load_bundle(args.model) features = _split_features(args.features) or bundle["features"] @@ -222,6 +340,13 @@ def _cmd_features(args: argparse.Namespace) -> None: # whole answer, so surface it instead of a traceback. raise SystemExit(str(exc.args[0] if exc.args else exc)) + if args.activity: + activities = _read_activities(args.activity) + as_of = args.as_of or pd.to_datetime(activities["activity_date"], errors="coerce").max() + act_feats = ingest.activities_to_features(activities, as_of=as_of, donors=features) + # Both sides already key on contact_id: a plain left join, no rename. + features = features.join(act_feats, how="left") + # reset_index first: contact_id is the index, and _neutralise_csv_injection # only walks columns, so an index left in place would skip the escaping and # then be written to the CSV anyway by to_csv(index=True). @@ -264,16 +389,45 @@ def _build_parser() -> argparse.ArgumentParser: features.add_argument( "--data", required=True, help="gift export CSV, or a directory of them" ) + features.add_argument( + "--activity", action="append", default=[], metavar="TYPE=PATH", + help="an activity log CSV tagged with its type, e.g. event=events.csv; " + "repeatable. Adds activities_to_features columns to the output.", + ) + features.add_argument( + "--as-of", default=None, dest="as_of", + help="cutoff date for --activity features (default: the latest " + "activity_date in the combined activity log)", + ) features.add_argument("--out", default=None, help="output CSV path (default: stdout)") features.set_defaults(func=_cmd_features) train = sub.add_parser("train", help="Train a model from a labelled CSV and save it.") - train.add_argument("--data", required=True, help="path to a labelled CSV") - train.add_argument("--target", required=True, help="name of the label column") - train.add_argument("--features", required=True, help="comma-separated feature columns") + train.add_argument("--task", choices=("plain", "upgrade"), default="plain", + help="'plain': fit --model on --features/--target (default). " + "'upgrade': read a raw gift export via --source and call " + "score_upgrade_prospects, writing scored donors to --out.") + train.add_argument("--data", required=True, help="labelled CSV (plain) or raw gift export (upgrade)") + train.add_argument("--target", default=None, help="name of the label column (plain)") + train.add_argument("--features", default=None, help="comma-separated feature columns (plain)") train.add_argument("--model", default="DonorPropensityModel", choices=_MODEL_CHOICES) - train.add_argument("--out", required=True, help="output model bundle path (.joblib)") + train.add_argument("--out", required=True, help="output model bundle (plain) or scored CSV (upgrade)") train.add_argument("--random-state", type=int, default=0, dest="random_state") + train.add_argument("--source", default=None, choices=_FEATURE_SOURCES, + help="which CRM the gift export came from (upgrade)") + train.add_argument("--threshold", type=float, default=1000.0, + help="leadership-giving level an upgrade crosses into (upgrade)") + train.add_argument("--band", type=float, nargs=2, default=(100.0, 999.0), + metavar=("LOW", "HIGH"), + help="upgrade-candidate FY-total band (upgrade)") + train.add_argument("--fiscal-year-start", type=int, default=7, dest="fiscal_year_start", + help="month (1-12) the fiscal year begins (upgrade)") + train.add_argument("--activity", action="append", default=[], metavar="TYPE=PATH", + help="an activity log CSV tagged with its type; repeatable (upgrade)") + train.add_argument("--donors", default=None, + help="optional donor-attributes CSV with a donor_id column (upgrade)") + train.add_argument("--as-of", default=None, dest="as_of", + help="scoring cutoff (default: the latest gift date) (upgrade)") train.set_defaults(func=_cmd_train) score = sub.add_parser("score", help="Score a CSV with a saved model.") diff --git a/philanthropy/ingest/_upgrade_snapshots.py b/philanthropy/ingest/_upgrade_snapshots.py index 1b25935..f4d61ce 100644 --- a/philanthropy/ingest/_upgrade_snapshots.py +++ b/philanthropy/ingest/_upgrade_snapshots.py @@ -154,33 +154,7 @@ def build_upgrade_snapshots( if low > high: raise ValueError(f"band[0] ({low}) must be <= band[1] ({high}).") - df = _to_frame(gifts) - missing = [col for col in _REQUIRED if col not in df.columns] - if missing: - raise KeyError( - f"gifts is missing {missing}. Every gift row needs 'donor_id', " - f"'gift_date' and 'gift_amount'; got {sorted(df.columns)}." - ) - - df = df.copy() - df["donor_id"] = df["donor_id"].astype("string").str.strip() - df["_date"] = pd.to_datetime(df["gift_date"], errors="coerce") - df["_amount"] = pd.to_numeric(df["gift_amount"], errors="coerce") - df = df[df["donor_id"].notna() & df["_date"].notna() & df["_amount"].notna()] - - if not df.empty: - fy = ( - FiscalYearTransformer(fiscal_year_start=fiscal_year_start) - .set_output(transform="pandas") - .fit_transform(df[["_date"]].rename(columns={"_date": "gift_date"})) - ) - df["_fy"] = fy["fiscal_year"].to_numpy() - df = df[df["_fy"].notna()] - df["_fy"] = df["_fy"].astype("int64") - - pivot_sum = _pivot(df, "sum") - pivot_max = _pivot(df, "max") - pivot_count = _pivot(df, "count") + df, pivot_sum, pivot_max, pivot_count = _prepare_gifts(gifts, fiscal_year_start) donors_norm = None if donors is not None: @@ -196,27 +170,10 @@ def build_upgrade_snapshots( continue donor_ids = candidates.index - snap = pd.DataFrame(index=donor_ids) - snap.index.name = "donor_id" - snap["fiscal_year"] = fy_t - snap["fy_total"] = candidates - snap["fy_total_prior1"] = _column(pivot_sum, fy_t - 1).reindex(donor_ids, fill_value=0.0) - snap["fy_total_prior2"] = _column(pivot_sum, fy_t - 2).reindex(donor_ids, fill_value=0.0) - snap["fy_trend"] = snap["fy_total"] - snap["fy_total_prior1"] - snap["largest_gift"] = _column(pivot_max, fy_t).reindex(donor_ids, fill_value=0.0) - snap["gift_count"] = _column(pivot_count, fy_t).reindex(donor_ids, fill_value=0).astype("int64") - snap["consecutive_years_given"] = _consecutive_years_given(pivot_sum, donor_ids, fy_t) - - fy_end = _fy_end(fy_t, fiscal_year_start) - last_gift = df[df["_fy"] <= fy_t].groupby("donor_id")["_date"].max().reindex(donor_ids) - snap["months_since_last_gift"] = ((fy_end - last_gift).dt.days / 30.0).round(2) - - if activities is not None: - act_feats = activities_to_features(activities, as_of=fy_end, donors=donors) - snap = snap.join(act_feats, how="left") - - if donors_norm is not None: - snap = snap.join(donors_norm, how="left") + snap = _snapshot_features_for_year( + df, pivot_sum, pivot_max, pivot_count, donor_ids, fy_t, + fiscal_year_start, activities, donors, donors_norm, + ) next_totals = _column(pivot_sum, fy_t + 1).reindex(donor_ids, fill_value=0.0) snap["target"] = (next_totals >= threshold).astype("int64") @@ -233,6 +190,12 @@ def build_upgrade_snapshots( # --------------------------------------------------------------------------- # # Internals +# +# ``_prepare_gifts`` and ``_snapshot_features_for_year`` are also imported +# directly (via ``philanthropy.ingest._upgrade_snapshots``) by +# ``philanthropy.models.score_upgrade_prospects``, which needs the same +# donor/fiscal-year feature logic for an unlabelled "current" row that this +# function's target computation (reading FY T+1) does not apply to. # --------------------------------------------------------------------------- # def _to_frame(gifts: Union[Iterable[Mapping], pd.DataFrame]) -> pd.DataFrame: if isinstance(gifts, pd.DataFrame): @@ -240,6 +203,94 @@ def _to_frame(gifts: Union[Iterable[Mapping], pd.DataFrame]) -> pd.DataFrame: return pd.DataFrame(list(gifts)) +def _prepare_gifts( + gifts: Union[Iterable[Mapping], pd.DataFrame], fiscal_year_start: int +) -> Tuple[pd.DataFrame, pd.DataFrame, pd.DataFrame, pd.DataFrame]: + """Normalise a raw gift log and pivot it donor x fiscal-year. + + Returns ``(df, pivot_sum, pivot_max, pivot_count)``: ``df`` is the + cleaned, per-gift frame (with ``donor_id``, ``_date``, ``_amount`` and + ``_fy`` columns) and the three pivots aggregate ``_amount`` over it. + """ + df = _to_frame(gifts) + missing = [col for col in _REQUIRED if col not in df.columns] + if missing: + raise KeyError( + f"gifts is missing {missing}. Every gift row needs 'donor_id', " + f"'gift_date' and 'gift_amount'; got {sorted(df.columns)}." + ) + + df = df.copy() + df["donor_id"] = df["donor_id"].astype("string").str.strip() + df["_date"] = pd.to_datetime(df["gift_date"], errors="coerce") + df["_amount"] = pd.to_numeric(df["gift_amount"], errors="coerce") + df = df[df["donor_id"].notna() & df["_date"].notna() & df["_amount"].notna()] + + if not df.empty: + fy = ( + FiscalYearTransformer(fiscal_year_start=fiscal_year_start) + .set_output(transform="pandas") + .fit_transform(df[["_date"]].rename(columns={"_date": "gift_date"})) + ) + df["_fy"] = fy["fiscal_year"].to_numpy() + df = df[df["_fy"].notna()] + df["_fy"] = df["_fy"].astype("int64") + + pivot_sum = _pivot(df, "sum") + pivot_max = _pivot(df, "max") + pivot_count = _pivot(df, "count") + return df, pivot_sum, pivot_max, pivot_count + + +def _snapshot_features_for_year( + df: pd.DataFrame, + pivot_sum: pd.DataFrame, + pivot_max: pd.DataFrame, + pivot_count: pd.DataFrame, + donor_ids: pd.Index, + fy_t: int, + fiscal_year_start: int, + activities: Optional[Union[Iterable[Mapping], pd.DataFrame]], + donors: Optional[pd.DataFrame], + donors_norm: Optional[pd.DataFrame], + as_of: Optional[pd.Timestamp] = None, +) -> pd.DataFrame: + """Gift-derived (and, if given, activity/donor) feature columns for one + ``(donor_ids, fy_t)`` snapshot, everything ``build_upgrade_snapshots`` + computes except ``target``. + + ``as_of`` clips the recency cutoff (``months_since_last_gift`` and the + ``activities_to_features`` window) to a date inside a still-open fiscal + year, for scoring a "current", not-yet-resolved FY; it defaults to the + end of ``fy_t`` (``build_upgrade_snapshots``'s own, always-resolved case). + """ + snap = pd.DataFrame(index=donor_ids) + snap.index.name = "donor_id" + snap["fiscal_year"] = fy_t + snap["fy_total"] = _column(pivot_sum, fy_t).reindex(donor_ids, fill_value=0.0) + snap["fy_total_prior1"] = _column(pivot_sum, fy_t - 1).reindex(donor_ids, fill_value=0.0) + snap["fy_total_prior2"] = _column(pivot_sum, fy_t - 2).reindex(donor_ids, fill_value=0.0) + snap["fy_trend"] = snap["fy_total"] - snap["fy_total_prior1"] + snap["largest_gift"] = _column(pivot_max, fy_t).reindex(donor_ids, fill_value=0.0) + snap["gift_count"] = _column(pivot_count, fy_t).reindex(donor_ids, fill_value=0).astype("int64") + snap["consecutive_years_given"] = _consecutive_years_given(pivot_sum, donor_ids, fy_t) + + cutoff = _fy_end(fy_t, fiscal_year_start) + if as_of is not None and as_of < cutoff: + cutoff = as_of + last_gift = df[df["_fy"] <= fy_t].groupby("donor_id")["_date"].max().reindex(donor_ids) + snap["months_since_last_gift"] = ((cutoff - last_gift).dt.days / 30.0).round(2) + + if activities is not None: + act_feats = activities_to_features(activities, as_of=cutoff, donors=donors) + snap = snap.join(act_feats, how="left") + + if donors_norm is not None: + snap = snap.join(donors_norm, how="left") + + return snap + + def _pivot(df: pd.DataFrame, aggfunc: str) -> pd.DataFrame: """Donor x fiscal-year table of ``aggfunc`` over gift amounts. diff --git a/philanthropy/models/__init__.py b/philanthropy/models/__init__.py index 40cfeb9..517ea92 100755 --- a/philanthropy/models/__init__.py +++ b/philanthropy/models/__init__.py @@ -13,6 +13,7 @@ from ._planned_giving import PlannedGivingIntentScorer from ._forecast import FinancialForecastModel from ._conformal_interval import GiftInterval, GiftIntervalCalibrator +from ._upgrade import score_upgrade_prospects __all__ = [ "AskAmountRecommender", @@ -26,4 +27,5 @@ "PropensityScorer", "ShareOfWalletRegressor", "PlannedGivingIntentScorer", + "score_upgrade_prospects", ] diff --git a/philanthropy/models/_upgrade.py b/philanthropy/models/_upgrade.py new file mode 100644 index 0000000..c3d8977 --- /dev/null +++ b/philanthropy/models/_upgrade.py @@ -0,0 +1,424 @@ +""" +philanthropy.models._upgrade +============================= +Fit-and-score entry point for the mid-level-to-leadership "upgrade" model: +which currently mid-level donors are most likely to cross the leadership +``threshold`` next fiscal year? + +``score_upgrade_prospects`` is the one-call version of the workflow +:func:`~philanthropy.ingest.build_upgrade_snapshots` sets up: every +fully-resolved historical ``(donor, fiscal year)`` pair becomes a training +row, a :class:`~philanthropy.models.MajorGiftClassifier` is fit on the stack +with a walk-forward, fiscal-year-aware validation split, and the CURRENT +band-qualifying donors (an unlabelled snapshot as of ``as_of``) are scored +with a model refit on every historical row. + +This lives in ``philanthropy.models``, not ``philanthropy.ingest`` where +``build_upgrade_snapshots`` lives: unlike that function, which only reshapes +tables (no estimator involved, by that module's own design), this function's +work is mostly a model-selection split, a classifier fit, and a permutation- +importance call. It still calls ``build_upgrade_snapshots`` directly for the +historical half rather than re-deriving the same target/leakage logic, the +same reasoning that function gives for reusing ``activities_to_features``. +""" + +from __future__ import annotations + +import warnings +from typing import Any, Dict, Iterable, Mapping, Optional, Tuple, Union + +import numpy as np +import pandas as pd + +from philanthropy.ingest import build_upgrade_snapshots +from philanthropy.ingest._upgrade_snapshots import ( + _column, + _fy_end, + _prepare_gifts, + _snapshot_features_for_year, +) +from philanthropy.inspection import donor_feature_importance +from philanthropy.model_selection import FiscalYearGroupedSplitter + +from ._propensity import MajorGiftClassifier + +__all__ = ["score_upgrade_prospects"] + +_TOP_N = 10 +_TOP_REASONS = 3 +_LOW_DATA_ROWS = 500 + + +def score_upgrade_prospects( + gifts: Union[Iterable[Mapping], pd.DataFrame], + *, + activities: Optional[Union[Iterable[Mapping], pd.DataFrame]] = None, + donors: Optional[pd.DataFrame] = None, + threshold: float = 1000.0, + band: Tuple[float, float] = (100.0, 999.0), + fiscal_year_start: int = 7, + as_of: Optional[Union[str, pd.Timestamp]] = None, + random_state: Optional[int] = None, +) -> Tuple[pd.DataFrame, Dict[str, Any]]: + """Fit an upgrade model on history, then score today's band-qualifying donors. + + Every fiscal year ``T`` where both ``T`` and ``T+1`` are fully resolved as + of ``as_of`` is a labelled training row (via + :func:`~philanthropy.ingest.build_upgrade_snapshots`); a + :class:`~philanthropy.models.MajorGiftClassifier` is fit on the stack of + those years and evaluated on the most recent + :class:`~philanthropy.model_selection.FiscalYearGroupedSplitter` fold, an + honest, held-out read. A second copy of the model, refit on every + historical row (more data, no held-out fold to protect), scores the + CURRENT band-qualifying donors: one unlabelled row per donor, built the + same way but for the fiscal year containing ``as_of`` and cut at ``as_of`` + rather than at that year's end, since it may still be in progress. + + Parameters + ---------- + gifts : iterable of mapping, or DataFrame + Gift-level rows with ``donor_id``, ``gift_date`` and ``gift_amount``, + the same shape :func:`~philanthropy.ingest.build_upgrade_snapshots` + takes. + activities : iterable of mapping, or DataFrame, optional + A long activity log, forwarded to + :func:`~philanthropy.ingest.build_upgrade_snapshots` for the + historical years and to the same feature logic for the current row. + donors : DataFrame, optional + Static donor attributes, indexed by donor id. Forwarded the same way. + Only its numeric columns become model features (see Notes); it is + still joined and returned for a non-numeric attribute a caller wants + to inspect alongside the score. + threshold : float, default=1000.0 + The leadership-giving level an upgrade crosses into. + band : (float, float), default=(100.0, 999.0) + Inclusive bounds on FY giving that define the upgrade-candidate + population, exactly as in ``build_upgrade_snapshots``. + fiscal_year_start : int, default=7 + Month (1-12) the fiscal year begins. + as_of : str or datetime-like, optional + The scoring cutoff: nothing dated after it is ever read, in either + the historical training rows or the current scored row. Defaults to + the latest gift date in ``gifts``, the same leakage-free default + every other ``reference_date``-style parameter in this package uses. + random_state : int, optional + Seed forwarded to the classifier fits and to the permutation + importance call, for reproducible scores and reasons. + + Returns + ------- + scores : pandas.DataFrame + One row per currently band-qualifying donor, indexed by ``donor_id``, + sorted by ``affinity_score`` descending. Columns: ``fiscal_year`` (the + current, possibly still-open FY); ``affinity_score`` (0-100, see + :meth:`~philanthropy.models.MajorGiftClassifier.predict_affinity_score`); + ``rank`` (1 = highest score); ``decile`` (1 = top 10% by rank, 10 = + bottom); ``top_reasons`` (a tuple of up to 3 ``(feature_name, + donor_value)`` pairs, see Notes); ``suggested_ask`` (``NaN`` -- see + Notes). Empty (but correctly typed) if no donor currently qualifies + for ``band``. + report : dict + ``n_training_rows``, ``n_training_fiscal_years``, ``n_scored``; + ``low_data_warning`` / ``low_data_message`` (flagged under roughly + ``500`` training rows); ``activity_id_match_warnings`` (list of str, + captured from ``activities_to_features``'s own low-match-rate + warning, not recomputed here); ``validated`` (whether a walk-forward + held-out fold existed at all), and, when it did, + ``validation_fiscal_year``, ``n_validation_rows``, ``top_n``, + ``model_upgrade_rate_top_n``, ``baseline_upgrade_rate_top_n`` + (the naive "highest FY total" rule, same ``top_n``), + ``overall_upgrade_rate``, and ``lift_over_baseline`` (the two rates' + ratio; ``None`` if the baseline rate is 0). + + Raises + ------ + ValueError + If ``band[0] > band[1]``, or if there is not one historical + ``(donor, fiscal year)`` row to train on as of ``as_of``. + KeyError + If ``gifts`` is missing ``donor_id``, ``gift_date`` or + ``gift_amount``. + + Notes + ----- + **Feature columns.** Only the numeric columns of the snapshot (the + gift-derived features, any numeric ``activities_to_features`` columns, + and any numeric ``donors`` columns) become model features; a non-numeric + ``donors`` column (e.g. a wealth-rating letter grade) is still joined and + returned for reference but dropped before fitting, the simplest rule + that needs no per-column encoding policy for a feature nobody asked this + function to build. A current-row feature column absent from history (or + vice versa), e.g. an activity type that only shows up in one window, is + reindexed to 0.0 rather than dropped, matching + ``activities_to_features``'s own "no rows of that type -> 0" rule. + + **Top reasons.** There is no per-instance explainer in this package + (SHAP is out of scope for this whole series); this reuses the existing + GLOBAL permutation importances (:func:`~philanthropy.inspection.donor_feature_importance`, + computed once on the held-out fold, or in-sample with a warning if there + was no fold to hold out) as fixed feature weights, then for each scored + donor ranks their OWN feature columns by ``|z-score within the current + scored population| * global_importance_weight`` (importance clipped at 0, + since a negative permutation importance means shuffling that feature + *helped*, not a reason to report), and keeps the top 3 as + ``(feature_name, donor_value)`` pairs. This is a heuristic, not a causal + attribution: it says which features are both unusual for this donor and + generally predictive, not "if X were different the score would change by + Y". It is deliberately simple rather than a general explainability + framework. + + **Suggested ask.** ``AskAmountRecommender`` needs its own ask-amount + label (what a gift officer actually asked for, or a realistic proxy for + it); nothing in ``build_upgrade_snapshots``'s output is that. Training it + on, say, the FY T+1 amount actually given would train on the very + quantity the upgrade label is derived from, an honesty problem, not a + convenience one. ``suggested_ask`` is left ``NaN``, out of scope for this + function until a real ask-amount training signal exists. + + Examples + -------- + Four donor archetypes across five fiscal years: a flat high-band donor, + an early riser that crosses the threshold in FY2022, a later riser that + crosses in FY2024, and a flat low-band donor. Only the two that never + cross ``threshold`` are still band-qualifying "today" (FY2025): + + >>> import pandas as pd + >>> from philanthropy.models import score_upgrade_prospects + >>> years = ["2020-08-01", "2021-08-01", "2022-08-01", "2023-08-01", "2024-08-01"] + >>> archetypes = { + ... "flat_high": [900, 900, 900, 900, 900], + ... "early_riser": [600, 1050, 1200, 1400, 1600], + ... "late_riser": [300, 500, 700, 1100, 1300], + ... "flat_low": [400, 400, 400, 400, 400], + ... } + >>> rows = [ + ... {"donor_id": f"{name}_{i}", "gift_date": year, "gift_amount": amount} + ... for name, amounts in archetypes.items() + ... for i in range(20) + ... for year, amount in zip(years, amounts) + ... ] + >>> gifts = pd.DataFrame(rows) + >>> scores, report = score_upgrade_prospects(gifts, random_state=0) + >>> list(scores.columns) + ['fiscal_year', 'affinity_score', 'rank', 'decile', 'top_reasons', 'suggested_ask'] + >>> len(scores) + 40 + >>> bool((scores["affinity_score"] >= 0).all() and (scores["affinity_score"] <= 100).all()) + True + >>> report["n_training_rows"] + 200 + >>> report["validated"] + True + """ + low, high = band + if low > high: + raise ValueError(f"band[0] ({low}) must be <= band[1] ({high}).") + + df_all, _, _, _ = _prepare_gifts(gifts, fiscal_year_start) + if df_all.empty: + raise ValueError( + "No usable gift rows (need donor_id, gift_date and gift_amount) " + "to train or score an upgrade model." + ) + as_of_ts = pd.Timestamp(as_of) if as_of is not None else df_all["_date"].max() + + # Nothing after as_of is ever read: re-derive the pivots from gifts cut at + # as_of, so the historical training years and the current row share one + # cutoff, and appending a future-dated row can never move either. + cut = df_all[df_all["_date"] <= as_of_ts] + if cut.empty: + raise ValueError(f"No gift rows on or before as_of={as_of_ts.date()}.") + df, pivot_sum, pivot_max, pivot_count = _prepare_gifts(cut, fiscal_year_start) + + current_fy = ( + as_of_ts.year + 1 if as_of_ts.month >= fiscal_year_start else as_of_ts.year + ) + min_fy, max_fy = int(df["_fy"].min()), int(df["_fy"].max()) + historical_years = [ + t for t in range(min_fy, max_fy + 1) + if _fy_end(t + 1, fiscal_year_start) <= as_of_ts + ] + + totals_current = _column(pivot_sum, current_fy) + current_candidates = totals_current[ + (totals_current >= low) & (totals_current <= high) & (totals_current < threshold) + ] + + donors_norm = None + if donors is not None: + donors_norm = donors.copy() + donors_norm.index = donors_norm.index.astype("string").str.strip() + + with warnings.catch_warnings(record=True) as caught: + warnings.simplefilter("always") + historical_snap = build_upgrade_snapshots( + df, fiscal_years=historical_years, threshold=threshold, band=band, + fiscal_year_start=fiscal_year_start, activities=activities, donors=donors, + ) + current_snap = None + if not current_candidates.empty: + current_snap = _snapshot_features_for_year( + df, pivot_sum, pivot_max, pivot_count, current_candidates.index, + current_fy, fiscal_year_start, activities, donors, donors_norm, + as_of=as_of_ts, + ) + activity_id_match_warnings = [str(w.message) for w in caught] + + if historical_snap.empty: + raise ValueError( + f"No historical (donor, fiscal year) rows to train on as of " + f"{as_of_ts.date()}: need at least one fiscal year T with both T " + "and T+1 fully resolved by then." + ) + + feature_cols = [ + c for c in historical_snap.columns + if c != "target" and pd.api.types.is_numeric_dtype(historical_snap[c]) + ] + X = historical_snap[feature_cols].to_numpy(dtype="float64") + y = historical_snap["target"].to_numpy() + fys = historical_snap["fiscal_year"].to_numpy() + n_unique_fys = int(np.unique(fys).size) + + low_data_warning = len(historical_snap) < _LOW_DATA_ROWS + low_data_message = None + if low_data_warning: + low_data_message = ( + f"Only {len(historical_snap)} historical donor-year rows " + f"(< {_LOW_DATA_ROWS}); scores and the validation report below " + "are low-confidence." + ) + warnings.warn(low_data_message, UserWarning, stacklevel=2) + + report: Dict[str, Any] = { + "n_training_rows": int(len(historical_snap)), + "n_training_fiscal_years": n_unique_fys, + "low_data_warning": bool(low_data_warning), + "low_data_message": low_data_message, + "activity_id_match_warnings": activity_id_match_warnings, + "current_fiscal_year": int(current_fy), + "n_scored": 0, + "validated": False, + "validation_fiscal_year": None, + "n_validation_rows": 0, + "top_n": None, + "model_upgrade_rate_top_n": None, + "baseline_upgrade_rate_top_n": None, + "overall_upgrade_rate": None, + "lift_over_baseline": None, + } + + if n_unique_fys >= 2: + n_splits = min(5, n_unique_fys - 1) + splitter = FiscalYearGroupedSplitter(n_splits=n_splits, drop_repeat_donors=False) + train_idx, test_idx = list(splitter.split(X, groups=fys))[-1] + + eval_model = MajorGiftClassifier(random_state=random_state).fit( + X[train_idx], y[train_idx] + ) + y_test = y[test_idx] + proba_test = eval_model.predict_proba(X[test_idx])[:, 1] + fy_total_test = historical_snap["fy_total"].to_numpy()[test_idx] + + top_n = min(_TOP_N, len(test_idx)) + model_top_n = np.argsort(-proba_test)[:top_n] + baseline_top_n = np.argsort(-fy_total_test)[:top_n] + baseline_rate = float(y_test[baseline_top_n].mean()) + + report.update({ + "validated": True, + "validation_fiscal_year": int(fys[test_idx][0]), + "n_validation_rows": int(len(test_idx)), + "top_n": int(top_n), + "model_upgrade_rate_top_n": float(y_test[model_top_n].mean()), + "baseline_upgrade_rate_top_n": baseline_rate, + "overall_upgrade_rate": float(y_test.mean()), + "lift_over_baseline": ( + float(y_test[model_top_n].mean() / baseline_rate) + if baseline_rate > 0 else None + ), + }) + importance_df = donor_feature_importance( + eval_model, X[test_idx], y[test_idx], feature_names=feature_cols, + random_state=random_state, + ) + else: + warnings.warn( + "Fewer than 2 distinct historical fiscal years, so no " + "walk-forward validation fold could be built. Scores are still " + "produced (fit on all historical rows), but there is no " + "held-out report, and the feature-importance weights behind " + "top_reasons are computed in-sample.", + UserWarning, stacklevel=2, + ) + in_sample_model = MajorGiftClassifier(random_state=random_state).fit(X, y) + importance_df = donor_feature_importance( + in_sample_model, X, y, feature_names=feature_cols, + random_state=random_state, + ) + + model = MajorGiftClassifier(random_state=random_state).fit(X, y) + + if current_snap is None or current_snap.empty: + scores = _empty_scores_frame() + else: + X_current = current_snap.reindex(columns=feature_cols, fill_value=0.0) + affinity = model.predict_affinity_score(X_current.to_numpy(dtype="float64")) + reasons = _top_reasons(X_current, importance_df) + + scores = pd.DataFrame( + { + "fiscal_year": current_snap["fiscal_year"].to_numpy(), + "affinity_score": affinity, + "top_reasons": reasons, + "suggested_ask": np.nan, + }, + index=current_snap.index, + ) + scores = scores.sort_values("affinity_score", ascending=False, kind="stable") + scores["rank"] = np.arange(1, len(scores) + 1) + scores["decile"] = np.ceil(scores["rank"] / len(scores) * 10).clip(upper=10).astype("int64") + scores = scores[ + ["fiscal_year", "affinity_score", "rank", "decile", "top_reasons", "suggested_ask"] + ] + + report["n_scored"] = int(len(scores)) + return scores, report + + +# --------------------------------------------------------------------------- # +# Internals +# --------------------------------------------------------------------------- # +def _top_reasons( + X: pd.DataFrame, importance_df: pd.DataFrame, top_k: int = _TOP_REASONS +) -> list: + """Per-donor top-``top_k`` ``(feature, value)`` reasons; see the + "Top reasons" note on :func:`score_upgrade_prospects`.""" + weights = ( + importance_df.set_index("feature")["importance_mean"] + .reindex(X.columns) + .fillna(0.0) + .clip(lower=0.0) + ) + z = (X - X.mean()).div(X.std(ddof=0).replace(0.0, np.nan)).fillna(0.0) + weighted = z.abs().mul(weights, axis=1) + + reasons = [] + for donor_id in X.index: + top_feats = weighted.loc[donor_id].sort_values(ascending=False, kind="stable").index[:top_k] + reasons.append(tuple((feat, X.loc[donor_id, feat]) for feat in top_feats)) + return reasons + + +def _empty_scores_frame() -> pd.DataFrame: + return pd.DataFrame( + { + "fiscal_year": pd.Series(dtype="int64"), + "affinity_score": pd.Series(dtype="float64"), + "rank": pd.Series(dtype="int64"), + "decile": pd.Series(dtype="int64"), + "top_reasons": pd.Series(dtype="object"), + "suggested_ask": pd.Series(dtype="float64"), + }, + index=pd.Index([], name="donor_id", dtype="object"), + ) diff --git a/tests/test_cli.py b/tests/test_cli.py index 36e1ba2..590d71f 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -303,3 +303,186 @@ def test_cli_features_then_train_then_score(tmp_path, capsys): scored = pd.read_csv(scores_path) assert "score" in scored.columns assert len(scored) == 80 + + +# --------------------------------------------------------------------------- # +# `features --activity` / `--as-of` +# --------------------------------------------------------------------------- # +def test_cli_features_activity_flag_adds_engagement_columns(tmp_path): + data = _make_gift_export(tmp_path, n_donors=3) + activity_path = tmp_path / "activities.csv" + activity_path.write_text("contact_id,activity_date\n0,2025-01-01\n1,2025-02-01\n") + out_path = tmp_path / "features.csv" + main(["features", "--source", "raisers_edge", "--data", str(data), + "--activity", f"event={activity_path}", "--out", str(out_path)]) + feats = pd.read_csv(out_path) + assert "event_count_12m" in feats.columns + row = feats.loc[feats["contact_id"] == 0].iloc[0] + assert row["event_count_12m"] == 1 + + +def test_cli_features_activity_as_of_cutoff(tmp_path): + data = _make_gift_export(tmp_path, n_donors=2) + activity_path = tmp_path / "activities.csv" + activity_path.write_text("contact_id,activity_date\n0,2025-06-01\n") + out_path = tmp_path / "features.csv" + main(["features", "--source", "raisers_edge", "--data", str(data), + "--activity", f"event={activity_path}", "--as-of", "2025-01-01", + "--out", str(out_path)]) + feats = pd.read_csv(out_path) + # The activity is after --as-of: cut before anything is counted, so the + # event type has no rows left anywhere and contributes no columns. + assert "event_count_12m" not in feats.columns + + +def test_cli_features_activity_bad_spec_exits(tmp_path): + data = _make_gift_export(tmp_path, n_donors=2) + with pytest.raises(SystemExit, match="--activity must be TYPE=PATH"): + main(["features", "--source", "raisers_edge", "--data", str(data), + "--activity", "no-equals-sign"]) + + +# --------------------------------------------------------------------------- # +# `train --task upgrade` +# --------------------------------------------------------------------------- # +_UPGRADE_YEARS = ["2020-08-01", "2021-08-01", "2022-08-01", "2023-08-01", "2024-08-01"] +_UPGRADE_ARCHETYPES = { + "flat_high": [900, 900, 900, 900, 900], + "early_riser": [600, 1050, 1200, 1400, 1600], + "late_riser": [300, 500, 700, 1100, 1300], + "flat_low": [400, 400, 400, 400, 400], +} + + +def _make_upgrade_gift_export(tmp_path, name="upgrade_gifts.csv", n_per_group=20): + rows = ["Constituent ID,Gift Date,Gift Amount\n"] + for archetype, amounts in _UPGRADE_ARCHETYPES.items(): + for i in range(n_per_group): + donor_id = f"{archetype}_{i}" + for year, amount in zip(_UPGRADE_YEARS, amounts): + rows.append(f"{donor_id},{year},{amount}.00\n") + path = tmp_path / name + path.write_text("".join(rows)) + return path + + +def test_cli_train_task_upgrade_writes_scored_csv_and_prints_report(tmp_path, capsys): + data = _make_upgrade_gift_export(tmp_path) + out_path = tmp_path / "scored.csv" + main(["train", "--task", "upgrade", "--source", "raisers_edge", + "--data", str(data), "--out", str(out_path), "--random-state", "0"]) + + scored = pd.read_csv(out_path) + assert len(scored) == 40 # flat_high + flat_low, still band-qualifying + assert list(scored.columns) == [ + "donor_id", "fiscal_year", "affinity_score", "rank", "decile", + "top_reasons", "suggested_ask", + ] + out = capsys.readouterr().out + assert "Wrote 40 scored rows" in out + assert "n_training_rows" in out + + +def test_cli_train_task_upgrade_requires_source(tmp_path): + data = _make_upgrade_gift_export(tmp_path) + with pytest.raises(SystemExit, match="requires --source"): + main(["train", "--task", "upgrade", "--data", str(data), + "--out", str(tmp_path / "scored.csv")]) + + +def test_cli_train_task_upgrade_npsp_source(tmp_path): + rows = ["Account ID,Close Date,Amount,Stage\n"] + for archetype, amounts in _UPGRADE_ARCHETYPES.items(): + for i in range(10): + donor_id = f"{archetype}_{i}" + for year, amount in zip(_UPGRADE_YEARS, amounts): + rows.append(f"{donor_id},{year},{amount}.00,Closed Won\n") + data = tmp_path / "opportunities.csv" + data.write_text("".join(rows)) + + out_path = tmp_path / "scored.csv" + main(["train", "--task", "upgrade", "--source", "npsp", "--data", str(data), + "--out", str(out_path), "--random-state", "0"]) + scored = pd.read_csv(out_path) + assert len(scored) == 20 # flat_high + flat_low, 10 each + + +def test_cli_train_task_upgrade_with_donors_csv(tmp_path): + data = _make_upgrade_gift_export(tmp_path) + donors_path = tmp_path / "donors.csv" + donor_ids = [f"{a}_{i}" for a in _UPGRADE_ARCHETYPES for i in range(20)] + lines = ["donor_id,wealth_rating\n"] + [f"{d},A\n" for d in donor_ids] + donors_path.write_text("".join(lines)) + + out_path = tmp_path / "scored.csv" + main(["train", "--task", "upgrade", "--source", "raisers_edge", + "--data", str(data), "--donors", str(donors_path), + "--out", str(out_path), "--random-state", "0"]) + assert len(pd.read_csv(out_path)) == 40 + + +def test_cli_train_task_upgrade_donors_csv_requires_donor_id_column(tmp_path): + data = _make_upgrade_gift_export(tmp_path) + donors_path = tmp_path / "donors.csv" + donors_path.write_text("not_donor_id,wealth_rating\n1,A\n") + with pytest.raises(SystemExit, match="donor_id"): + main(["train", "--task", "upgrade", "--source", "raisers_edge", + "--data", str(data), "--donors", str(donors_path), + "--out", str(tmp_path / "scored.csv")]) + + +def test_cli_train_plain_task_unaffected_by_new_flags(tmp_path): + """--task defaults to 'plain' and every existing train behaviour is + untouched: this is the same call test_cli_train_score_validate makes.""" + data = _make_csv(tmp_path, "train.csv") + model_path = tmp_path / "m.joblib" + main(["train", "--data", str(data), "--target", "is_major_donor", + "--features", FEATURES, "--out", str(model_path)]) + assert model_path.exists() + + +def test_python_and_cli_upgrade_paths_produce_identical_scores(tmp_path): + """The brief's acceptance test: score_upgrade_prospects called directly + on the parsed raw export must match `train --task upgrade`'s CLI output + for the same three CSVs, to floating-point tolerance.""" + from philanthropy.cli import _read_activities, _read_raw_gifts + from philanthropy.models import score_upgrade_prospects + + data = _make_upgrade_gift_export(tmp_path) + activity_path = tmp_path / "activities.csv" + donor_ids = [f"{a}_{i}" for a in _UPGRADE_ARCHETYPES for i in range(20)] + activity_path.write_text( + "contact_id,activity_date\n" + + "\n".join(f"{d},2024-08-01" for d in donor_ids[:10]) + + "\n" + ) + donors_path = tmp_path / "donors.csv" + donors_path.write_text( + "donor_id,wealth_rating\n" + "\n".join(f"{d},A" for d in donor_ids) + "\n" + ) + + gifts = _read_raw_gifts("raisers_edge", str(data)) + activities = _read_activities([f"event={activity_path}"]) + donors = pd.read_csv(donors_path).set_index("donor_id") + direct_scores, direct_report = score_upgrade_prospects( + gifts, activities=activities, donors=donors, random_state=0 + ) + + scored_path = tmp_path / "scored.csv" + main([ + "train", "--task", "upgrade", "--source", "raisers_edge", + "--data", str(data), "--activity", f"event={activity_path}", + "--donors", str(donors_path), "--out", str(scored_path), + "--random-state", "0", + ]) + cli_scores = pd.read_csv(scored_path).set_index("donor_id").sort_index() + direct_sorted = direct_scores.sort_index() + + assert list(cli_scores.index) == list(direct_sorted.index) + pd.testing.assert_series_equal( + cli_scores["affinity_score"].astype(float), + direct_sorted["affinity_score"].astype(float), + check_names=False, check_exact=False, + ) + assert list(cli_scores["rank"]) == list(direct_sorted["rank"]) + assert list(cli_scores["decile"]) == list(direct_sorted["decile"]) diff --git a/tests/test_score_upgrade_prospects.py b/tests/test_score_upgrade_prospects.py new file mode 100644 index 0000000..a8cfe63 --- /dev/null +++ b/tests/test_score_upgrade_prospects.py @@ -0,0 +1,271 @@ +""" +tests/test_score_upgrade_prospects.py +Tests for philanthropy.models.score_upgrade_prospects. + +Fixture: four donor archetypes across five fiscal years (FY2021-FY2025). +"early_riser" crosses `threshold` at FY2022, "late_riser" at FY2024; +"flat_high" and "flat_low" never cross it and are the only two still +band-qualifying "today" (FY2025, the fiscal year containing the default +`as_of`). +""" + +import numpy as np +import pandas as pd +import pytest + +from philanthropy.models import score_upgrade_prospects + +_YEARS = ["2020-08-01", "2021-08-01", "2022-08-01", "2023-08-01", "2024-08-01"] +_ARCHETYPES = { + "flat_high": [900, 900, 900, 900, 900], + "early_riser": [600, 1050, 1200, 1400, 1600], + "late_riser": [300, 500, 700, 1100, 1300], + "flat_low": [400, 400, 400, 400, 400], +} + + +def _archetype_gifts(n_per_group=20): + rows = [ + {"donor_id": f"{name}_{i}", "gift_date": year, "gift_amount": amount} + for name, amounts in _ARCHETYPES.items() + for i in range(n_per_group) + for year, amount in zip(_YEARS, amounts) + ] + return pd.DataFrame(rows) + + +# --------------------------------------------------------------------------- # +# Shape and basic behaviour +# --------------------------------------------------------------------------- # +def test_basic_output_shape_and_columns(): + scores, report = score_upgrade_prospects(_archetype_gifts(), random_state=0) + assert list(scores.columns) == [ + "fiscal_year", "affinity_score", "rank", "decile", "top_reasons", "suggested_ask", + ] + assert scores.index.name == "donor_id" + assert len(scores) == 40 # flat_high + flat_low, 20 each + + +def test_only_currently_band_qualifying_donors_are_scored(): + scores, _ = score_upgrade_prospects(_archetype_gifts(), random_state=0) + names = {donor_id.rsplit("_", 1)[0] for donor_id in scores.index} + assert names == {"flat_high", "flat_low"} + + +def test_affinity_score_bounded_0_to_100(): + scores, _ = score_upgrade_prospects(_archetype_gifts(), random_state=0) + assert (scores["affinity_score"] >= 0).all() + assert (scores["affinity_score"] <= 100).all() + + +def test_sorted_descending_with_matching_rank_and_decile(): + scores, _ = score_upgrade_prospects(_archetype_gifts(), random_state=0) + assert list(scores["affinity_score"]) == sorted(scores["affinity_score"], reverse=True) + assert list(scores["rank"]) == list(range(1, len(scores) + 1)) + assert scores["decile"].min() >= 1 + assert scores["decile"].max() <= 10 + + +def test_top_reasons_are_feature_value_pairs(): + scores, _ = score_upgrade_prospects(_archetype_gifts(), random_state=0) + reasons = scores["top_reasons"].iloc[0] + assert len(reasons) <= 3 + for feature, value in reasons: + assert isinstance(feature, str) + assert np.isscalar(value) + + +def test_suggested_ask_is_nan_not_forced(): + # No ask-amount label exists in this data; see the docstring's "Suggested + # ask" note for why this is left NaN rather than a fabricated number. + scores, _ = score_upgrade_prospects(_archetype_gifts(), random_state=0) + assert scores["suggested_ask"].isna().all() + + +def test_report_keys_present(): + _, report = score_upgrade_prospects(_archetype_gifts(), random_state=0) + for key in ( + "n_training_rows", "n_training_fiscal_years", "low_data_warning", + "low_data_message", "activity_id_match_warnings", "current_fiscal_year", + "n_scored", "validated", "validation_fiscal_year", "n_validation_rows", + "top_n", "model_upgrade_rate_top_n", "baseline_upgrade_rate_top_n", + "overall_upgrade_rate", "lift_over_baseline", + ): + assert key in report + + +# --------------------------------------------------------------------------- # +# Validation and errors +# --------------------------------------------------------------------------- # +def test_band_low_above_high_raises(): + with pytest.raises(ValueError): + score_upgrade_prospects(_archetype_gifts(), band=(999, 100)) + + +def test_no_usable_gift_rows_raises(): + gifts = pd.DataFrame({ + "donor_id": ["1"], "gift_date": ["not-a-date"], "gift_amount": [500], + }) + with pytest.raises(ValueError, match="No usable gift rows"): + score_upgrade_prospects(gifts) + + +def test_as_of_before_every_gift_raises(): + gifts = _archetype_gifts() + with pytest.raises(ValueError, match="No gift rows on or before as_of"): + score_upgrade_prospects(gifts, as_of="2000-01-01") + + +def test_no_historical_rows_raises(): + # A single fiscal year: no T with both T and T+1 resolved. + gifts = pd.DataFrame({ + "donor_id": ["1"], "gift_date": ["2024-08-01"], "gift_amount": [500], + }) + with pytest.raises(ValueError, match="No historical"): + score_upgrade_prospects(gifts) + + +def test_low_data_warning_flagged_under_500_rows(): + _, report = score_upgrade_prospects(_archetype_gifts(), random_state=0) + assert report["n_training_rows"] < 500 + assert report["low_data_warning"] is True + assert report["low_data_message"] is not None + with pytest.warns(UserWarning, match="historical donor-year rows"): + score_upgrade_prospects(_archetype_gifts(), random_state=0) + + +def test_no_donors_currently_in_band_returns_empty_scores(): + # h0..h5: FY2020 (band) -> FY2021 (still below threshold, target=0 for + # the one historical training row at T=FY2020) -> FY2022 (the current + # year: already above threshold, so excluded from the current band). + # "grad" only ever gives in the current year, also above threshold. + rows = [ + {"donor_id": f"h{i}", "gift_date": "2019-08-01", "gift_amount": 300} + for i in range(6) + ] + [ + {"donor_id": f"h{i}", "gift_date": "2020-08-01", "gift_amount": 500} + for i in range(6) + ] + [ + {"donor_id": f"h{i}", "gift_date": "2021-08-01", "gift_amount": 5000} + for i in range(6) + ] + [ + {"donor_id": "grad", "gift_date": "2021-08-01", "gift_amount": 5000}, + ] + gifts = pd.DataFrame(rows) + scores, report = score_upgrade_prospects(gifts) + assert scores.empty + assert list(scores.columns) == [ + "fiscal_year", "affinity_score", "rank", "decile", "top_reasons", "suggested_ask", + ] + assert report["n_scored"] == 0 + + +# --------------------------------------------------------------------------- # +# activities integration: the match-rate warning is captured, not recomputed +# --------------------------------------------------------------------------- # +def test_activity_match_rate_warning_is_captured_in_the_report(): + gifts = _archetype_gifts() + # activities_to_features only checks the match rate when `donors` is + # given (a real donor population, distinct from the gift log itself). + donor_ids = [f"{name}_{i}" for name in _ARCHETYPES for i in range(20)] + donors = pd.DataFrame(index=pd.Index(donor_ids, name="donor_id")) + # Every activity contact_id is a donor id that does not exist in + # `donors`: a 0% match rate, well under the 80% cutoff. + activities = [ + {"contact_id": f"nobody_{i}", "activity_date": "2024-08-01", "activity_type": "event"} + for i in range(5) + ] + _, report = score_upgrade_prospects( + gifts, activities=activities, donors=donors, random_state=0 + ) + assert len(report["activity_id_match_warnings"]) >= 1 + assert "match rate" in report["activity_id_match_warnings"][0] or \ + "distinct" in report["activity_id_match_warnings"][0] + + +def test_activity_match_rate_warning_does_not_escape_uncaptured(): + # The warning is swallowed into the report, not left to surface on the + # caller's own warnings filter (recomputing match-rate logic is exactly + # what this avoids). + gifts = _archetype_gifts() + donor_ids = [f"{name}_{i}" for name in _ARCHETYPES for i in range(20)] + donors = pd.DataFrame(index=pd.Index(donor_ids, name="donor_id")) + activities = [ + {"contact_id": "nobody", "activity_date": "2024-08-01", "activity_type": "event"} + ] + with pytest.warns(UserWarning, match="historical donor-year rows"): + # Only the (unrelated) low-data warning should reach here. + score_upgrade_prospects(gifts, activities=activities, donors=donors, random_state=0) + + +def test_activities_columns_influence_current_row_without_crashing(): + gifts = _archetype_gifts() + activities = [ + {"contact_id": "flat_high_0", "activity_date": "2024-08-01", "activity_type": "event"}, + ] + scores, _ = score_upgrade_prospects(gifts, activities=activities, random_state=0) + assert "flat_high_0" in scores.index + + +# --------------------------------------------------------------------------- # +# donors integration: non-numeric columns are joined but excluded from X +# --------------------------------------------------------------------------- # +def test_donors_numeric_column_used_non_numeric_ignored(): + gifts = _archetype_gifts() + donor_ids = [f"flat_high_{i}" for i in range(20)] + [f"flat_low_{i}" for i in range(20)] + donors = pd.DataFrame( + { + "wealth_rating": ["A"] * 40, + "capacity_estimate": list(range(40)), + }, + index=pd.Index(donor_ids, name="donor_id"), + ) + scores, _ = score_upgrade_prospects(gifts, donors=donors, random_state=0) + assert len(scores) == 40 + + +# --------------------------------------------------------------------------- # +# Leakage: as_of is the hard cutoff for both halves of the function +# --------------------------------------------------------------------------- # +def test_current_row_ignores_a_gift_dated_after_as_of_in_the_same_fiscal_year(): + gifts = _archetype_gifts() + as_of = "2024-08-01" # inside FY2025, the fiscal year every FY2025 gift lands in + + before, report_before = score_upgrade_prospects(gifts, as_of=as_of, random_state=0) + + # One day after as_of, still within FY2025: if this leaked into the + # current row, flat_low_0's FY total would jump from 400 to 5400, well + # past threshold, and it would vanish from the band entirely. + future_row = pd.DataFrame([ + {"donor_id": "flat_low_0", "gift_date": "2024-08-02", "gift_amount": 5000} + ]) + gifts_after = pd.concat([gifts, future_row], ignore_index=True) + after, report_after = score_upgrade_prospects(gifts_after, as_of=as_of, random_state=0) + + assert "flat_low_0" in before.index + assert "flat_low_0" in after.index + pd.testing.assert_frame_equal(before.sort_index(), after.sort_index()) + assert report_before["n_training_rows"] == report_after["n_training_rows"] + + +def test_appending_a_far_future_gift_does_not_change_training_or_scores(): + gifts = _archetype_gifts() + as_of = "2024-08-01" + + before, report_before = score_upgrade_prospects(gifts, as_of=as_of, random_state=0) + + future_row = pd.DataFrame([ + {"donor_id": "flat_low_0", "gift_date": "2030-08-01", "gift_amount": 100_000} + ]) + gifts_after = pd.concat([gifts, future_row], ignore_index=True) + after, report_after = score_upgrade_prospects(gifts_after, as_of=as_of, random_state=0) + + pd.testing.assert_frame_equal(before.sort_index(), after.sort_index()) + assert report_before["n_training_rows"] == report_after["n_training_rows"] + + +def test_as_of_defaults_to_latest_gift_date(): + gifts = _archetype_gifts() + explicit, _ = score_upgrade_prospects(gifts, as_of="2024-08-01", random_state=0) + default, _ = score_upgrade_prospects(gifts, random_state=0) + pd.testing.assert_frame_equal(explicit.sort_index(), default.sort_index()) diff --git a/tests/test_sklearn_compliance.py b/tests/test_sklearn_compliance.py index d4534fd..17e8c9b 100755 --- a/tests/test_sklearn_compliance.py +++ b/tests/test_sklearn_compliance.py @@ -229,7 +229,10 @@ def test_every_public_estimator_is_covered_by_the_battery_or_documented(): for module, names in ((_models, _models.__all__), (_pp, _pp.__all__)): for name in names: cls = getattr(module, name) - if not issubclass(cls, BaseEstimator): + # A plain fit-and-score function (e.g. score_upgrade_prospects) + # is not a class at all, so issubclass() itself would raise; + # skip it the same way a non-estimator class is skipped below. + if not isinstance(cls, type) or not issubclass(cls, BaseEstimator): continue if cls not in covered: uncovered.append(f"{module.__name__}.{name}") From 94217152c25bee6a71329dde3271cadbdb1e94e4 Mon Sep 17 00:00:00 2001 From: Shivam Lalakiya <50960482+shivamlalakiya@users.noreply.github.com> Date: Fri, 25 Sep 2026 14:39:06 -0500 Subject: [PATCH 2/2] Fix flaky index-dtype check in the upgrade CLI/Python parity test test_python_and_cli_upgrade_paths_produce_identical_scores failed on some pandas versions because the two code paths' donor_id index ended up as StringDtype with a different na_value sentinel (nan vs pd.NA) even though the actual index values already matched (checked separately on the line above). assert_series_equal was enforcing that internal dtype flavor by default; pass check_index_type=False so the test asserts what it's actually meant to: identical scores, not identical extension-dtype metadata. --- tests/test_cli.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_cli.py b/tests/test_cli.py index 590d71f..1892d8a 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -482,7 +482,7 @@ def test_python_and_cli_upgrade_paths_produce_identical_scores(tmp_path): pd.testing.assert_series_equal( cli_scores["affinity_score"].astype(float), direct_sorted["affinity_score"].astype(float), - check_names=False, check_exact=False, + check_names=False, check_exact=False, check_index_type=False, ) assert list(cli_scores["rank"]) == list(direct_sorted["rank"]) assert list(cli_scores["decile"]) == list(direct_sorted["decile"])