diff --git a/CHANGELOG.md b/CHANGELOG.md index fd926a0..2c2b53b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -10,6 +10,14 @@ Format: [Keep a Changelog](https://keepachangelog.com/en/1.1.0/) user-supplied export's headers to canonical names and raises one `ValueError` listing every still-missing required column, for callers building a column-mapping UI over an arbitrary CRM export. +- `philanthropy.ingest.activities_to_features(activities, *, as_of, + donors=None)`: aggregates a long, multi-source activity log (event + attendance, volunteer shifts, email clicks, ...) into per-donor, + per-activity-type engagement features (`_count_12m`, + `_count_36m`, `_days_since_last`, `_distinct`, plus + `_hours_12m` / `_amount_12m` when those columns are present), + cut at `as_of` so nothing dated after the cutoff is counted. A new activity + type never needs new model code; it just yields its own columns. ## [0.8.0] - 2026-09-24 diff --git a/docs/reference/index.md b/docs/reference/index.md index 02db716..1635452 100755 --- a/docs/reference/index.md +++ b/docs/reference/index.md @@ -56,6 +56,7 @@ Everything reachable from `philanthropy.__all__` is listed below. A symbol not l | `civicrm_contributions_to_features`, `read_civicrm_contributions` | `ingest` | Tracks CiviCRM's contribution export labels and APIv4 field names, which move with the CRM. | | `raisers_edge_gifts_to_features`, `read_raisers_edge_gifts`, `DEFAULT_EXCLUDED_GIFT_TYPES` | `ingest` | Tracks Raiser's Edge export labels and the RE NXT gift-type vocabulary; the excluded-type default will grow as real exports arrive. | | `map_columns` | `ingest` | The one-error-per-missing-column message shape may still change. | +| `activities_to_features` | `ingest` | The activity-type feature set (`_count_12m`, `_distinct`, ...) may grow as more source types are onboarded. | | `plot_affinity_distribution`, `plot_retention_waterfall` | `visualisation` | Chart composition is presentation, not contract. | | `fetch_kdd98_donors` | `datasets` | Returns the raw upstream columns untyped; may gain as-of date parsing as the real-data leakage replication in #124 lands. | | `make_donor_panel` | `datasets` | The returned dict may gain keys (pledges, appeals, soft credits) as more of the library needs panel-shaped fixtures; existing keys and their columns will not change silently. | diff --git a/philanthropy/ingest/__init__.py b/philanthropy/ingest/__init__.py index 673cb86..6bf462e 100644 --- a/philanthropy/ingest/__init__.py +++ b/philanthropy/ingest/__init__.py @@ -20,8 +20,14 @@ ``map_columns`` renames a user-supplied export's headers to the canonical names a bridge above expects, raising one error listing every column still missing after the rename. + +``activities_to_features`` aggregates a long, multi-source activity log +(event attendance, volunteer shifts, email clicks, ...) into per-donor, +per-type engagement features, generalising the pattern above to an +open-ended set of activity types discovered from the data itself. """ +from ._activities import activities_to_features from ._civicrm import ( civicrm_contributions_to_features, read_civicrm_contributions, @@ -39,6 +45,7 @@ __all__ = [ "DEFAULT_EXCLUDED_GIFT_TYPES", + "activities_to_features", "civicrm_contributions_to_features", "constituent_events_to_features", "map_columns", diff --git a/philanthropy/ingest/_activities.py b/philanthropy/ingest/_activities.py new file mode 100644 index 0000000..7cb09d8 --- /dev/null +++ b/philanthropy/ingest/_activities.py @@ -0,0 +1,236 @@ +""" +philanthropy.ingest._activities +================================ +Turn a long, multi-source "activity" log (event attendance, volunteer +shifts, email clicks, ...) into per-donor engagement features. + +A no-code upload flow lets a user hand over as many activity files as they +have, each tagged with a type (``"event"``, ``"volunteer"``, ...) and mapped +onto a shared shape: a donor id, a date, and optionally an amount, hours, or +name. :func:`activities_to_features` is the single aggregator every one of +those types runs through, so a new activity type never needs new model code: +it just yields its own ``_...`` columns the next time the function sees +it. + +This generalises the aggregation pattern in +:mod:`philanthropy.ingest._constituent_events` (a fixed handful of event +types rolled into named columns) to an open-ended set of types discovered +from the data itself, while keeping that module's conventions: an explicit, +sorted donor id key, explicit output dtypes, and a reference date that never +moves with "now". + +**Leakage.** ``as_of`` is required, not optional: every row dated after it is +dropped before anything is counted, so a gala attended after a cutoff can +never leak into that cutoff's features. The 12- and 36-month windows, the +lifetime distinct count, and the days-since-last figure are all computed +after that cut, from that cut backwards. +""" + +from __future__ import annotations + +import warnings +from typing import Iterable, Mapping, Optional, Union + +import pandas as pd + +__all__ = ["activities_to_features"] + +_REQUIRED = ("contact_id", "activity_date", "activity_type") + + +def activities_to_features( + activities: Union[Iterable[Mapping], pd.DataFrame], + *, + as_of: Union[str, pd.Timestamp], + donors: Optional[Union[pd.DataFrame, pd.Series, Iterable]] = None, +) -> pd.DataFrame: + """Aggregate a long activity log into per-donor, per-type features. + + Parameters + ---------- + activities : iterable of mapping, or DataFrame + Long-format rows, one per activity: ``contact_id``, ``activity_date``, + ``activity_type`` are required; ``amount`` and ``hours`` are used when + present (summed, per type, over the trailing 12 months); ``name`` is + accepted but not aggregated. + as_of : str or datetime-like + The cutoff. Rows with ``activity_date`` after ``as_of`` are dropped + before any feature is computed; every window and recency figure is + measured back from this date, not from "now" or from the batch's own + latest date. + donors : DataFrame, Series, or iterable, optional + The donor population this activity log is being matched against (its + index if a DataFrame, its values otherwise). Used only to report the + match rate: the share of distinct ``contact_id`` values in the + (cutoff) activity log that are also in ``donors``. Does not affect + which rows appear in the output; a donor absent from the activity log + entirely does not get a row here regardless of ``donors``. + + Returns + ------- + features : pandas.DataFrame + One row per donor with at least one activity at or before ``as_of``, + indexed by ``contact_id``. For each activity type present in that + (cutoff) data, four columns: ``_count_12m``, ``_count_36m`` + (counts in the trailing 12 / 36 months before ``as_of``), + ``_days_since_last`` (days from the type's most recent activity + to ``as_of``), and ``_distinct`` (lifetime count of distinct + activity dates of that type, unwindowed). Plus ``_hours_12m`` / + ``_amount_12m`` (trailing-12-month sums) when the input carries + an ``hours`` / ``amount`` column at all. A donor with no rows of a + given type gets 0 in that type's columns; a type with no rows anywhere + in the (cutoff) data contributes no columns at all. Rows are sorted by + ``contact_id`` for determinism. + + Raises + ------ + KeyError + If ``contact_id``, ``activity_date``, or ``activity_type`` is absent. + + Warns + ----- + UserWarning + If ``donors`` is given and fewer than 80% of the distinct + ``contact_id`` values in the (cutoff) activity log are found in it; + the usual cause is an activity export keyed on email while the CRM + export it is being joined against is keyed on an internal id. + + Examples + -------- + >>> rows = [ + ... {"contact_id": "1", "activity_date": "2024-03-01", + ... "activity_type": "volunteer", "hours": 2}, + ... {"contact_id": "1", "activity_date": "2023-01-01", + ... "activity_type": "volunteer", "hours": 3}, + ... {"contact_id": "2", "activity_date": "2024-06-01", + ... "activity_type": "event"}, + ... ] + >>> feats = activities_to_features(rows, as_of="2024-12-31") + >>> int(feats.loc["1", "volunteer_count_12m"]) + 1 + >>> int(feats.loc["1", "volunteer_distinct"]) + 2 + >>> float(feats.loc["1", "volunteer_hours_12m"]) + 2.0 + >>> "event_count_12m" in feats.columns + True + >>> int(feats.loc["2", "event_count_12m"]) + 1 + """ + df = _to_frame(activities) + as_of_ts = pd.Timestamp(as_of) + + if df.empty: + return pd.DataFrame(index=pd.Index([], name="contact_id", dtype="object")) + + missing = [col for col in _REQUIRED if col not in df.columns] + if missing: + raise KeyError( + f"Activity log is missing {missing}. Every activity row needs " + f"'contact_id', 'activity_date' and 'activity_type'; got " + f"{sorted(df.columns)}." + ) + + df = df.copy() + df["_contact_id"] = df["contact_id"].astype("string").str.strip() + df["_ts"] = pd.to_datetime(df["activity_date"], errors="coerce") + df["_type"] = df["activity_type"].astype("string").str.strip() + + # A row we can't place in time or attribute to a donor and a type + # contributes to nothing; drop it rather than let a NaT or blank type + # poison a window or split into its own meaningless column. + df = df[ + df["_contact_id"].notna() + & (df["_contact_id"].str.len() > 0) + & df["_ts"].notna() + & df["_type"].notna() + & (df["_type"].str.len() > 0) + ] + # The cutoff: nothing after as_of is counted anywhere below, including in + # which types and donors even appear in the output. + df = df[df["_ts"] <= as_of_ts] + if df.empty: + return pd.DataFrame(index=pd.Index([], name="contact_id", dtype="object")) + + has_amount = "amount" in df.columns + has_hours = "hours" in df.columns + if has_amount: + df["_amount"] = pd.to_numeric(df["amount"], errors="coerce").fillna(0.0) + if has_hours: + df["_hours"] = pd.to_numeric(df["hours"], errors="coerce").fillna(0.0) + + if donors is not None: + _warn_on_low_match_rate(df["_contact_id"], donors) + + window_12m = as_of_ts - pd.DateOffset(months=12) + window_36m = as_of_ts - pd.DateOffset(months=36) + + donor_ids = pd.Index(sorted(df["_contact_id"].unique()), name="contact_id") + out = pd.DataFrame(index=donor_ids) + + for activity_type in sorted(df["_type"].unique()): + type_rows = df[df["_type"] == activity_type] + grouped = type_rows.groupby("_contact_id") + recent_12m = type_rows[type_rows["_ts"] > window_12m].groupby("_contact_id") + recent_36m = type_rows[type_rows["_ts"] > window_36m].groupby("_contact_id") + + prefix = f"{activity_type}_" + out[prefix + "count_12m"] = recent_12m.size().reindex(donor_ids, fill_value=0) + out[prefix + "count_36m"] = recent_36m.size().reindex(donor_ids, fill_value=0) + last_activity = grouped["_ts"].max().reindex(donor_ids) + days_since_last = (as_of_ts - last_activity).dt.days + out[prefix + "days_since_last"] = days_since_last.fillna(0).astype("int64") + out[prefix + "distinct"] = ( + grouped["_ts"].nunique().reindex(donor_ids, fill_value=0) + ) + + if has_hours: + hours_12m = type_rows[type_rows["_ts"] > window_12m].groupby("_contact_id")["_hours"].sum() + out[prefix + "hours_12m"] = hours_12m.reindex(donor_ids, fill_value=0.0) + if has_amount: + amount_12m = type_rows[type_rows["_ts"] > window_12m].groupby("_contact_id")["_amount"].sum() + out[prefix + "amount_12m"] = amount_12m.reindex(donor_ids, fill_value=0.0) + + for col in out.columns: + if col.endswith(("_count_12m", "_count_36m", "_days_since_last", "_distinct")): + out[col] = out[col].astype("int64") + else: + out[col] = out[col].astype("float64") + + return out + + +# --------------------------------------------------------------------------- # +# Internals +# --------------------------------------------------------------------------- # +def _to_frame(activities: Union[Iterable[Mapping], pd.DataFrame]) -> pd.DataFrame: + if isinstance(activities, pd.DataFrame): + return activities + return pd.DataFrame(list(activities)) + + +def _donor_id_index(donors: Union[pd.DataFrame, pd.Series, Iterable]) -> pd.Index: + if isinstance(donors, pd.DataFrame): + return pd.Index(donors.index.astype("string").str.strip()) + if isinstance(donors, pd.Series): + return pd.Index(donors.astype("string").str.strip()) + return pd.Index(pd.Series(list(donors)).astype("string").str.strip()) + + +def _warn_on_low_match_rate( + activity_contact_ids: pd.Series, donors: Union[pd.DataFrame, pd.Series, Iterable] +) -> None: + donor_ids = set(_donor_id_index(donors)) + distinct_activity_ids = set(activity_contact_ids.unique()) + matched = len(distinct_activity_ids & donor_ids) + match_rate = matched / len(distinct_activity_ids) + if match_rate < 0.8: + warnings.warn( + f"Only {match_rate:.0%} of the {len(distinct_activity_ids)} distinct " + f"contact_id values in this activity log were found in donors " + f"({matched} matched). A low match rate usually means the activity " + f"export is keyed on a different id (e.g. email) than the CRM " + f"export it is being joined against; re-export using the same " + f"donor id in both files.", + stacklevel=2, + ) diff --git a/tests/test_activities_to_features.py b/tests/test_activities_to_features.py new file mode 100644 index 0000000..6fb975e --- /dev/null +++ b/tests/test_activities_to_features.py @@ -0,0 +1,246 @@ +""" +tests/test_activities_to_features.py +Tests for philanthropy.ingest.activities_to_features. + +The point of the module is the same aggregation +philanthropy.ingest._constituent_events.constituent_events_to_features does, +generalised to an open-ended set of activity types: every type present in +the (cutoff) data gets its own count_12m / count_36m / days_since_last / +distinct block, a type with zero rows anywhere gets none of those columns at +all, and a donor with zero rows of a present type gets zeros rather than +nulls. as_of is the leakage boundary most tests below are built around. +""" + +import warnings + +import pandas as pd +import pytest + +from philanthropy.ingest import activities_to_features + + +def _row(contact_id, date, activity_type, **extra): + row = {"contact_id": contact_id, "activity_date": date, "activity_type": activity_type} + row.update(extra) + return row + + +def test_counts_within_and_outside_windows(): + rows = [ + _row("1", "2024-12-01", "event"), # within 12m + _row("1", "2023-06-01", "event"), # within 36m, outside 12m + _row("1", "2020-01-01", "event"), # outside 36m + ] + feats = activities_to_features(rows, as_of="2024-12-31") + + assert feats.loc["1", "event_count_12m"] == 1 + assert feats.loc["1", "event_count_36m"] == 2 + assert feats.loc["1", "event_distinct"] == 3 + + +def test_days_since_last_measured_from_as_of(): + rows = [_row("1", "2024-01-01", "event")] + feats = activities_to_features(rows, as_of="2024-01-11") + + assert feats.loc["1", "event_days_since_last"] == 10 + + +def test_donor_with_no_rows_of_a_type_gets_zero(): + rows = [ + _row("1", "2024-01-01", "event"), + _row("2", "2024-01-01", "volunteer"), + ] + feats = activities_to_features(rows, as_of="2024-12-31") + + assert feats.loc["2", "event_count_12m"] == 0 + assert feats.loc["2", "event_count_36m"] == 0 + assert feats.loc["2", "event_days_since_last"] == 0 + assert feats.loc["2", "event_distinct"] == 0 + + +def test_type_absent_from_whole_input_produces_no_columns(): + rows = [_row("1", "2024-01-01", "event")] + feats = activities_to_features(rows, as_of="2024-12-31") + + assert not any(col.startswith("volunteer_") for col in feats.columns) + + +def test_amount_column_present_yields_amount_12m(): + rows = [ + _row("1", "2024-06-01", "gift", amount=100), + _row("1", "2020-01-01", "gift", amount=999), # outside 12m window + ] + feats = activities_to_features(rows, as_of="2024-12-31") + + assert feats.loc["1", "gift_amount_12m"] == 100.0 + + +def test_hours_column_present_yields_hours_12m(): + rows = [_row("1", "2024-06-01", "volunteer", hours=3.5)] + feats = activities_to_features(rows, as_of="2024-12-31") + + assert feats.loc["1", "volunteer_hours_12m"] == 3.5 + + +def test_no_amount_or_hours_column_yields_no_such_columns(): + rows = [_row("1", "2024-06-01", "event")] + feats = activities_to_features(rows, as_of="2024-12-31") + + assert "event_amount_12m" not in feats.columns + assert "event_hours_12m" not in feats.columns + + +def test_multiple_activity_types_each_get_their_own_columns(): + rows = [ + _row("1", "2024-01-01", "event"), + _row("1", "2024-02-01", "volunteer"), + ] + feats = activities_to_features(rows, as_of="2024-12-31") + + for prefix in ("event", "volunteer"): + for suffix in ("count_12m", "count_36m", "days_since_last", "distinct"): + assert f"{prefix}_{suffix}" in feats.columns + + +# --------------------------------------------------------------------------- # +# Leakage +# --------------------------------------------------------------------------- # +def test_rows_after_as_of_are_ignored(): + rows = [_row("1", "2024-06-01", "event")] + before = activities_to_features(rows, as_of="2024-12-31") + + rows_with_future = rows + [_row("1", "2025-06-01", "event")] + after = activities_to_features(rows_with_future, as_of="2024-12-31") + + pd.testing.assert_frame_equal(before, after) + + +def test_future_row_of_a_new_type_does_not_add_columns(): + rows = [_row("1", "2024-06-01", "event")] + feats = activities_to_features( + rows + [_row("1", "2025-06-01", "volunteer")], as_of="2024-12-31" + ) + + assert not any(col.startswith("volunteer_") for col in feats.columns) + + +def test_idempotent_on_repeat(): + rows = [ + _row("1", "2024-06-01", "event"), + _row("2", "2023-01-01", "volunteer", hours=2), + ] + first = activities_to_features(rows, as_of="2024-12-31") + second = activities_to_features(rows, as_of="2024-12-31") + + pd.testing.assert_frame_equal(first, second) + + +# --------------------------------------------------------------------------- # +# Match rate against donors +# --------------------------------------------------------------------------- # +def test_low_match_rate_against_donors_warns(): + rows = [_row(str(i), "2024-01-01", "event") for i in range(10)] + donors = pd.DataFrame(index=pd.Index([str(i) for i in range(2)], name="contact_id")) + + with pytest.warns(UserWarning, match="match"): + activities_to_features(rows, as_of="2024-12-31", donors=donors) + + +def test_high_match_rate_against_donors_does_not_warn(): + rows = [_row(str(i), "2024-01-01", "event") for i in range(10)] + donors = pd.DataFrame(index=pd.Index([str(i) for i in range(10)], name="contact_id")) + + with warnings.catch_warnings(): + warnings.simplefilter("error") + activities_to_features(rows, as_of="2024-12-31", donors=donors) + + +def test_donors_as_plain_iterable_of_ids(): + rows = [_row(str(i), "2024-01-01", "event") for i in range(10)] + donors = [str(i) for i in range(2)] + + with pytest.warns(UserWarning, match="match"): + activities_to_features(rows, as_of="2024-12-31", donors=donors) + + +def test_donors_as_series_of_ids(): + rows = [_row(str(i), "2024-01-01", "event") for i in range(10)] + donors = pd.Series([str(i) for i in range(2)]) + + with pytest.warns(UserWarning, match="match"): + activities_to_features(rows, as_of="2024-12-31", donors=donors) + + +def test_donors_does_not_add_rows_for_donors_absent_from_activities(): + rows = [_row("1", "2024-01-01", "event")] + donors = pd.DataFrame(index=pd.Index(["1", "2", "3"], name="contact_id")) + + feats = activities_to_features(rows, as_of="2024-12-31", donors=donors) + + assert list(feats.index) == ["1"] + + +# --------------------------------------------------------------------------- # +# Validation and empty input +# --------------------------------------------------------------------------- # +def test_missing_required_column_raises(): + rows = [{"contact_id": "1", "activity_date": "2024-01-01"}] + + with pytest.raises(KeyError, match="activity_type"): + activities_to_features(rows, as_of="2024-12-31") + + +def test_empty_input_returns_empty_frame_with_no_columns(): + feats = activities_to_features([], as_of="2024-12-31") + + assert feats.empty + assert feats.index.name == "contact_id" + + +def test_every_row_filtered_out_by_as_of_returns_empty_frame(): + rows = [_row("1", "2030-01-01", "event")] + + feats = activities_to_features(rows, as_of="2024-12-31") + + assert feats.empty + + +def test_blank_activity_type_is_dropped(): + rows = [ + _row("1", "2024-01-01", "event"), + _row("1", "2024-01-02", ""), + ] + feats = activities_to_features(rows, as_of="2024-12-31") + + assert list(feats.columns) == [ + "event_count_12m", + "event_count_36m", + "event_days_since_last", + "event_distinct", + ] + + +def test_unparseable_date_row_is_dropped(): + rows = [ + _row("1", "2024-01-01", "event"), + _row("1", "not-a-date", "event"), + ] + feats = activities_to_features(rows, as_of="2024-12-31") + + assert feats.loc["1", "event_distinct"] == 1 + + +def test_dataframe_input_accepted(): + df = pd.DataFrame( + [_row("1", "2024-01-01", "event"), _row("2", "2024-02-01", "event")] + ) + feats = activities_to_features(df, as_of="2024-12-31") + + assert list(feats.index) == ["1", "2"] + + +def test_output_sorted_by_contact_id(): + rows = [_row("9", "2024-01-01", "event"), _row("1", "2024-01-01", "event")] + feats = activities_to_features(rows, as_of="2024-12-31") + + assert list(feats.index) == ["1", "9"]