diff --git a/scripts/ingest-deadlines/README.md b/scripts/ingest-deadlines/README.md index e0eab0d..a37dfa7 100644 --- a/scripts/ingest-deadlines/README.md +++ b/scripts/ingest-deadlines/README.md @@ -21,6 +21,18 @@ node ingest.mjs --apply # write Needs `pdftotext` (`brew install poppler`) on PATH. Reads Supabase credentials from `web/.env.local`. Re-runnable: it only ever fills blanks. +## What it brings in + +Per round: Early Decision, Early Action and Regular Decision dates. + +Per school: the US application fee and whether a fee waiver is accepted. Worth +having because the answer is so often "nothing" — of the 1,128 schools the grid +lists a fee for, **611 charge $0**. The median is $0, the 90th percentile $70, +the highest $150. Waivers are accepted at 540, US-only at 90, refused at 29. + +Needs the `application_fee_cents` / `fee_waiver_policy` columns, added in +`20261001000000_college_application_fees.sql`. Apply that before running. + ## Writing: use emit-sql `public.colleges` is **RLS read-only** in normal operation, so the anon key @@ -36,6 +48,10 @@ lands whole or not at all, and every column is written through `coalesce(new, existing)` — a curated value survives even if the selection logic were wrong. +Deadlines only ever fill a blank. Fees are refreshed on every run: nobody +curates those by hand, the grid is their only source, and a school that drops +its fee should stop being shown as charging one. + This is how the September 2026 run was applied: 935 rows, taking real regular-decision dates from 46 colleges to 977. diff --git a/scripts/ingest-deadlines/emit-sql.mjs b/scripts/ingest-deadlines/emit-sql.mjs index e57cad6..0f356a7 100644 --- a/scripts/ingest-deadlines/emit-sql.mjs +++ b/scripts/ingest-deadlines/emit-sql.mjs @@ -24,7 +24,7 @@ const rows = parseGrid(readFileSync(txt, 'utf8')) const colleges = [] for (let from = 0; ; from += 1000) { const { data, error } = await supabase.from('colleges') - .select('scorecard_id, name, early_decision, early_action, regular_decision') + .select('scorecard_id, name, early_decision, early_action, regular_decision, application_fee_cents, fee_waiver_policy') .order('scorecard_id').range(from, from + 999) if (error) throw new Error(error.message) colleges.push(...data); if (data.length < 1000) break @@ -55,21 +55,33 @@ for (const row of single) { if (!value || college[column]) continue // never overwrite patch[column] = value } + // Fees are refreshed rather than only filled: unlike a hand-checked + // deadline, nobody curates these by hand, the grid is their only source, + // and a school that drops its fee should stop being shown as charging one. + if (row.fee != null) patch.application_fee_cents = row.fee * 100 + if (row.waiver) patch.fee_waiver_policy = row.waiver if (Object.keys(patch).length) updates.push({ id: college.scorecard_id, patch }) } const q = (v) => (v == null ? 'null' : `'${String(v).replace(/'/g, "''")}'`) +const n = (v) => (v == null ? 'null' : String(v)) const values = updates.map((u) => - `(${u.id},${q(u.patch.early_decision)},${q(u.patch.early_action)},${q(u.patch.regular_decision)})`).join(',\n ') + `(${u.id},${q(u.patch.early_decision)},${q(u.patch.early_action)},${q(u.patch.regular_decision)},` + + `${n(u.patch.application_fee_cents)},${q(u.patch.fee_waiver_policy)})`).join(',\n ') // coalesce so a column this run has nothing for keeps whatever is there. +// fees_verified_at only moves when a fee actually came through, so it stays an +// honest record of when the figure was last confirmed. const sql = `update public.colleges c set - early_decision = coalesce(v.ed, c.early_decision), - early_action = coalesce(v.ea, c.early_action), - regular_decision = coalesce(v.rd, c.regular_decision), - verified_at = now() + early_decision = coalesce(v.ed, c.early_decision), + early_action = coalesce(v.ea, c.early_action), + regular_decision = coalesce(v.rd, c.regular_decision), + application_fee_cents = coalesce(v.fee, c.application_fee_cents), + fee_waiver_policy = coalesce(v.waiver, c.fee_waiver_policy), + fees_verified_at = case when v.fee is not null then now() else c.fees_verified_at end, + verified_at = now() from (values ${values} -) as v(id, ed, ea, rd) +) as v(id, ed, ea, rd, fee, waiver) where c.scorecard_id = v.id;` writeFileSync(process.argv[2] ?? 'deadlines.sql', sql) console.log(`${updates.length} rows, ${sql.length} bytes -> ${process.argv[2] ?? 'deadlines.sql'}`) diff --git a/scripts/ingest-deadlines/parse.mjs b/scripts/ingest-deadlines/parse.mjs index 61e6bf1..ca2aa60 100644 --- a/scripts/ingest-deadlines/parse.mjs +++ b/scripts/ingest-deadlines/parse.mjs @@ -36,7 +36,56 @@ export function columnsFrom(headerLine) { // 'type' heads the school-type column. Names live to its left, so it tells // us whether a line's first field is a name at all. const type = /\btype\b/.exec(headerLine) - return type ? { ...cols, typeAt: type.index } : null + if (!type) return null + // The fee columns sit to the right of the rounds: what a US applicant pays, + // what an international one pays, and whether a waiver is taken. + const us = /\bUS\b/.exec(headerLine) + const intl = /Int'l/.exec(headerLine) + const waiver = /\bfee waiver\b/.exec(headerLine) + return { + ...cols, + typeAt: type.index, + feeAt: us?.index ?? null, + feeEnd: intl?.index ?? null, + waiverAt: waiver?.index ?? null, + } +} + +/** + * The grid's three fee-waiver answers, as the column stores them. + * + * Longest label first, and the order matters: "Not Accepted" contains + * "Accepted", so testing for the short one first turned every school that + * refuses waivers into one that takes them. + */ +const WAIVER_POLICY = [ + ['Not Accepted', 'not_accepted'], + ['U.S. only', 'us_only'], + ['Accepted', 'accepted'], +] + +/** + * What a school charges a US applicant, and whether it takes a waiver. + * + * Returns `fee` in whole dollars — 0 is a real answer and the common one, so + * it must stay distinct from "the column was blank", which is undefined. + */ +export function feesOn(line, cols) { + const out = {} + if (cols.feeAt != null && cols.feeEnd != null) { + // The amount is right-aligned under its header, so read the span up to the + // next column rather than from a fixed offset. + const text = line.slice(Math.max(0, cols.feeAt - 6), cols.feeEnd - 1) + const m = /\$\s?([\d,]+)/.exec(text) + if (m) out.fee = Number(m[1].replace(/,/g, '')) + } + if (cols.waiverAt != null) { + const text = line.slice(Math.max(0, cols.waiverAt - 6), cols.waiverAt + 18) + for (const [label, value] of WAIVER_POLICY) { + if (text.includes(label)) { out.waiver = value; break } + } + } + return out } /** @@ -149,6 +198,7 @@ export function parseGrid(text) { if (round && !row[round]) row[round] = value } if (rollingAt >= 0 && roundFor(rollingAt, cols) === 'rd' && !row.rd) row.rd = 'Rolling' + Object.assign(row, feesOn(line, cols)) if (ROUNDS.some((k) => row[k])) out.push(row) } return out diff --git a/scripts/ingest-deadlines/parse.test.mjs b/scripts/ingest-deadlines/parse.test.mjs index cf3b24e..c6a4239 100644 --- a/scripts/ingest-deadlines/parse.test.mjs +++ b/scripts/ingest-deadlines/parse.test.mjs @@ -83,3 +83,35 @@ test('normaliseName survives the abbreviations the two sources disagree on', () assert.equal(normaliseName('Saint Johns University'), normaliseName('St. Johns University')) assert.notEqual(normaliseName('Miami University'), normaliseName('University of Miami')) }) + +const HDR_FEES = + " member type \u00b9 ED EDII EA EAII REA Rolling US Int'l fee waiver essay" + +test('reads what a school charges, and whether it takes a waiver', () => { + const rows = grid( + HDR_FEES, + " Paid College Coed Rolling $85 $85 Accepted Y", + ) + assert.equal(find(rows, 'Paid College').fee, 85) + assert.equal(find(rows, 'Paid College').waiver, 'accepted') +}) + +test('free is a figure, not a missing one', () => { + // 611 of the grid's 1,128 schools charge nothing. Zero has to survive as + // zero: "free to apply" and "we have no idea" are different answers. + const rows = grid( + HDR_FEES, + " Free College Coed Rolling $0 $0 Accepted Y", + ) + assert.equal(find(rows, 'Free College').fee, 0) +}) + +test('a school that refuses waivers is not read as accepting them', () => { + // "Not Accepted" contains "Accepted", and testing the short label first + // turned all 29 refusing schools into accepting ones. + const rows = grid( + HDR_FEES, + " Strict College Coed Rolling $50 $50 Not Accepted Y", + ) + assert.equal(find(rows, 'Strict College').waiver, 'not_accepted') +}) diff --git a/supabase/migrations/20261001000000_college_application_fees.sql b/supabase/migrations/20261001000000_college_application_fees.sql new file mode 100644 index 0000000..e07e485 --- /dev/null +++ b/supabase/migrations/20261001000000_college_application_fees.sql @@ -0,0 +1,42 @@ +-- What a school charges to apply, and whether it takes a waiver. +-- +-- Both come from the Common App requirements grid, the same PDF the deadline +-- ingest already parses — it carries `US`, `Int'l` and `fee waiver` columns +-- beside the round dates. See scripts/ingest-deadlines. +-- +-- This is worth storing because the answer is so often "nothing": of the 1,128 +-- schools the grid lists a fee for, 611 charge $0. A student looking at a list +-- of ten schools and bracing for $700 of fees is usually wrong, and has no way +-- to find out short of opening ten admissions pages. +-- +-- Cents, not dollars, matching every other money column here (avg_net_price_cents, +-- cost_of_attendance_cents). Null means we hold no figure; 0 means free, and the +-- two must stay distinguishable — "we don't know" and "it costs nothing" are +-- very different things to tell someone. + +alter table public.colleges + add column if not exists application_fee_cents int, + add column if not exists fee_waiver_policy text, + add column if not exists fees_verified_at timestamptz; + +comment on column public.colleges.application_fee_cents is + 'US application fee in cents. 0 means free to apply; null means no figure on file.'; +comment on column public.colleges.fee_waiver_policy is + 'From the grid''s fee-waiver column: accepted | us_only | not_accepted. Null means the grid left it blank.'; +comment on column public.colleges.fees_verified_at is + 'When the fee columns were last refreshed from the Common App grid.'; + +-- The grid publishes exactly these three answers; anything else is a parse bug +-- rather than a new policy, and should fail loudly instead of being stored. +alter table public.colleges + drop constraint if exists colleges_fee_waiver_policy_check; +alter table public.colleges + add constraint colleges_fee_waiver_policy_check + check (fee_waiver_policy is null or fee_waiver_policy in ('accepted', 'us_only', 'not_accepted')); + +-- A negative fee is meaningless and would read as a discount. +alter table public.colleges + drop constraint if exists colleges_application_fee_nonneg; +alter table public.colleges + add constraint colleges_application_fee_nonneg + check (application_fee_cents is null or application_fee_cents >= 0); diff --git a/web/src/components/ApplicationTrackingModule.tsx b/web/src/components/ApplicationTrackingModule.tsx index ea0fa7d..fa3bc99 100644 --- a/web/src/components/ApplicationTrackingModule.tsx +++ b/web/src/components/ApplicationTrackingModule.tsx @@ -158,7 +158,7 @@ export default function ApplicationTrackingModule({ open, onClose, onEditIncome, const id = collegeAppId(college) if (current.some(a => a.collegeId === id)) return const [mapX, mapY] = projectToMap(college.longitude, college.latitude, college.state) ?? [null, null] - const entry: ApplicationEntry = { collegeId: id, category, deadlineType: 'RD', status: 'not-started', name: college.name, subtitle: collegeSubtitle(college), source: 'scorecard', state: college.state, city: college.city, mapX, mapY, website: domainOf(college.url), ownership: college.ownership, institutionType: college.institution_type } + const entry: ApplicationEntry = { collegeId: id, category, deadlineType: 'RD', status: 'not-started', name: college.name, subtitle: collegeSubtitle(college), source: 'scorecard', state: college.state, city: college.city, mapX, mapY, website: domainOf(college.url), ownership: college.ownership, institutionType: college.institution_type, applicationFeeCents: college.application_fee_cents, feeWaiverPolicy: college.fee_waiver_policy } // Only seed tasks when a shared one is already done or dated elsewhere; otherwise // leave them unset so the default checklist keeps tracking the entry. const tasks = initialTasksFor(entry, current) diff --git a/web/src/components/CollegeDiscoverTab.test.tsx b/web/src/components/CollegeDiscoverTab.test.tsx index 04a8da9..724f2e5 100644 --- a/web/src/components/CollegeDiscoverTab.test.tsx +++ b/web/src/components/CollegeDiscoverTab.test.tsx @@ -23,6 +23,7 @@ const H = vi.hoisted(() => { legacy_slug: null, emoji: null, early_action: null, early_decision: null, regular_decision: null, fafsa_priority: null, css_profile: null, aid_notification: null, meets_full_need: null, no_loan_policy: null, curated_at: null, + application_fee_cents: null, fee_waiver_policy: null, fees_verified_at: null, ...over, }) const SAMPLE: College[] = [ diff --git a/web/src/components/SchoolApplicationPage.tsx b/web/src/components/SchoolApplicationPage.tsx index 4d7cad5..b74a7fb 100644 --- a/web/src/components/SchoolApplicationPage.tsx +++ b/web/src/components/SchoolApplicationPage.tsx @@ -18,7 +18,7 @@ import { useEffect, useRef, useState } from 'react' import { C, MODULE_COLORS } from '../lib/designTokens' import { Bar, CollegeLogo, SecLabel } from './moduleUI' import JourneyStepper from './JourneyStepper' -import { TASK_PHASES, tasksForEntry, isSharedTask, suggestedWaiverDue, WAIVER_LEAD_DAYS } from '../data/applicationTasks' +import { TASK_PHASES, tasksForEntry, isSharedTask, suggestedWaiverDue, applicationFeeFact, WAIVER_LEAD_DAYS } from '../data/applicationTasks' import { DEADLINE_TYPE_LABEL, DEADLINE_TYPE_MEANING, daysLabel, urgencyColor, type DeadlineEvent } from '../data/applicationDeadlines' import NetPriceFact from './NetPriceFact' import { toIsoDay } from '../lib/personalDeadlines' @@ -193,6 +193,7 @@ export default function SchoolApplicationPage({ // A month before this school's deadline, for the fee-waiver task. Null when // we hold no deadline to count back from. const waiverSuggestion = deadline ? toIsoDay(suggestedWaiverDue(deadline.date)) : undefined + const fee = applicationFeeFact(app) const setDue = (task: AppTask, due: string | undefined) => { if (isSharedTask(task)) onSetSharedDue(task.id, due) @@ -263,7 +264,7 @@ export default function SchoolApplicationPage({ {/* facts */} -
+
Deadline {deadline && preSubmission ? ( @@ -291,6 +292,13 @@ export default function SchoolApplicationPage({
+
+ To apply +
+ {fee.amount} +
+
{fee.note}
+
Tasks
{done} of {tasks.length} done
diff --git a/web/src/data/applicationTasks.test.ts b/web/src/data/applicationTasks.test.ts index 4de25ac..2f20471 100644 --- a/web/src/data/applicationTasks.test.ts +++ b/web/src/data/applicationTasks.test.ts @@ -1,5 +1,5 @@ import { describe, it, expect } from 'vitest' -import { defaultTasksFor, deriveTaskEvents, initialTasksFor, setSharedTask, sharedTaskSummary, tasksForEntry, tasksForRound, updateSharedTask, suggestedWaiverDue, WAIVER_LEAD_DAYS } from './applicationTasks' +import { defaultTasksFor, deriveTaskEvents, initialTasksFor, setSharedTask, sharedTaskSummary, tasksForEntry, tasksForRound, updateSharedTask, suggestedWaiverDue, applicationFeeFact, WAIVER_LEAD_DAYS } from './applicationTasks' import { withKnownCategory, type ApplicationEntry } from './applicationsChecklist' const app = (collegeId: string, fields: Partial = {}): ApplicationEntry => ({ @@ -200,3 +200,39 @@ describe('fee waivers need lead time', () => { expect(tasksForEntry(entry)).toBe(entry.tasks) }) }) + +describe('application fees', () => { + const idsOf = (a: ApplicationEntry) => defaultTasksFor(a).map((t) => t.id) + + it('says free rather than $0, and never invents either answer', () => { + expect(applicationFeeFact(app('a', { applicationFeeCents: 0 })).amount).toBe('Free to apply') + expect(applicationFeeFact(app('a', { applicationFeeCents: 8500 })).amount).toBe('$85') + // No figure is its own answer. An invented $0 is a promise; an invented + // fee is a reason not to apply. + expect(applicationFeeFact(app('a')).amount).toBe('Fee not on file') + }) + + it('says whether a waiver is taken, when the grid told us', () => { + const note = (p?: string) => applicationFeeFact(app('a', { applicationFeeCents: 8500, feeWaiverPolicy: p })).note + expect(note('accepted')).toBe('Fee waivers accepted') + expect(note('us_only')).toBe('Fee waivers accepted') + expect(note('not_accepted')).toBe('No fee waivers here') + expect(note(undefined)).toBe('Ask about a waiver') + }) + + it('drops the waiver task where there is no fee to waive', () => { + expect(idsOf(app('a', { applicationFeeCents: 0 }))).not.toContain('waiver') + expect(idsOf(app('a', { applicationFeeCents: 8500 }))).toContain('waiver') + }) + + it('falls back to the school type when no fee is on file', () => { + // Most of the 6,273 colleges have no grid row at all. + expect(idsOf(app('a'))).toContain('waiver') + expect(idsOf(app('c', { institutionType: '2yr' }))).not.toContain('waiver') + }) + + it('keeps the waiver on a free two-year school that actually charges', () => { + // The real figure beats the guess, in both directions. + expect(idsOf(app('c', { institutionType: '2yr', applicationFeeCents: 4000 }))).toContain('waiver') + }) +}) diff --git a/web/src/data/applicationTasks.ts b/web/src/data/applicationTasks.ts index 8380187..e0410c5 100644 --- a/web/src/data/applicationTasks.ts +++ b/web/src/data/applicationTasks.ts @@ -27,6 +27,20 @@ const isPrivate = (app: ApplicationEntry) => const isEarlyBinding = (app: ApplicationEntry) => app.deadlineType === 'ED' || app.deadlineType === 'REA' +/** + * Whether there is a fee here to waive. + * + * 611 of the 1,128 schools in the Common App grid charge nothing to apply, so + * a waiver step on every school is noise on most of them. A known $0 settles + * it outright. Where no figure is on file we fall back to the school's type: + * community colleges and trade schools almost never charge, four-year schools + * usually do. + */ +const chargesAFee = (app: ApplicationEntry): boolean => { + if (app.applicationFeeCents != null) return app.applicationFeeCents > 0 + return !isTwoYearOrTrade(app) +} + const t = (id: string, label: string, phase: TaskPhase): AppTask => ({ id, label, done: false, phase }) /** The default checklist for a school, tailored to its type and deadline plan. */ @@ -48,11 +62,7 @@ export function defaultTasksFor(app: ApplicationEntry): AppTask[] { // Asking for the waiver is not a submit-day job. The request goes through // the counselor, who is doing the same for a whole cohort in October and // November, so it belongs with the work done ahead of the deadline. - // - // Not for community colleges and trade schools: most charge nothing to - // apply, and a step about waiving a fee that does not exist is noise on the - // one list that should be shortest. - if (fourYear) tasks.push(t('waiver', 'Ask your counselor about a fee waiver', 'before')) + if (chargesAFee(app)) tasks.push(t('waiver', 'Ask your counselor about a fee waiver', 'before')) // ── Submit ── tasks.push(t('fee', 'Pay the application fee, if one is owed', 'submit')) @@ -143,6 +153,30 @@ export const SHARED_TASK_IDS: ReadonlySet = new Set(['recs', 'transcript */ export const WAIVER_LEAD_DAYS = 30 +/** + * What this school costs to apply to, as a student should read it. + * + * "Free to apply" is the single most useful thing on this card for a student + * bracing for a few hundred dollars of fees: more than half the schools in + * the Common App grid charge nothing at all. Where no figure is on file we say + * so rather than guess — an invented $0 is a promise, and an invented fee is a + * reason not to apply. + */ +export function applicationFeeFact(app: ApplicationEntry): { amount: string; note: string; free: boolean } { + const cents = app.applicationFeeCents + if (cents == null) { + return { amount: 'Fee not on file', note: 'Check the school’s admissions page', free: false } + } + if (cents === 0) return { amount: 'Free to apply', note: 'No application fee', free: true } + const dollars = Math.round(cents / 100) + const note = app.feeWaiverPolicy === 'accepted' || app.feeWaiverPolicy === 'us_only' + ? 'Fee waivers accepted' + : app.feeWaiverPolicy === 'not_accepted' + ? 'No fee waivers here' + : 'Ask about a waiver' + return { amount: `$${dollars}`, note, free: false } +} + /** The date to aim for, given when the application is actually due. */ export function suggestedWaiverDue(deadline: Date): Date { const d = new Date(deadline) diff --git a/web/src/data/applicationsChecklist.ts b/web/src/data/applicationsChecklist.ts index ee54abb..ec2da95 100644 --- a/web/src/data/applicationsChecklist.ts +++ b/web/src/data/applicationsChecklist.ts @@ -145,6 +145,11 @@ export interface ApplicationEntry { ownership?: string | null /** '4yr' | '2yr' | 'trade' — drives which default application tasks apply. */ institutionType?: string | null + /** What this school charges to apply, in cents, and whether it takes a + * waiver. Snapshotted at add-time like the rest: 0 is a real and common + * answer, so it must stay distinct from undefined ("no figure on file"). */ + applicationFeeCents?: number | null + feeWaiverPolicy?: string | null /** Per-school application to-do list (seeded from a smart default, then edited). */ tasks?: AppTask[] } diff --git a/web/src/lib/collegeMatch.test.ts b/web/src/lib/collegeMatch.test.ts index e21c62f..cc865c5 100644 --- a/web/src/lib/collegeMatch.test.ts +++ b/web/src/lib/collegeMatch.test.ts @@ -77,6 +77,9 @@ function mk(over: Partial = {}): College { meets_full_need: null, no_loan_policy: null, curated_at: null, + application_fee_cents: null, + fee_waiver_policy: null, + fees_verified_at: null, ...over, } } diff --git a/web/src/lib/collegeSearch.ts b/web/src/lib/collegeSearch.ts index e833fc4..c540b6e 100644 --- a/web/src/lib/collegeSearch.ts +++ b/web/src/lib/collegeSearch.ts @@ -15,7 +15,8 @@ export const SEARCH_COLS = 'id,scorecard_id,name,slug,institution_type,city,state,region,ownership,locale,size,latitude,longitude,' + 'admit_rate,sat_reading_25,sat_reading_75,sat_math_25,sat_math_75,act_25,act_75,' + 'avg_net_price_cents,net_price_by_income,cost_of_attendance_cents,programs,grad_rate,' + - 'transfer_rate,median_earnings_10yr_cents,pell_pct,npc_url,url' + 'transfer_rate,median_earnings_10yr_cents,pell_pct,npc_url,url,' + + 'application_fee_cents,fee_waiver_policy' /** * These reject rather than returning [] on failure. An empty array is a real diff --git a/web/src/types/database.ts b/web/src/types/database.ts index d156a28..5bb43ce 100644 --- a/web/src/types/database.ts +++ b/web/src/types/database.ts @@ -68,6 +68,9 @@ export type Database = { act_75: number | null admit_rate: number | null aid_notification: string | null + application_fee_cents: number | null + fee_waiver_policy: string | null + fees_verified_at: string | null avg_net_price_cents: number | null city: string | null cost_of_attendance_cents: number | null @@ -120,6 +123,9 @@ export type Database = { act_75?: number | null admit_rate?: number | null aid_notification?: string | null + application_fee_cents?: number | null + fee_waiver_policy?: string | null + fees_verified_at?: string | null avg_net_price_cents?: number | null city?: string | null cost_of_attendance_cents?: number | null @@ -172,6 +178,9 @@ export type Database = { act_75?: number | null admit_rate?: number | null aid_notification?: string | null + application_fee_cents?: number | null + fee_waiver_policy?: string | null + fees_verified_at?: string | null avg_net_price_cents?: number | null city?: string | null cost_of_attendance_cents?: number | null