diff --git a/package.json b/package.json index 0f50c328..9a12613f 100644 --- a/package.json +++ b/package.json @@ -13,6 +13,7 @@ "seed:bills-admin": "tsx scripts/seed-bills-admin.ts", "eval:bills": "tsx --env-file-if-exists=.env.local --env-file-if-exists=.env src/app/bills/evals/run.ts", "test:feeds": "tsx --test src/lib/feeds.test.ts", + "test:bills": "NODE_OPTIONS=--conditions=import tsx --test src/app/bills/utils/merge-bill.test.ts src/app/bills/services/refresh-decision.test.ts src/app/bills/prompt/analysis-schema.test.ts", "test:polls": "NODE_OPTIONS=--conditions=import tsx --test src/lib/charts/inline-chart.test.ts src/lib/polls/downloads.test.ts src/lib/polls/crosstabs.test.ts" }, "dependencies": { diff --git a/src/app/bills/README.md b/src/app/bills/README.md index a137603c..d71f3a1a 100644 --- a/src/app/bills/README.md +++ b/src/app/bills/README.md @@ -22,8 +22,12 @@ Every analyzed bill produces a `BillAnalysis` (see `services/billApi.ts`) with: - **`final_judgment`** — `yes | no | abstain`. - **`question_period_questions`** — exactly 3 critical MP-style questions (no "Mr./Madam Speaker" prefix). -- **`rationale`**, `short_title`, `needs_more_info`, `missing_details`, and - `steel_man` (an editorially-maintained field, not LLM-generated — see below). +- **`steel_man`** — the strongest good-faith case *against* the judgment just + given, rendered on the bill page under "The other side". +- **`isSocialIssue`** — whether the bill is primarily a social/rights/identity + issue. When true the judgment is forced to `abstain`: Builder MP weighs bills + on economic tenets and takes no position on social questions. +- **`rationale`**, `short_title`, `needs_more_info`, `missing_details`. ### Build Canada's tenets @@ -43,42 +47,72 @@ barriers → align; protectionism, new red tape, or large redistributive spendin → conflict) so borderline judgments stay consistent. These tenets and signals track Build Canada's documented positions at [buildcanada.com](https://www.buildcanada.com). -### The two LLM touchpoints +### The LLM touchpoint -- **`summarizeBillText`** (`services/billApi.ts`) — the main analysis pass - (`gpt-5`, high reasoning effort). Produces the full `BillAnalysis` above. -- **`socialIssueGrader`** (`services/social-issue-grader.ts`) — a small binary - classifier: is the bill *primarily* a social/rights/identity/culture issue? - When yes, the app treats the bill as out of the economic-tenet scope and - the judgment is `abstain`. +**`summarizeBillText`** (`services/billApi.ts`) is the only LLM call: one +`gpt-5` pass at high reasoning effort producing the whole `BillAnalysis`, +including `is_social_issue`. It uses **OpenAI Structured Outputs** against +`prompt/analysis-schema.ts`, so the shape is guaranteed — no markdown fence to +strip, no enum to re-case, no parse fallback. The prompt carries the judgment; +the schema carries the format. -Both functions degrade gracefully to deterministic fallback output when -`OPENAI_API_KEY` is unset (they never throw). +`fromRawAnalysis` then applies the one rule the prompt states but cannot enforce +on itself: a bill that is primarily a social issue abstains, whatever judgment +the model reached. -> **Note on `steel_man`:** it is a persisted, human-editable field (admin edit -> page), **not** produced by the analysis prompt. `summarizeBillText` correctly -> leaves it empty; editors fill it in. The eval suite deliberately does not -> assert it. +The function degrades gracefully to deterministic fallback output when +`OPENAI_API_KEY` is unset (it never throws). A fallback carries `isFallback` and +is never persisted or posted to Slack. ## Data flow +Analysis happens on a schedule, never inside a page render. + ``` -Civics Project API ──► getBillFromCivicsProjectApi ──► fetchBillMarkdown (xml→md) - │ - summarizeBillText + socialIssueGrader (LLM) - │ - onBillNotInDatabase → persist to MongoDB - │ - page.tsx / [id]/page.tsx ◄── getUnifiedBillById ◄── Bill model + ┌─ src/instrumentation.ts (interval) ─┐ + │ ├─► refreshBills() [Mongo lease] + └─ POST /bills/api/refresh (secret) ─┘ │ + ├─ every bill: updateBillFacts + │ (status, stages, sponsor) + └─ changed/new only, up to budget: + fetchBillMarkdown (xml→md) + summarizeBillText (LLM) + saveBillAnalysis → MongoDB + notifyNewBillAnalysis → Slack + + page.tsx ──────────┐ + ├─► mergeBillLists / applyApiFacts ──► API facts + stored verdict + [id]/page.tsx ─────┘ ``` -- **List page** (`page.tsx` → `BillExplorer.tsx`) reads analyzed bills from the - DB and renders filterable cards. -- **Detail page** (`[id]/page.tsx`) renders the summary, per-tenet breakdown - (`components/BillDetail/BillTenets.tsx`), judgment badge - (`components/Judgement/`), and QP questions. -- Analyzed results are cached in **MongoDB** (`models/Bill.ts`) so a bill is only - sent to the LLM once (re-run explicitly via the reprocess route). +**Precedence, applied on both pages** (`utils/merge-bill.ts`): factual fields — +status, stages, sponsor, genres — come from the Civics API; the verdict — +summary, tenets, judgment, rationale, steel man — comes from the database. Both +pages go through the same helper, so they cannot disagree about a bill's status. + +- **List page** (`page.tsx` → `BillExplorer.tsx`) renders filterable cards. +- **Detail page** (`[id]/page.tsx`) renders the summary, per-tenet breakdown, + judgment badge, steel man, QP questions, and a provenance line saying when the + verdict was computed and from which bill text. +- A bill the sweep has not reached yet shows its real facts with "Analysis + pending" in place of a verdict. + +### The refresh sweep + +`services/refresh.ts` is the only thing that spends OpenAI calls automatically. +It takes a Mongo lease (`models/JobLock.ts`) so multiple replicas are safe, and +re-analyzes a bill only when it is new, has no analysis, or its bill text has +changed — the decision lives in `services/refresh-decision.ts` and is unit +tested. A per-sweep `analysisBudget` (default 10) bounds the cost; bills over +budget still get their facts refreshed and are picked up next sweep. + +Run one by hand: + +```bash +curl -X POST localhost:5050/bills/api/refresh \ + -H "Authorization: Bearer $BILLS_CRON_SECRET" +# optional body: {"analysisBudget": 3, "force": true} +``` ## Directory map @@ -87,12 +121,17 @@ Civics Project API ──► getBillFromCivicsProjectApi ──► fetchBillMark | `page.tsx`, `BillExplorer.tsx` | List page + client-side filtering | | `[id]/page.tsx`, `components/BillDetail/*` | Bill detail view | | `[id]/edit/page.tsx` | Admin edit form (gated) | -| `prompt/summary-and-vote-prompt.ts` | Tenets, social-issue rules, judgment signals, output schema | -| `services/billApi.ts` | Civics fetch, `summarizeBillText`, markdown conversion, DB persistence | -| `services/social-issue-grader.ts` | Binary social-issue classifier | +| `prompt/summary-and-vote-prompt.ts` | Tenets, social-issue rules, judgment signals | +| `prompt/analysis-schema.ts` | Structured-output JSON schema for the response | +| `services/billApi.ts` | Civics fetch, `summarizeBillText`, markdown conversion, DB writes | +| `services/refresh.ts`, `services/refresh-decision.ts` | The scheduled sweep and its re-analysis decision | +| `utils/merge-bill.ts` | API-facts / stored-verdict precedence, shared by both pages | +| `models/JobLock.ts` | Mongo lease so one sweep runs at a time | +| `src/instrumentation.ts` (repo root) | Schedules the sweep on server start | | `server/*` | DB + Civics read helpers (`getUnifiedBillById`, etc.) | | `models/*` | Mongoose `Bill` and `User` schemas | | `api/[id]/route.ts`, `api/[id]/reprocess/route.ts` | Update / re-analyze a bill (gated) | +| `api/refresh/route.ts` | Run the sweep on demand (bearer token) | | `utils/xml-to-md/` | Deterministic bill-XML → markdown conversion | | `evals/` | Manual LLM eval suite — see `evals/README.md` | @@ -112,7 +151,7 @@ Configured in `env.ts`. Put these in `.env.local` for local dev. | Var | Purpose | |---|---| -| `OPENAI_API_KEY` | LLM analysis + social grader (fallback output if unset) | +| `OPENAI_API_KEY` | LLM analysis (fallback output if unset) | | `CIVICS_PROJECT_API_KEY` | Fetch bills from the Civics Project API | | `CIVICS_PROJECT_BASE_URL` | Defaults to `https://api.civicsproject.org` | | `MONGO_URI` (or `MONGODB_URI`) | Analyzed-bill + user store | @@ -121,17 +160,21 @@ Configured in `env.ts`. Put these in `.env.local` for local dev. | `NEXT_PUBLIC_APP_URL` | Absolute URL for OG/metadata | | `BILLS_DEV_OPEN_ACCESS` | Dev-only admin bypass (see Auth) | | `BILLS_SLACK_WEBHOOK_URL` | Slack incoming webhook for #builder-mp — posts each newly generated analysis (summary, overall vote, tenet breakdown). No posts if unset. | +| `BILLS_REFRESH_ENABLED` | `true` starts the scheduled sweep. Off by default, so dev and one-off containers never spend tokens. | +| `BILLS_REFRESH_INTERVAL_MINUTES` | Sweep interval. Defaults to 60. | +| `BILLS_CRON_SECRET` | Bearer token for `POST /bills/api/refresh`. Without it the route returns 503. | ## Local development ```bash pnpm dev # Next.js dev server on :5050 — visit /bills +pnpm test:bills # unit tests for the merge precedence and sweep decision ``` ## Evaluating the LLM features The `evals/` directory holds a manual, token-conscious eval suite that runs the -real `summarizeBillText` and `socialIssueGrader` against committed, hand-labeled +real `summarizeBillText` against committed, hand-labeled **real Parliament-45 bills** (fixtures span align / conflict / abstain / administrative). It gates on deterministic structural checks and reports judgment + social-issue accuracy. diff --git a/src/app/bills/[id]/page.tsx b/src/app/bills/[id]/page.tsx index 3d4e253b..59c3b4e1 100644 --- a/src/app/bills/[id]/page.tsx +++ b/src/app/bills/[id]/page.tsx @@ -31,6 +31,9 @@ import { } from "@/app/bills/consts/general"; import { BillShare } from "@/app/bills/components/BillDetail/BillShare"; import { shouldShowDetermination } from "@/app/bills/utils/should-show-determination/should-show-determination.util"; +import { applyApiFacts } from "@/app/bills/utils/merge-bill"; +import { BillProvenance } from "@/app/bills/components/BillDetail/BillProvenance"; +import { BillSteelMan } from "@/app/bills/components/BillDetail/BillSteelMan"; // Next.js requires route segment configs to be literal values (not imported constants) export const revalidate = 120; // seconds - cache individual bill pages @@ -61,18 +64,24 @@ export default async function BillDetail({ params }: Params) { env.NODE_ENV === "production" ? BUILD_CANADA_URL : origin || BUILD_CANADA_URL; - // Try database first, then fallback to API - const dbBill = await getBillByIdFromDB(id); - let unifiedBill: UnifiedBill | null = null; + // The stored analysis and the API's current facts are independent reads, so + // fetch them together. The API call is allowed to fail — a Civics outage + // should cost the page its freshest status, not the whole page. + const [dbBill, apiBill] = await Promise.all([ + getBillByIdFromDB(id), + getBillFromCivicsProjectApi(id).catch((error) => { + console.error(`[bills] Civics fetch failed for ${id}:`, error); + return null; + }), + ]); - if (dbBill) { - unifiedBill = fromBuildCanadaDbBill(dbBill); - } else { - const apiBill = await getBillFromCivicsProjectApi(id); - if (apiBill) { - unifiedBill = await fromCivicsProjectApiBill(apiBill); - } - } + const apiUnified = apiBill ? fromCivicsProjectApiBill(apiBill) : null; + // Facts from the API, verdict from the database — the same precedence the + // list page applies, so the two pages can no longer disagree about a bill's + // status or stages. + const unifiedBill: UnifiedBill | null = dbBill + ? applyApiFacts(fromBuildCanadaDbBill(dbBill), apiUnified) + : apiUnified; if (!unifiedBill) { return ( @@ -135,6 +144,8 @@ export default async function BillDetail({ params }: Params) { shouldDisplay: shouldDisplayDetermination, }} /> + + {shouldDisplayDetermination && } {shouldDisplayDetermination && unifiedBill.question_period_questions && unifiedBill.question_period_questions.length > 0 && ( diff --git a/src/app/bills/api/[id]/reprocess/route.ts b/src/app/bills/api/[id]/reprocess/route.ts index b089dc87..af106f8c 100644 --- a/src/app/bills/api/[id]/reprocess/route.ts +++ b/src/app/bills/api/[id]/reprocess/route.ts @@ -9,6 +9,7 @@ import { type ApiBillDetail, fetchBillMarkdown, getBillFromCivicsProjectApi, + saveBillAnalysis, summarizeBillText, } from "@/app/bills/services/billApi"; import { notifyNewBillAnalysis } from "@/app/bills/services/slack-notifier"; @@ -106,47 +107,9 @@ export async function POST( ); } - const latestStageDate = - apiBill.stages && apiBill.stages.length > 0 - ? apiBill.stages[apiBill.stages.length - 1].date - : (apiBill.updatedAt ?? apiBill.date); - - await Bill.updateOne( - { billId: id }, - { - $set: { - // Refreshed metadata from the Civics Project API - title: apiBill.title, - status: apiBill.status, - sponsorParty: apiBill.sponsorParty, - genres: apiBill.genres, - supportedRegion: apiBill.supportedRegion, - stages: apiBill.stages?.map((stage) => ({ - stage: stage.stage, - state: stage.state, - house: stage.house, - date: new Date(stage.date), - })), - billTextsCount: Array.isArray(apiBill.billTexts) - ? apiBill.billTexts.length - : 0, - source, - // Regenerated AI analysis - summary: analysis.summary, - short_title: - apiBill.shortTitle ?? analysis.short_title ?? existing.short_title, - tenet_evaluations: analysis.tenet_evaluations, - final_judgment: analysis.final_judgment, - rationale: analysis.rationale, - needs_more_info: analysis.needs_more_info, - missing_details: analysis.missing_details, - steel_man: analysis.steel_man, - question_period_questions: analysis.question_period_questions ?? [], - lastUpdatedOn: new Date(latestStageDate), - }, - }, - { upsert: false }, - ); + // One write path, shared with the refresh sweep: it applies the same + // provenance stamp and the same social-issue-forces-abstain rule. + await saveBillAnalysis({ bill: apiBill, analysis, source }); await notifyNewBillAnalysis({ billId: id, diff --git a/src/app/bills/api/refresh/route.ts b/src/app/bills/api/refresh/route.ts new file mode 100644 index 00000000..c3518a0f --- /dev/null +++ b/src/app/bills/api/refresh/route.ts @@ -0,0 +1,55 @@ +import { NextResponse } from "next/server"; +import { timingSafeEqual } from "node:crypto"; +import { refreshBills } from "@/app/bills/services/refresh"; + +export const dynamic = "force-dynamic"; +// A sweep analyzes up to its budget of bills at reasoning.effort "high". +export const maxDuration = 300; + +function tokenMatches(provided: string, expected: string): boolean { + const a = Buffer.from(provided); + const b = Buffer.from(expected); + // timingSafeEqual throws on a length mismatch, which is itself a mismatch. + return a.length === b.length && timingSafeEqual(a, b); +} + +/** + * Run the refresh sweep on demand. + * + * The scheduled run lives in `src/instrumentation.ts`; this is the manual kick, + * and the seam for moving the schedule out to an external scheduler later + * without changing what a sweep does. + * + * Optional body: { "analysisBudget": number, "force": boolean }. + */ +export async function POST(request: Request) { + const secret = process.env.BILLS_CRON_SECRET; + if (!secret) { + return NextResponse.json( + { error: "BILLS_CRON_SECRET is not configured" }, + { status: 503 }, + ); + } + + const provided = request.headers.get("authorization")?.replace(/^Bearer /, ""); + if (!provided || !tokenMatches(provided, secret)) { + return NextResponse.json({ error: "Unauthorized" }, { status: 401 }); + } + + let body: { analysisBudget?: number; force?: boolean } = {}; + try { + body = (await request.json()) ?? {}; + } catch { + // No body is fine — the defaults are the normal case. + } + + const result = await refreshBills({ + analysisBudget: + typeof body.analysisBudget === "number" ? body.analysisBudget : undefined, + force: body.force === true, + }); + + // A skipped sweep is a normal outcome (another replica holds the lease), not + // an error — 200 with the reason, so a scheduler does not retry into a loop. + return NextResponse.json(result); +} diff --git a/src/app/bills/components/BillDetail/BillProvenance.tsx b/src/app/bills/components/BillDetail/BillProvenance.tsx new file mode 100644 index 00000000..574cd48d --- /dev/null +++ b/src/app/bills/components/BillDetail/BillProvenance.tsx @@ -0,0 +1,56 @@ +import React from "react"; +import type { UnifiedBill } from "@/app/bills/utils/billConverters"; + +interface BillProvenanceProps { + bill: UnifiedBill; +} + +const DATE_FORMAT = new Intl.DateTimeFormat("en-CA", { + day: "numeric", + month: "long", + year: "numeric", + timeZone: "UTC", +}); + +/** + * Says which version of the bill was judged, and when. + * + * Bills are amended as they move through the House, and a verdict on the first + * reading text is not a verdict on the text that passed. Naming the date is the + * difference between an opinion and a citable one. + */ +export function BillProvenance({ bill }: BillProvenanceProps) { + if (bill.analysisPending) { + return ( +

+ Analysis pending — this bill has not been assessed yet. The facts above + come from the Parliament of Canada record. +

+ ); + } + + if (!bill.analysisGeneratedAt) return null; + + const generatedAt = new Date(bill.analysisGeneratedAt); + if (Number.isNaN(generatedAt.getTime())) return null; + + return ( +

+ Assessed {DATE_FORMAT.format(generatedAt)} + {bill.analysisSourceRef ? ( + <> + {" from the "} + + bill text published at that date + + + ) : null} + . Later amendments may not be reflected. +

+ ); +} diff --git a/src/app/bills/components/BillDetail/BillSteelMan.tsx b/src/app/bills/components/BillDetail/BillSteelMan.tsx new file mode 100644 index 00000000..bee62ec7 --- /dev/null +++ b/src/app/bills/components/BillDetail/BillSteelMan.tsx @@ -0,0 +1,34 @@ +import React from "react"; +import type { UnifiedBill } from "@/app/bills/utils/billConverters"; +import { Markdown } from "../Markdown/markdown"; +import { Card, CardContent, CardHeader, CardTitle } from "../ui/card"; + +interface BillSteelManProps { + bill: UnifiedBill; +} + +/** + * The strongest good-faith case against the verdict this page just gave. + * + * The field has existed on the model and the admin edit form since the start, + * but nothing generated it and nothing rendered it. A tool that issues verdicts + * on legislation is more credible for showing its own best counter-argument. + */ +export function BillSteelMan({ bill }: BillSteelManProps) { + const steelMan = bill.steel_man?.trim(); + if (!steelMan) return null; + + return ( + + + The other side + + +

+ The strongest case against this judgement. +

+ {steelMan} + + + ); +} diff --git a/src/app/bills/components/BillDetail/index.ts b/src/app/bills/components/BillDetail/index.ts index 120e4044..4c25e0a4 100644 --- a/src/app/bills/components/BillDetail/index.ts +++ b/src/app/bills/components/BillDetail/index.ts @@ -5,3 +5,5 @@ export { BillFullText } from "./BillFullText"; export { BillAnalysis } from "./BillAnalysis"; export { BillQuestions } from "./BillQuestions"; export { BillContact } from "./BillContact"; +export { BillSteelMan } from "./BillSteelMan"; +export { BillProvenance } from "./BillProvenance"; diff --git a/src/app/bills/consts/general.ts b/src/app/bills/consts/general.ts index 77ed50fd..1571185a 100644 --- a/src/app/bills/consts/general.ts +++ b/src/app/bills/consts/general.ts @@ -9,3 +9,6 @@ export const BUILD_CANADA_TWITTER_HANDLE = "@buildcanada"; // Note: Route segment configs (export const revalidate in page.tsx files) must use literal values // due to Next.js static analysis requirements. Use these constants only for runtime fetch calls. export const BILL_API_REVALIDATE_INTERVAL = 600; // Bill API data cache (fetch revalidation) + +/** The Parliament whose bills Builder MP covers. */ +export const CANADIAN_PARLIAMENT_NUMBER = 45; diff --git a/src/app/bills/env.ts b/src/app/bills/env.ts index 2bd28c77..30c77dbe 100644 --- a/src/app/bills/env.ts +++ b/src/app/bills/env.ts @@ -31,7 +31,10 @@ export const env = { "CIVICS_PROJECT_API_KEY", process.env.CIVICS_PROJECT_API_KEY, ), - CIVICS_PROJECT_BASE_URL: optional("CIVICS_PROJECT_BASE_URL", ENDPOINT), + // Honours the env var when set; ENDPOINT is the default, not an override. + CIVICS_PROJECT_BASE_URL: + optional("CIVICS_PROJECT_BASE_URL", process.env.CIVICS_PROJECT_BASE_URL) ?? + ENDPOINT, MONGO_URI: optional( "MONGO_URI", (process.env.MONGO_URI || process.env.MONGODB_URI)?.trim(), @@ -48,6 +51,20 @@ export const env = { "BILLS_SLACK_WEBHOOK_URL", process.env.BILLS_SLACK_WEBHOOK_URL, ), + // The scheduled refresh sweep (src/instrumentation.ts). Off unless enabled. + BILLS_REFRESH_ENABLED: optional( + "BILLS_REFRESH_ENABLED", + process.env.BILLS_REFRESH_ENABLED, + ), + BILLS_REFRESH_INTERVAL_MINUTES: optional( + "BILLS_REFRESH_INTERVAL_MINUTES", + process.env.BILLS_REFRESH_INTERVAL_MINUTES, + ), + // Bearer token for POST /bills/api/refresh. + BILLS_CRON_SECRET: optional( + "BILLS_CRON_SECRET", + process.env.BILLS_CRON_SECRET, + ), }; /** diff --git a/src/app/bills/evals/README.md b/src/app/bills/evals/README.md index c92a8f11..65cd2e56 100644 --- a/src/app/bills/evals/README.md +++ b/src/app/bills/evals/README.md @@ -1,9 +1,12 @@ # /bills LLM eval suite -Manual evals for the two LLM touchpoints in `/bills`: +Manual evals for the single LLM touchpoint in `/bills`: -- `summarizeBillText` (`services/billApi.ts`) — summary, 8 tenet evaluations, `final_judgment`, 3 Question Period questions. -- `socialIssueGrader` (`services/social-issue-grader.ts`) — binary "is this primarily a social issue" classifier. +- `summarizeBillText` (`services/billApi.ts`) — summary, 8 tenet evaluations, `final_judgment`, steel man, 3 Question Period questions, and `is_social_issue`. + +(There used to be a second call, `socialIssueGrader`, answering the social-issue +question separately from the first 8000 characters. It disagreed with the main +call's own answer, so it is gone; one call now answers both.) The suite calls the **real** functions (full prompt → parse → normalize pipeline) against committed, hand-labeled bill fixtures. It is **run manually only** — it spends OpenAI tokens on a cache miss and is never wired into `build`, `lint`, or CI. @@ -13,37 +16,33 @@ The suite calls the **real** functions (full prompt → parse → normalize pipe export OPENAI_API_KEY=sk-... # a real key is needed for a real eval pnpm eval:bills # all fixtures (cached where possible) pnpm eval:bills --refresh # bypass cache, re-call the API -pnpm eval:bills --only=social # only the social-issue classifier -pnpm eval:bills --only=analysis # only summarizeBillText pnpm eval:bills --grep=tax # only fixtures whose id contains "tax" pnpm eval:bills --fallback # force the no-key fallback path (0 tokens) ``` Exit code is non-zero when any **structural** check fails. Accuracy (judgment -vs label, social-issue confusion matrix) and cross-consistency are reported but -never gate the run — they are probabilistic. +vs label, social-issue confusion matrix) is reported but never gates the run — +it is probabilistic. ## What it checks - **Structural** (`checks/analysis-checks.ts`, deterministic, gates the run): - non-empty summary (steel_man is a human-editable field, not LLM-generated, so - it is not checked); exactly 8 tenets with ids 1–8 and valid - `aligns|conflicts|neutral`; valid `final_judgment`; exactly 3 non-empty QP - questions with no "Mr./Madam Speaker" prefix; no `Build Canada`/`we`/`our` - self-reference in prose; (soft warning) tenet text not quoted in the summary. + non-empty summary; non-empty steel man; exactly 8 tenets with ids 1–8 and + valid `aligns|conflicts|neutral`; valid `final_judgment`; `abstain` whenever + `isSocialIssue`; non-empty `missing_details` whenever `needs_more_info`; + exactly 3 non-empty QP questions with no "Mr./Madam Speaker" prefix; no + `Build Canada`/`we`/`our` self-reference in prose; (soft warning) tenet text + not quoted in the summary. - **Judgment accuracy** — `final_judgment` vs the fixture's `finalJudgment` label. -- **Social-issue accuracy** — `socialIssueGrader` vs `isSocialIssue` label, with - a confusion matrix and precision/recall. -- **Cross-consistency warning** — flags when `summarizeBillText` abstains but - `socialIssueGrader` disagrees. This is expected: `summarizeBillText` ignores - its own `is_social_issue` field, and the app's stored `isSocialIssue` comes - from the separate grader. +- **Social-issue accuracy** — the analysis's `is_social_issue` vs the fixture's + `isSocialIssue` label, with a confusion matrix and precision/recall. ## Caching Each response is cached to `.cache/` (gitignored), keyed by a hash of the input text **and** the prompt text. Editing a prompt invalidates its entries -automatically; for other changes (model, reasoning effort) bump `VERSION` in +automatically; for other changes (model, reasoning effort, the response schema) +bump `VERSION` in `lib/cache.ts`. Re-runs read from disk, so iterating on checks/fixtures costs no tokens. `--refresh` forces a re-call. A machine-readable `report.json` is written to `.cache/` each run for diffing. diff --git a/src/app/bills/evals/checks/analysis-checks.ts b/src/app/bills/evals/checks/analysis-checks.ts index 09150bca..767c9360 100644 --- a/src/app/bills/evals/checks/analysis-checks.ts +++ b/src/app/bills/evals/checks/analysis-checks.ts @@ -32,9 +32,7 @@ function warn(name: string, pass: boolean, message: string): CheckResult { export function checkAnalysis(a: BillAnalysis): CheckResult[] { const results: CheckResult[] = []; - // summary non-empty. NOTE: steel_man is intentionally NOT checked here — it - // is a human-editable editorial field (admin edit page), not produced by - // SUMMARY_AND_VOTE_PROMPT, so summarizeBillText correctly leaves it "". + // summary non-empty results.push( typeof a.summary === "string" && a.summary.trim().length > 0 ? ok("summary-present") @@ -126,6 +124,35 @@ export function checkAnalysis(a: BillAnalysis): CheckResult[] { ), ); + // steel man: the model is asked for it, and the bill page renders it + results.push( + typeof a.steel_man === "string" && a.steel_man.trim().length > 0 + ? ok("steel-man-present") + : fail("steel-man-present", "steel_man is empty or not a string"), + ); + + // A social issue must abstain. fromRawAnalysis enforces this, so a failure + // here means the enforcement was bypassed, not that the model disagreed. + results.push( + !a.isSocialIssue || a.final_judgment === "abstain" + ? ok("social-issue-abstains") + : fail( + "social-issue-abstains", + `isSocialIssue but final_judgment=${JSON.stringify(a.final_judgment)}`, + ), + ); + + // needs_more_info is a claim; missing_details says what is missing + results.push( + !a.needs_more_info || + (Array.isArray(a.missing_details) && a.missing_details.length > 0) + ? ok("missing-details-when-needed") + : fail( + "missing-details-when-needed", + "needs_more_info is true but missing_details is empty", + ), + ); + // soft: tenet titles should not be quoted verbatim in the summary const summaryLower = (a.summary || "").toLowerCase(); const leakedTenets = Object.values(TENETS).filter((title) => { diff --git a/src/app/bills/evals/lib/cache.ts b/src/app/bills/evals/lib/cache.ts index 7400db48..bf818d27 100644 --- a/src/app/bills/evals/lib/cache.ts +++ b/src/app/bills/evals/lib/cache.ts @@ -9,7 +9,9 @@ import { fileURLToPath } from "node:url"; * prompt invalidates its cache automatically — this is for other changes (e.g. * model, reasoning effort) that the key would otherwise miss. */ -const VERSION = "1"; +// Bumped when the response shape changes independently of the prompt text — +// v2 is the move to Structured Outputs (`prompt/analysis-schema.ts`). +const VERSION = "2"; const CACHE_DIR = join(dirname(fileURLToPath(import.meta.url)), "..", ".cache"); diff --git a/src/app/bills/evals/lib/report.ts b/src/app/bills/evals/lib/report.ts index d5b6ce4e..74423e9a 100644 --- a/src/app/bills/evals/lib/report.ts +++ b/src/app/bills/evals/lib/report.ts @@ -18,8 +18,6 @@ export type FixtureReport = { checks: CheckResult[]; judgment?: { actual: string; expected?: string; match?: boolean }; social?: { actual: boolean; expected: boolean; match: boolean }; - /** actual abstain-vs-grader agreement, for the cross-consistency flag */ - consistency?: { analysisAbstain: boolean; socialIssue: boolean; agree: boolean }; cached: boolean; /** true when produced via the no-key fallback path (--fallback), not the API */ fallback?: boolean; @@ -88,11 +86,6 @@ export function printReport(reports: FixtureReport[]): { errorFailures: number } ` ${mark} social_issue=${r.social.actual} expected=${r.social.expected}`, ); } - if (r.consistency && !r.consistency.agree) { - console.log( - ` ${c.yellow("⚠")} consistency: analysis abstain=${r.consistency.analysisAbstain} but grader social_issue=${r.consistency.socialIssue}`, - ); - } } /* ---- structural summary ---- */ @@ -129,22 +122,6 @@ export function printReport(reports: FixtureReport[]): { errorFailures: number } ); } - /* ---- consistency warnings ---- */ - const disagreements = reports.filter((r) => r.consistency && !r.consistency.agree); - if (disagreements.length) { - console.log(c.bold("\n=== Cross-consistency warnings ===\n")); - console.log( - c.yellow( - ` ${disagreements.length} fixture(s): summarizeBillText abstain disagrees with socialIssueGrader`, - ), - ); - console.log( - c.dim( - " (expected: summarizeBillText ignores its own is_social_issue field; abstain is model-driven)", - ), - ); - } - return { errorFailures }; } diff --git a/src/app/bills/evals/run.ts b/src/app/bills/evals/run.ts index ee0ad433..8b2b7b27 100644 --- a/src/app/bills/evals/run.ts +++ b/src/app/bills/evals/run.ts @@ -4,8 +4,6 @@ * * pnpm eval:bills # run all fixtures (cached where possible) * pnpm eval:bills --refresh # bypass cache, re-call the API - * pnpm eval:bills --only=social # only the social-issue classifier - * pnpm eval:bills --only=analysis # only summarizeBillText * pnpm eval:bills --grep=tax # only fixtures whose id includes "tax" * pnpm eval:bills --fallback # force no-key fallback path (0 tokens) * @@ -13,10 +11,6 @@ * consistency are reported but never gate the run (they are probabilistic). */ import { summarizeBillText } from "@/app/bills/services/billApi"; -import { - socialIssueGrader, - SOCIAL_ISSUE_GRADER_PROMPT, -} from "@/app/bills/services/social-issue-grader"; import { SUMMARY_AND_VOTE_PROMPT } from "@/app/bills/prompt/summary-and-vote-prompt"; import { FIXTURES, loadFixtureText } from "./fixtures/bills"; import { checkAnalysis } from "./checks/analysis-checks"; @@ -29,7 +23,6 @@ function parseArgs(argv: string[]) { return { refresh: argv.includes("--refresh"), fallback: argv.includes("--fallback"), - only: get("only") as "social" | "analysis" | undefined, grep: get("grep"), }; } @@ -50,9 +43,6 @@ async function main() { ); } - const runAnalysis = args.only !== "social"; - const runSocial = args.only !== "analysis"; - const fixtures = FIXTURES.filter( (f) => !args.grep || f.id.includes(args.grep), ); @@ -73,57 +63,35 @@ async function main() { cached: false, fallback: args.fallback, }; - let analysisAbstain: boolean | undefined; - let socialResult: boolean | undefined; - - if (runAnalysis) { - // In fallback mode the result is deterministic and free — skip the cache. - const { value: analysis, cached } = args.fallback - ? { value: await summarizeBillText(text, { bypassCap: true }), cached: false } - : await runCached( - "analysis", - text, - SUMMARY_AND_VOTE_PROMPT, - () => summarizeBillText(text, { bypassCap: true }), - { refresh: args.refresh, stats }, - ); - report.cached = cached; - report.checks = checkAnalysis(analysis); - analysisAbstain = analysis.final_judgment === "abstain"; - report.judgment = { - actual: analysis.final_judgment, - expected: f.expected.finalJudgment, - match: f.expected.finalJudgment - ? analysis.final_judgment === f.expected.finalJudgment - : undefined, - }; - } - - if (runSocial) { - const { value: social } = args.fallback - ? { value: await socialIssueGrader(text) } - : await runCached( - "social", - text, - SOCIAL_ISSUE_GRADER_PROMPT, - () => socialIssueGrader(text), - { refresh: args.refresh, stats }, - ); - socialResult = social; - report.social = { - actual: social, - expected: f.expected.isSocialIssue, - match: social === f.expected.isSocialIssue, - }; - } + // One call answers both the judgment and the social-issue question; the + // separate grader that used to answer it again (and disagree) is gone. + const { value: analysis, cached } = args.fallback + ? { + value: await summarizeBillText(text, { bypassCap: true }), + cached: false, + } + : await runCached( + "analysis", + text, + SUMMARY_AND_VOTE_PROMPT, + () => summarizeBillText(text, { bypassCap: true }), + { refresh: args.refresh, stats }, + ); - if (analysisAbstain !== undefined && socialResult !== undefined) { - report.consistency = { - analysisAbstain, - socialIssue: socialResult, - agree: analysisAbstain === socialResult, - }; - } + report.cached = cached; + report.checks = checkAnalysis(analysis); + report.judgment = { + actual: analysis.final_judgment, + expected: f.expected.finalJudgment, + match: f.expected.finalJudgment + ? analysis.final_judgment === f.expected.finalJudgment + : undefined, + }; + report.social = { + actual: analysis.isSocialIssue, + expected: f.expected.isSocialIssue, + match: analysis.isSocialIssue === f.expected.isSocialIssue, + }; reports.push(report); } diff --git a/src/app/bills/models/Bill.ts b/src/app/bills/models/Bill.ts index 38185ea9..cd8cf050 100644 --- a/src/app/bills/models/Bill.ts +++ b/src/app/bills/models/Bill.ts @@ -58,6 +58,14 @@ export interface BillDocument extends mongoose.Document { votes?: VoteRecord[]; billTextsCount?: number; // track number of bill texts to detect changes isSocialIssue?: boolean; + /** + * When the stored analysis was generated, and the bill-text URL it was + * generated from. Together these let the refresh sweep tell a stale verdict + * from a current one, and let the page tell the reader which version of the + * bill was judged. + */ + analysisGeneratedAt?: Date; + analysisSourceRef?: string; question_period_questions?: Array<{ question: string }>; } @@ -130,6 +138,8 @@ const BillSchema = new Schema( votes: { type: [VoteSchema], default: [] }, billTextsCount: { type: Number }, isSocialIssue: { type: Boolean, default: false }, + analysisGeneratedAt: { type: Date }, + analysisSourceRef: { type: String }, question_period_questions: { type: [{ question: { type: String, required: true } }], default: [], diff --git a/src/app/bills/models/JobLock.ts b/src/app/bills/models/JobLock.ts new file mode 100644 index 00000000..3892f833 --- /dev/null +++ b/src/app/bills/models/JobLock.ts @@ -0,0 +1,61 @@ +import { Schema, model, models } from "mongoose"; + +export interface JobLockDocument { + _id: string; + expiresAt: Date; + holder?: string; +} + +/** + * A lease, so that only one process runs a given job at a time. + * + * The refresh sweep spends OpenAI calls. One container runs the site today, but + * a second replica would otherwise mean a second sweep analyzing the same bills + * at the same time — this makes scaling out safe rather than expensive. + */ +const JobLockSchema = new Schema({ + _id: { type: String, required: true }, + expiresAt: { type: Date, required: true }, + holder: { type: String }, +}); + +export const JobLock = + models.JobLock || model("JobLock", JobLockSchema); + +/** + * Take the lease if it is free or expired. Returns false when another process + * holds it. The TTL bounds how long a crashed holder can block the job. + */ +export async function acquireLock( + name: string, + ttlMs: number, + holder: string, +): Promise { + const now = new Date(); + try { + await JobLock.findOneAndUpdate( + { _id: name, expiresAt: { $lt: now } }, + { $set: { expiresAt: new Date(now.getTime() + ttlMs), holder } }, + { upsert: true }, + ); + return true; + } catch (error) { + // A duplicate-key error is the expected outcome when the lease is held: the + // filter misses, the upsert tries to insert, and the _id already exists. + if ((error as { code?: number })?.code === 11000) return false; + console.error(`[bills] Failed to acquire lock "${name}":`, error); + return false; + } +} + +export async function releaseLock(name: string, holder: string): Promise { + try { + await JobLock.updateOne( + { _id: name, holder }, + { $set: { expiresAt: new Date(0) } }, + ); + } catch (error) { + // The TTL expiry is the backstop, so a failed release is not fatal. + console.error(`[bills] Failed to release lock "${name}":`, error); + } +} diff --git a/src/app/bills/page.tsx b/src/app/bills/page.tsx index 1b7b064e..4b87a8d1 100644 --- a/src/app/bills/page.tsx +++ b/src/app/bills/page.tsx @@ -1,6 +1,8 @@ import { BillSummary } from "./types"; import BillExplorer from "./BillExplorer"; import { getAllBillsFromDB } from "@/app/bills/server/get-all-bills-from-db"; +import { getApiBills } from "@/app/bills/server/get-api-bills"; +import { mergeBillLists } from "@/app/bills/utils/merge-bill"; import { fromBuildCanadaDbBill } from "@/app/bills/utils/billConverters"; import type { Metadata } from "next"; import { headers } from "next/headers"; @@ -10,15 +12,8 @@ import { BUILD_CANADA_TWITTER_HANDLE, PROJECT_NAME } from "@/app/bills/consts/ge import FAQModalTrigger from "./FAQModalTrigger"; import { PageHeader } from "@/components/ui/page-header"; -const CANADIAN_PARLIAMENT_NUMBER = 45; type HomeSearchParams = { cache?: string }; -function toIsoString(value?: Date | string): string | undefined { - if (!value) return undefined; - const parsed = value instanceof Date ? value : new Date(value); - return Number.isNaN(parsed.getTime()) ? undefined : parsed.toISOString(); -} - // Force runtime generation (avoid build-time pre-render) and cache in-memory. export const dynamic = "auto"; // Next.js requires route segment configs to be literal values (not imported constants) @@ -79,36 +74,8 @@ export async function generateMetadata(): Promise { const shouldUseLocalCache = process.env.NODE_ENV === "production"; let mergedBillsCache: { data: BillSummary[]; expiresAt: number } | null = null; -async function getApiBills(): Promise { - try { - const response = await fetch( - `${env.CIVICS_PROJECT_BASE_URL}/canada/bills/${CANADIAN_PARLIAMENT_NUMBER}`, - { - // Cache for 5 minutes in production, no cache in development - ...(process.env.NODE_ENV === "production" - ? { next: { revalidate: 300 } } - : { cache: "no-store" }), - headers: { - "Content-Type": "application/json", - Authorization: env.CIVICS_PROJECT_API_KEY - ? `Bearer ${env.CIVICS_PROJECT_API_KEY}` - : "", - }, - }, - ); - if (!response.ok) { - throw new Error("Failed to fetch bills from API"); - } - const { data } = await response.json(); - return Array.isArray(data) ? (data as BillSummary[]) : (data?.bills ?? []); - } catch (error) { - console.error("Error fetching API bills:", error); - return []; - } -} - async function getMergedBills(): Promise { - const uri = process.env.MONGO_URI || ""; + const uri = env.MONGO_URI || ""; const hasValidMongoUri = uri.startsWith("mongodb://") || uri.startsWith("mongodb+srv://"); // The API and DB reads are independent — run them concurrently instead of @@ -118,75 +85,7 @@ async function getMergedBills(): Promise { hasValidMongoUri ? getAllBillsFromDB() : Promise.resolve([]), ]); - // Convert DB bills to UnifiedBill format first, then to BillSummary - const dbBillsAsUnified = dbBills.map(fromBuildCanadaDbBill); - - // Create a map of DB bills by billId for quick lookup - const dbBillsMap = new Map( - dbBillsAsUnified.map((bill) => [bill.billId, bill]), - ); - - // Merge API bills with DB data - const mergedBills: BillSummary[] = apiBills.map((apiBill) => { - const dbBill = dbBillsMap.get(apiBill.billID); - - if (dbBill) { - // Merge API bill with DB data (DB data takes precedence for analysis fields) - return { - ...dbBill, - ...apiBill, - shortTitle: dbBill.short_title || apiBill.shortTitle, - summary: dbBill.summary, - isSocialIssue: dbBill.isSocialIssue, - final_judgment: dbBill.final_judgment as BillSummary["final_judgment"], - rationale: dbBill.rationale, - needs_more_info: dbBill.needs_more_info, - missing_details: dbBill.missing_details, - genres: dbBill.genres, - parliamentNumber: dbBill.parliamentNumber, - sessionNumber: dbBill.sessionNumber, - }; - } - - // Return API bill as-is if no DB data - return apiBill; - }); - - // Add any DB-only bills that aren't in the API response - for (const [billId, dbBill] of dbBillsMap) { - if (!mergedBills.find((bill) => bill.billID === billId)) { - // Convert DB bill to BillSummary format - const billSummary: BillSummary = { - billID: dbBill.billId, - title: dbBill.title, - shortTitle: dbBill.short_title, - stages: dbBill.stages || [], - description: dbBill.summary || "", - status: (dbBill.status as BillSummary["status"]) || "Introduced", - sponsorParty: dbBill.sponsorParty || "Unknown", - sponsorName: "Unknown", - chamber: - (dbBill.chamber as "House of Commons" | "Senate") || - "House of Commons", - introducedOn: - toIsoString(dbBill.introducedOn) || new Date().toISOString(), - lastUpdatedOn: - toIsoString(dbBill.lastUpdatedOn) || new Date().toISOString(), - summary: dbBill.summary, - isSocialIssue: dbBill.isSocialIssue, - final_judgment: dbBill.final_judgment as BillSummary["final_judgment"], - rationale: dbBill.rationale, - needs_more_info: dbBill.needs_more_info, - missing_details: dbBill.missing_details, - genres: dbBill.genres, - parliamentNumber: dbBill.parliamentNumber, - sessionNumber: dbBill.sessionNumber, - }; - mergedBills.push(billSummary); - } - } - - return mergedBills; + return mergeBillLists(apiBills, dbBills.map(fromBuildCanadaDbBill)); } function clearMergedBillsCache(): void { diff --git a/src/app/bills/prompt/analysis-schema.test.ts b/src/app/bills/prompt/analysis-schema.test.ts new file mode 100644 index 00000000..83799508 --- /dev/null +++ b/src/app/bills/prompt/analysis-schema.test.ts @@ -0,0 +1,120 @@ +import assert from "node:assert/strict"; +import { test } from "node:test"; +import { BILL_ANALYSIS_SCHEMA } from "./analysis-schema"; + +// The schema is `as const`, so every member is readonly; the walker below wants +// a plain shape to read. +const SCHEMA = BILL_ANALYSIS_SCHEMA as unknown as Node; + +type Node = { + type?: string; + properties?: Record; + required?: string[]; + additionalProperties?: boolean; + items?: Node; + enum?: unknown[]; +}; + +/** + * OpenAI rejects a malformed `strict: true` schema at request time, and + * summarizeBillText turns that rejection into a fallback `abstain`. A silent + * downgrade of every analysis is worth a test that costs nothing. + */ +function walkObjects(node: Node, path: string, visit: (n: Node, p: string) => void) { + if (node.type === "object") { + visit(node, path); + for (const [key, child] of Object.entries(node.properties ?? {})) { + walkObjects(child, `${path}.${key}`, visit); + } + } + if (node.type === "array" && node.items) { + walkObjects(node.items, `${path}[]`, visit); + } +} + +test("every object sets additionalProperties: false", () => { + walkObjects(SCHEMA, "root", (node, path) => { + assert.equal( + node.additionalProperties, + false, + `${path} must set additionalProperties: false`, + ); + }); +}); + +test("every object lists all of its properties as required", () => { + walkObjects(SCHEMA, "root", (node, path) => { + const properties = Object.keys(node.properties ?? {}).sort(); + const required = [...(node.required ?? [])].sort(); + assert.deepEqual( + required, + properties, + `${path}: strict mode requires every property in \`required\``, + ); + }); +}); + +test("the schema asks for the fields the analysis reads", () => { + const properties = Object.keys( + (SCHEMA).properties ?? {}, + ); + // These three were read by the parser but never requested, so they came back + // empty on every bill ever analyzed. + for (const field of ["steel_man", "needs_more_info", "missing_details"]) { + assert.ok(properties.includes(field), `schema must request ${field}`); + } + // Answered here rather than by a second call that disagreed with this one. + assert.ok(properties.includes("is_social_issue")); +}); + +test("judgment and alignment are closed enums", () => { + const root = SCHEMA; + assert.deepEqual(root.properties?.final_judgment.enum, [ + "yes", + "no", + "abstain", + ]); + assert.deepEqual( + root.properties?.tenet_evaluations.items?.properties?.alignment.enum, + ["aligns", "conflicts", "neutral"], + ); +}); + +test("the tenet ids are a closed set of eight", () => { + const tenets = SCHEMA.properties?.tenet_evaluations; + assert.deepEqual(tenets?.items?.properties?.id.enum, [1, 2, 3, 4, 5, 6, 7, 8]); +}); + +test("no keyword that strict mode rejects", () => { + // OpenAI 400s a strict schema carrying any of these, and summarizeBillText + // turns a 400 into a fallback `abstain` — so every bill would silently lose + // its analysis. Array counts belong in `description` and the eval checks. + const banned = [ + "minItems", + "maxItems", + "uniqueItems", + "contains", + "minContains", + "maxContains", + "unevaluatedItems", + "minLength", + "maxLength", + "pattern", + "format", + "minimum", + "maximum", + "multipleOf", + "patternProperties", + "unevaluatedProperties", + "propertyNames", + "minProperties", + "maxProperties", + ]; + const serialized = JSON.stringify(BILL_ANALYSIS_SCHEMA); + for (const keyword of banned) { + assert.ok( + !serialized.includes(`"${keyword}"`), + `schema uses ${keyword}, which strict mode rejects`, + ); + } +}); diff --git a/src/app/bills/prompt/analysis-schema.ts b/src/app/bills/prompt/analysis-schema.ts new file mode 100644 index 00000000..accf5c11 --- /dev/null +++ b/src/app/bills/prompt/analysis-schema.ts @@ -0,0 +1,135 @@ +import { TENETS } from "@/app/bills/prompt/summary-and-vote-prompt"; + +/** + * The exact shape `summarizeBillText` expects back from the model. + * + * This is sent as an OpenAI Structured Output (`strict: true`), which makes the + * shape a guarantee rather than a request: the enums cannot come back + * mis-cased, `tenet_evaluations` cannot come back short, and the response + * cannot come back fenced in markdown. That replaces the hand-written "Output + * format" block the prompt used to carry — which showed the model invalid JSON + * (unescaped nested quotes, a trailing comma) and cost a `JSON.parse` failure + * and a permanent `abstain` whenever the model imitated it too closely. + * + * `strict: true` requires every property to appear in `required` and every + * object to set `additionalProperties: false`. Optionality is expressed by + * allowing `null`, not by omitting from `required`. + * + * Strict mode also rejects a number of JSON Schema keywords outright, array + * length among them (`minItems` / `maxItems`). A schema carrying one is a 400 + * on every request, which `summarizeBillText` would turn into a fallback + * `abstain` for every bill — so array counts are stated in `description` and + * asserted afterwards by the eval checks, never in the schema. + */ + +const TENET_IDS = Object.keys(TENETS).map(Number); + +export const BILL_ANALYSIS_SCHEMA = { + type: "object", + additionalProperties: false, + required: [ + "summary", + "short_title", + "tenet_evaluations", + "final_judgment", + "rationale", + "steel_man", + "needs_more_info", + "missing_details", + "question_period_questions", + "is_social_issue", + ], + properties: { + summary: { + type: "string", + description: + "3-5 sentences in plain language followed by markdown bullet points covering the highlights. No other text.", + }, + short_title: { + type: "string", + description: "A 1-2 word title for the bill.", + }, + tenet_evaluations: { + type: "array", + description: `Exactly ${TENET_IDS.length} entries, one per tenet, in ascending id order (${TENET_IDS.join(", ")}). Do not repeat or omit an id.`, + items: { + type: "object", + additionalProperties: false, + required: ["id", "alignment", "explanation"], + properties: { + id: { type: "integer", enum: TENET_IDS }, + alignment: { type: "string", enum: ["aligns", "conflicts", "neutral"] }, + explanation: { + type: "string", + description: + "A short explanation of how this bill relates to this tenet. Do not quote the tenet back.", + }, + }, + }, + }, + final_judgment: { + type: "string", + enum: ["yes", "no", "abstain"], + }, + rationale: { + type: "string", + description: + "2 sentences explaining the overall judgment, then markdown bullet points giving the reasoning and what might be changed.", + }, + steel_man: { + type: "string", + description: + "The strongest good-faith case for the side the judgment did NOT take, in 2-4 sentences. If the judgment is 'yes', argue why a builder might still oppose it; if 'no', argue why a builder might still support it; if 'abstain', give the strongest case that this bill does carry economic weight. Address the bill's actual provisions, not generalities. Never concede the judgment.", + }, + needs_more_info: { + type: "boolean", + description: + "True when the bill text is too thin or too technical to judge confidently.", + }, + missing_details: { + type: "array", + items: { type: "string" }, + description: + "What would be needed to judge confidently. Empty when needs_more_info is false.", + }, + question_period_questions: { + type: "array", + description: "Exactly 3 questions.", + items: { + type: "object", + additionalProperties: false, + required: ["question"], + properties: { + question: { + type: "string", + description: + "A critical question about this bill only, phrased as a Member of Parliament would actually ask it in Question Period. Omit any 'Mr. Speaker' / 'Madam Speaker' prefix.", + }, + }, + }, + }, + is_social_issue: { + type: "boolean", + description: + "True when the bill is primarily a social issue per the criteria above.", + }, + }, +} as const; + +/** The model's reply, before tenet titles are filled in from TENETS. */ +export type RawBillAnalysis = { + summary: string; + short_title: string; + tenet_evaluations: Array<{ + id: number; + alignment: "aligns" | "conflicts" | "neutral"; + explanation: string; + }>; + final_judgment: "yes" | "no" | "abstain"; + rationale: string; + steel_man: string; + needs_more_info: boolean; + missing_details: string[]; + question_period_questions: Array<{ question: string }>; + is_social_issue: boolean; +}; diff --git a/src/app/bills/prompt/summary-and-vote-prompt.ts b/src/app/bills/prompt/summary-and-vote-prompt.ts index 8a5353de..e67a81e9 100644 --- a/src/app/bills/prompt/summary-and-vote-prompt.ts +++ b/src/app/bills/prompt/summary-and-vote-prompt.ts @@ -82,7 +82,7 @@ You are analyzing Canadian legislation. You must assess whether the bill aligns - Never advocate for adding more red tape. - Always advocate for safety and security for Canadians. - Never self reference Build Canada, or use "We" or "Our", use the idea of "Builders" instead. - - Never self reference the tenents outside of the tenet evaluations. + - Never self reference the tenets outside of the tenet evaluations. ## Your Task @@ -97,84 +97,20 @@ You are analyzing Canadian legislation. You must assess whether the bill aligns 4.2 Output “yes” if the bill aligns overall with Build Canada's tenets. 4.3 Output “no” if it conflicts overall with Build Canada's tenets. 5. Generate 3 critical questions, pertaining to this and only about this bill, for Question Period in the House of Commons phrased in a way that a Member of Parliament might actually ask in Question Period. Omit any prefix like "Mr. Speaker" or "Madam Speaker". - - Important: All enum values must be lowercase exactly as specified. - - tenet_evaluations.alignment: aligns|conflicts|neutral - - final_judgment: yes|no|abstain - - is_social_issue: yes|no - - Never mention the tenents in the summary, questions, or rationale. - - Output format (return valid JSON only): - - \`\`\`json - { - "summary": "Your 3-5 sentence summary here in plain language. Use bullet points to summarize the highlights of the bill. Do not include any other text in the summary. Use markdown formatting.", - "short_title": "A short title for the bill. Use 1-2 words to describe the bill.", - "tenet_evaluations": [ - { - "id": 1, - "title": "${TENETS[1]}", - "alignment": "aligns|conflicts|neutral", - "explanation": "Short explanation of how this bill relates to this tenet" - }, - { - "id": 2, - "title": "${TENETS[2]}", - "alignment": "aligns|conflicts|neutral", - "explanation": "Short explanation of how this bill relates to this tenet" - }, - { - "id": 3, - "title": "${TENETS[3]}", - "alignment": "aligns|conflicts|neutral", - "explanation": "Short explanation of how this bill relates to this tenet" - }, - { - "id": 4, - "title": "${TENETS[4]}", - "alignment": "aligns|conflicts|neutral", - "explanation": "Short explanation of how this bill relates to this tenet" - }, - { - "id": 5, - "title": "${TENETS[5]}", - "alignment": "aligns|conflicts|neutral", - "explanation": "Short explanation of how this bill relates to this tenet" - }, - { - "id": 6, - "title": "${TENETS[6]}", - "alignment": "aligns|conflicts|neutral", - "explanation": "Short explanation of how this bill relates to this tenet" - }, - { - "id": 7, - "title": "${TENETS[7]}", - "alignment": "aligns|conflicts|neutral", - "explanation": "Short explanation of how this bill relates to this tenet" - }, - { - "id": 8, - "title": "${TENETS[8]}", - "alignment": "aligns|conflicts|neutral", - "explanation": "Short explanation of how this bill relates to this tenet" - } - ], - "question_period_questions": [ - { - "question": "A crticial question, pertaining to this and only about this bill, for Question Period in the House of Commons phrased in a way that a Member of Parliament might actually ask in Question Period. Omit any prefix like "Mr. Speaker" or "Madam Speaker"" - }, - { - "question": "A crticial question, pertaining to this and only about this bill, for Question Period in the House of Commons phrased in a way that a Member of Parliament might actually ask in Question Period. Omit any prefix like "Mr. Speaker" or "Madam Speaker"" - }, - { - "question": "A crticial question, pertaining to this and only about this bill, for Question Period in the House of Commons phrased in a way that a Member of Parliament might actually ask in Question Period. Omit any prefix like "Mr. Speaker" or "Madam Speaker"" - }, - - ], - "final_judgment": "yes|no|abstain", - "rationale": "2 sentences explaining the overall judgment and then bullet points explaining the rationale for the judgment and suggestions for what we might change. Use markdown formatting.", - "is_social_issue": "yes|no" - } - \`\`\` + 6. Write a steel man: the strongest good-faith case for the side you did NOT + take. If the judgment is "yes", argue why a builder might still oppose the + bill; if "no", why a builder might still support it; if "abstain", the + strongest case that the bill does carry real economic weight. Address the + bill's actual provisions, not generalities, and never concede the judgment. + 7. Set is_social_issue per the social-issue criteria above. A bill that is + primarily a social issue must also take "abstain" as its final judgment. + 8. Set needs_more_info when the bill text is too thin or too technical to + judge confidently, and list what is missing in missing_details. + + The response shape is enforced for you. Do not describe it, do not wrap it in + markdown, and do not add commentary around it — spend your effort on the + judgment, not the formatting. + + - Return exactly one entry per tenet, in ascending id order. + - Never mention the tenets in the summary, questions, rationale, or steel man. `; diff --git a/src/app/bills/server/get-api-bills.ts b/src/app/bills/server/get-api-bills.ts new file mode 100644 index 00000000..a0e620b7 --- /dev/null +++ b/src/app/bills/server/get-api-bills.ts @@ -0,0 +1,44 @@ +import { env } from "@/app/bills/env"; +import { CANADIAN_PARLIAMENT_NUMBER } from "@/app/bills/consts/general"; +import type { BillSummary } from "@/app/bills/types"; + +/** + * The Civics Project list of every bill in the current Parliament. + * + * Shared by the /bills list page and the refresh sweep so both see the same + * bills. Never throws — an outage yields an empty list, and the page falls back + * to whatever is stored. + * + * Unlike its siblings in `server/`, this carries no `server-only` guard: it + * reads a public API rather than the database, and the guard would make the + * refresh sweep impossible to run outside a Next request (`pnpm sweep:bills`). + */ +export async function getApiBills(): Promise { + try { + const response = await fetch( + `${env.CIVICS_PROJECT_BASE_URL}/canada/bills/${CANADIAN_PARLIAMENT_NUMBER}`, + { + // Cache for 5 minutes in production, no cache in development + ...(process.env.NODE_ENV === "production" + ? { next: { revalidate: 300 } } + : { cache: "no-store" }), + headers: { + "Content-Type": "application/json", + Authorization: env.CIVICS_PROJECT_API_KEY + ? `Bearer ${env.CIVICS_PROJECT_API_KEY}` + : "", + }, + }, + ); + if (!response.ok) { + throw new Error( + `Failed to fetch bills from API: ${response.status} ${response.statusText}`, + ); + } + const { data } = await response.json(); + return Array.isArray(data) ? (data as BillSummary[]) : (data?.bills ?? []); + } catch (error) { + console.error("Error fetching API bills:", error); + return []; + } +} diff --git a/src/app/bills/services/billApi.ts b/src/app/bills/services/billApi.ts index 6ce7d64a..83d9debd 100644 --- a/src/app/bills/services/billApi.ts +++ b/src/app/bills/services/billApi.ts @@ -1,9 +1,18 @@ import { xmlToMarkdown } from "@/app/bills/utils/xml-to-md/xml-to-md.util"; -import { SUMMARY_AND_VOTE_PROMPT } from "@/app/bills/prompt/summary-and-vote-prompt"; +import { + SUMMARY_AND_VOTE_PROMPT, + TENETS, +} from "@/app/bills/prompt/summary-and-vote-prompt"; +import { + BILL_ANALYSIS_SCHEMA, + type RawBillAnalysis, +} from "@/app/bills/prompt/analysis-schema"; import OpenAI from "openai"; -import { BILL_API_REVALIDATE_INTERVAL } from "@/app/bills/consts/general"; +import { + BILL_API_REVALIDATE_INTERVAL, + CANADIAN_PARLIAMENT_NUMBER, +} from "@/app/bills/consts/general"; import { env } from "@/app/bills/env"; -import type { BillDocument } from "@/app/bills/models/Bill"; export type ApiStage = { stage: string; @@ -38,28 +47,26 @@ export type ApiBillDetail = { billTexts?: unknown[]; }; -const CANADIAN_PARLIAMENT_NUMBER = 45; - -const FALLBACK_TENET_TITLES = [ - "Canada should aim to be the world's most prosperous country", - "Promote economic freedom, ambition, and breaking from bureaucratic inertia", - "Drive national productivity and global competitiveness", - "Grow exports of Canadian products and resources", - "Encourage investment, innovation, and resource development", - "Deliver better public services at lower cost (government efficiency)", - "Reform taxes to incentivize work, risk-taking, and innovation", - "Focus on large-scale prosperity, not incrementalism", -]; +/** + * Tenet titles are owned by TENETS and filled in here by id, rather than being + * asked of the model. One source of truth instead of three, and the model + * spends no tokens echoing text we already have. + */ +export function tenetTitle(id: number): string { + return TENETS[id as keyof typeof TENETS] ?? `Tenet ${id}`; +} function makeFallbackTenets( explanation: string, ): BillAnalysis["tenet_evaluations"] { - return FALLBACK_TENET_TITLES.map((title, index) => ({ - id: index + 1, - title, - alignment: "neutral", - explanation, - })); + return Object.keys(TENETS) + .map(Number) + .map((id) => ({ + id, + title: tenetTitle(id), + alignment: "neutral" as const, + explanation, + })); } /** Types for AI analysis results */ @@ -78,6 +85,12 @@ export interface BillAnalysis { missing_details: string[]; steel_man: string; question_period_questions?: Array<{ question: string }>; + /** + * Whether the bill is primarily a social issue. Answered by the same call + * that produces the judgment — a separate grader used to answer it again + * from the first 8000 characters and disagree. + */ + isSocialIssue: boolean; // Set when this is a degraded placeholder (no OpenAI key, parse failure, // API error, rate cap) rather than a real analysis. Callers must not // persist or Slack-notify a fallback. Never written to Mongo. @@ -95,7 +108,7 @@ export async function getBillFromCivicsProjectApi( : { cache: "no-store" }), headers: { "Content-Type": "application/json", - Authorization: `Bearer ${process.env.CIVICS_PROJECT_API_KEY}`, + Authorization: `Bearer ${env.CIVICS_PROJECT_API_KEY ?? ""}`, }, }); if (!response.ok) { @@ -148,43 +161,18 @@ export async function summarizeBillText( ): Promise { if (!process.env.OPENAI_API_KEY) { console.log("No OPENAI API key, using fallback analysis"); - // Fallback analysis - const text = input?.trim() || ""; - const truncatedSummary = - text.length <= 500 ? text : `${text.slice(0, 500)}…`; - - return { - summary: truncatedSummary || "No bill text available for analysis.", - short_title: undefined, - tenet_evaluations: makeFallbackTenets("Unable to analyze without AI"), - final_judgment: "abstain", - rationale: undefined, - needs_more_info: true, - missing_details: ["AI analysis capabilities required"], - steel_man: - "The steel man for this bill is the bill that aligns with the tenets of Build Canada.", - question_period_questions: [], - isFallback: true, - }; + return degradedAnalysis(input, "Unable to analyze without AI", [ + "AI analysis capabilities required", + ]); } if (!options?.bypassCap && summarizeCapExceeded()) { console.error( `[LLM_SUMMARIZE_CAP] exceeded ${SUMMARIZE_CAP_PER_HOUR} calls/hour — refusing OpenAI call`, ); - const text = input?.trim() || ""; - return { - summary: text.length <= 500 ? text : `${text.slice(0, 500)}…`, - short_title: undefined, - tenet_evaluations: makeFallbackTenets("Analysis rate cap exceeded"), - final_judgment: "abstain", - rationale: undefined, - needs_more_info: true, - missing_details: ["Analysis rate cap exceeded"], - steel_man: "Analysis rate cap exceeded", - question_period_questions: [], - isFallback: true, - }; + return degradedAnalysis(input, "Analysis rate cap exceeded", [ + "Analysis rate cap exceeded", + ]); } try { @@ -198,79 +186,101 @@ export async function summarizeBillText( reasoning: { effort: "high", }, + // Structured Outputs: the shape below is guaranteed, so there is no + // markdown fence to strip, no enum to re-case, and no parse fallback. + text: { + format: { + type: "json_schema", + name: "bill_analysis", + strict: true, + schema: BILL_ANALYSIS_SCHEMA as unknown as Record, + }, + }, }); - const responseText = response.output_text; - // Parse JSON response - try { - const parsed = JSON.parse(responseText); - const rawFj = String(parsed.final_judgment || "") - .trim() - .toLowerCase(); - const normalizedFj: "yes" | "no" | "abstain" = - rawFj === "yes" || rawFj === "no" || rawFj === "abstain" - ? rawFj - : "abstain"; - const analysis: BillAnalysis = { - summary: parsed.summary ?? "", - short_title: parsed.short_title ?? undefined, - tenet_evaluations: parsed.tenet_evaluations ?? [], - final_judgment: normalizedFj, - rationale: parsed.rationale ?? undefined, - needs_more_info: parsed.needs_more_info ?? false, - missing_details: parsed.missing_details ?? [], - steel_man: parsed.steel_man ?? "", - question_period_questions: Array.isArray( - parsed.question_period_questions, - ) - ? parsed.question_period_questions - : [], - }; - return analysis; - } catch (parseError) { - console.error("Failed to parse AI response as JSON:", parseError); - - // Fallback to extracting summary from text response - const summaryMatch = responseText.match( - /summary['":\s]*["']([^"']+)["']/i, - ); - const summary = summaryMatch - ? summaryMatch[1] - : `${responseText.slice(0, 500)}…`; - - return { - summary, - short_title: undefined, - tenet_evaluations: makeFallbackTenets("JSON parse failed"), - final_judgment: "abstain", - rationale: undefined, - needs_more_info: true, - missing_details: ["Valid AI response format"], - steel_man: "Analysis parsing failed", - question_period_questions: [], - isFallback: true, - }; + // A refusal or an incomplete response leaves output_text empty; treat that + // as a degraded result rather than parsing "" and throwing. + const responseText = response.output_text; + if (!responseText) { + console.error("[LLM_SUMMARIZE] empty response", { + status: response.status, + incomplete: response.incomplete_details, + }); + return degradedAnalysis(input, "Model returned no analysis", [ + "A complete model response", + ]); } + + const parsed = JSON.parse(responseText) as RawBillAnalysis; + return fromRawAnalysis(parsed); } catch (error) { console.error("Error analyzing bill:", error); - // Fallback analysis - const text = input?.trim() || ""; - const truncatedSummary = - text.length <= 500 ? text : `${text.slice(0, 500)}…`; + return degradedAnalysis(input, "Analysis failed", [ + "Technical issue resolution", + ]); + } +} - return { - summary: truncatedSummary || "Error occurred during analysis.", - short_title: undefined, - tenet_evaluations: makeFallbackTenets("Analysis failed"), - final_judgment: "abstain", - rationale: "Technical error during analysis", - needs_more_info: true, - missing_details: ["Technical issue resolution"], - steel_man: "Technical error during analysis", - question_period_questions: [], - isFallback: true, - }; +/** + * Fill in tenet titles from TENETS and apply the one rule the prompt states but + * cannot enforce on itself: a bill that is primarily a social issue abstains. + * Builder MP takes no position on social questions, so a "yes"/"no" alongside + * is_social_issue would contradict the product. `migrations/1.ts` existed to + * repair exactly this drift after the fact. + */ +export function fromRawAnalysis(raw: RawBillAnalysis): BillAnalysis { + const judgment = raw.is_social_issue ? "abstain" : raw.final_judgment; + if (raw.is_social_issue && raw.final_judgment !== "abstain") { + console.warn( + `[LLM_SUMMARIZE] social issue judged "${raw.final_judgment}" — forcing abstain`, + ); } + + return { + summary: raw.summary, + short_title: raw.short_title || undefined, + tenet_evaluations: raw.tenet_evaluations.map((t) => ({ + id: t.id, + title: tenetTitle(t.id), + alignment: t.alignment, + explanation: t.explanation, + })), + final_judgment: judgment, + rationale: raw.rationale || undefined, + needs_more_info: raw.needs_more_info, + missing_details: raw.missing_details ?? [], + steel_man: raw.steel_man, + question_period_questions: raw.question_period_questions ?? [], + isSocialIssue: raw.is_social_issue, + }; +} + +/** + * A placeholder for every path where no real analysis was produced. Always + * `isFallback: true`, which is what stops callers persisting it over good data + * or announcing it in Slack. + */ +function degradedAnalysis( + input: string, + reason: string, + missingDetails: string[], +): BillAnalysis { + const text = input?.trim() || ""; + return { + summary: + (text.length <= 500 ? text : `${text.slice(0, 500)}…`) || + "No bill text available for analysis.", + short_title: undefined, + tenet_evaluations: makeFallbackTenets(reason), + final_judgment: "abstain", + rationale: undefined, + needs_more_info: true, + missing_details: missingDetails, + steel_man: reason, + question_period_questions: [], + isSocialIssue: false, + isFallback: true, + }; } export async function fetchBillMarkdown( @@ -293,153 +303,139 @@ export async function fetchBillMarkdown( return null; } -export async function onBillNotInDatabase(params: { - billId: string; - source?: string; - markdown?: string | null; - bill: ApiBillDetail; - analysis: BillAnalysis; - billTextsCount: number; - isSocialIssue: boolean; -}): Promise { - console.log("Saving bill to database:", params.billId); - - // Import here to avoid circular dependencies +/** Mongo is reachable and configured. */ +async function connectIfConfigured(): Promise { + const uri = env.MONGO_URI || ""; + if (!uri.startsWith("mongodb://") && !uri.startsWith("mongodb+srv://")) { + console.warn("[bills] No valid MongoDB URI, skipping write"); + return false; + } + // Imported here to avoid a circular dependency through the models. const { connectToDatabase } = await import("@/app/bills/lib/mongoose"); - const { Bill } = await import("@/app/bills/models/Bill"); - - try { - const { env } = await import("@/app/bills/env"); - const uri = env.MONGO_URI || ""; - const hasValidMongoUri = - uri.startsWith("mongodb://") || uri.startsWith("mongodb+srv://"); - - if (!hasValidMongoUri) { - console.warn("No valid MongoDB URI, skipping bill save"); - return; - } + await connectToDatabase(); + return true; +} - await connectToDatabase(); +/** The fields that come from the Civics Project API rather than the model. */ +function factsFromApiBill(bill: ApiBillDetail, source?: string) { + const latestStageDate = + bill.stages && bill.stages.length > 0 + ? bill.stages[bill.stages.length - 1].date + : (bill.updatedAt ?? bill.date); - // Check if bill already exists and if we need to update it - const existing = (await Bill.findOne({ billId: params.billId }) - .lean() - .exec()) as BillDocument | null; - if (existing) { - const countChanged = existing.billTextsCount !== params.billTextsCount; - const sourceChanged = - (existing.source || null) !== (params.source || null); - const existingQP = Array.isArray(existing.question_period_questions) - ? existing.question_period_questions - : []; - const newQP = Array.isArray(params.analysis.question_period_questions) - ? params.analysis.question_period_questions - : []; - const qpMissingOrDifferent = - (existingQP.length === 0 && newQP.length > 0) || - JSON.stringify(existingQP) !== JSON.stringify(newQP); - const shortTitleMissing = - !existing.short_title && - (params.bill.shortTitle || params.analysis.short_title); + return { + title: bill.title, + status: bill.status, + sponsorParty: bill.sponsorParty, + genres: bill.genres, + supportedRegion: bill.supportedRegion, + stages: bill.stages?.map((stage) => ({ + stage: stage.stage, + state: stage.state, + house: stage.house, + date: new Date(stage.date), + })), + billTextsCount: Array.isArray(bill.billTexts) ? bill.billTexts.length : 0, + source: source ?? bill.source, + lastUpdatedOn: new Date(latestStageDate), + }; +} - if ( - sourceChanged || - countChanged || - qpMissingOrDifferent || - shortTitleMissing - ) { - if (sourceChanged) { - console.log( - `Updating bill ${params.billId} - source changed from ${existing.source || ""} to ${params.source || ""}`, - ); - } else if (countChanged) { - console.log( - `Updating bill ${params.billId} - billTexts count changed from ${existing.billTextsCount} to ${params.billTextsCount}`, - ); - } else if (qpMissingOrDifferent) { - console.log( - `Updating bill ${params.billId} - adding/updating Question Period questions (${existingQP.length} -> ${newQP.length})`, - ); - } else if (shortTitleMissing) { - console.log( - `Updating bill ${params.billId} - adding missing short_title`, - ); - } +/** + * Bring a stored bill's factual fields up to date with the API — status, + * stages, sponsor, genres, source. No LLM call, so this is cheap enough to run + * over every bill on every sweep. + * + * Never upserts: a row with no analysis would read as "found" to the detail + * page. New bills are created by `saveBillAnalysis` once they have one. + * + * Returns true when a document was written. The schema sets `timestamps: true`, + * so `updatedAt` changes on every call and this is "written", not "changed". + */ +export async function updateBillFacts( + bill: ApiBillDetail, + source?: string, +): Promise { + if (!(await connectIfConfigured())) return false; + const { Bill } = await import("@/app/bills/models/Bill"); - await Bill.updateOne( - { billId: params.billId }, - { - title: params.bill.title, - short_title: params.bill.shortTitle || params.analysis.short_title, - summary: params.analysis.summary, - tenet_evaluations: params.analysis.tenet_evaluations, - final_judgment: params.analysis.final_judgment, - rationale: params.analysis.rationale, - needs_more_info: params.analysis.needs_more_info, - missing_details: params.analysis.missing_details, - steel_man: params.analysis.steel_man, - status: params.bill.status, - sponsorParty: params.bill.sponsorParty, - genres: params.bill.genres, - billTextsCount: params.billTextsCount, - lastUpdatedOn: new Date(), - isSocialIssue: params.isSocialIssue, - question_period_questions: newQP, - source: params.source, - }, - ); - } - return; - } + try { + const result = await Bill.updateOne( + { billId: bill.billID }, + { $set: factsFromApiBill(bill, source) }, + { upsert: false }, + ); + return result.modifiedCount > 0; + } catch (error) { + console.error(`[bills] Failed to update facts for ${bill.billID}:`, error); + return false; + } +} - // Convert API bill to DB format - const latestStageDate = - params.bill.stages && params.bill.stages.length > 0 - ? params.bill.stages[params.bill.stages.length - 1].date - : (params.bill.updatedAt ?? params.bill.date); +/** + * Write a freshly generated analysis, creating the bill if it is new. + * + * The decision of *whether* to spend an OpenAI call lives in the caller (the + * refresh sweep, or an admin pressing reprocess); by the time we are here the + * analysis has been paid for, so it is always written. An earlier version + * re-derived that decision here and silently dropped writes whose source had + * not changed. + */ +export async function saveBillAnalysis(params: { + bill: ApiBillDetail; + analysis: BillAnalysis; + source?: string; +}): Promise { + const { bill, analysis, source } = params; - const classifiedIsSocialIssue = params.isSocialIssue; + if (analysis.isFallback) { + // A degraded placeholder must never overwrite real stored data, nor create + // a row that then looks analyzed. + console.warn( + `[bills] Refusing to persist fallback analysis for ${bill.billID}`, + ); + return; + } - const billData = { - billId: params.bill.billID, - parliamentNumber: params.bill.parliamentNumber, - sessionNumber: params.bill.sessionNumber, - title: params.bill.title, - short_title: params.bill.shortTitle || params.analysis.short_title, - summary: params.analysis.summary, - tenet_evaluations: params.analysis.tenet_evaluations, - final_judgment: params.analysis.final_judgment, - rationale: params.analysis.rationale, - needs_more_info: params.analysis.needs_more_info, - missing_details: params.analysis.missing_details, - steel_man: params.analysis.steel_man, - status: params.bill.status, - sponsorParty: params.bill.sponsorParty, - chamber: params.bill.billID.startsWith("S") - ? "Senate" - : "House of Commons", - genres: params.bill.genres, - supportedRegion: params.bill.supportedRegion, - introducedOn: new Date(params.bill.date), - lastUpdatedOn: new Date(latestStageDate), - source: params.source || params.bill.source, - stages: params.bill.stages?.map((stage) => ({ - stage: stage.stage, - state: stage.state, - house: stage.house, - date: new Date(stage.date), - })), - votes: [], // API doesn't provide detailed vote records - billTextsCount: params.billTextsCount, - isSocialIssue: classifiedIsSocialIssue, - question_period_questions: - params.analysis.question_period_questions ?? [], - }; + if (!(await connectIfConfigured())) return; + const { Bill } = await import("@/app/bills/models/Bill"); - await Bill.create(billData); - console.log("Successfully saved bill to database:", params.billId); + try { + const facts = factsFromApiBill(bill, source); + await Bill.updateOne( + { billId: bill.billID }, + { + $set: { + ...facts, + short_title: bill.shortTitle || analysis.short_title, + summary: analysis.summary, + tenet_evaluations: analysis.tenet_evaluations, + final_judgment: analysis.final_judgment, + rationale: analysis.rationale, + needs_more_info: analysis.needs_more_info, + missing_details: analysis.missing_details, + steel_man: analysis.steel_man, + isSocialIssue: analysis.isSocialIssue, + question_period_questions: analysis.question_period_questions ?? [], + // Provenance: which text this verdict was computed from, and when. + // Surfaced to readers on the bill page. + analysisGeneratedAt: new Date(), + analysisSourceRef: facts.source, + }, + $setOnInsert: { + billId: bill.billID, + parliamentNumber: bill.parliamentNumber, + sessionNumber: bill.sessionNumber, + chamber: bill.billID.startsWith("S") ? "Senate" : "House of Commons", + introducedOn: new Date(bill.date), + votes: [], + }, + }, + { upsert: true }, + ); + console.log(`[bills] Saved analysis for ${bill.billID}`); } catch (error) { - console.error("Error saving bill to database:", error); - // Don't throw - this shouldn't break the page if DB save fails + // Never throw — a failed write must not break the caller's sweep. + console.error(`[bills] Error saving ${bill.billID} to database:`, error); } } diff --git a/src/app/bills/services/refresh-decision.test.ts b/src/app/bills/services/refresh-decision.test.ts new file mode 100644 index 00000000..98da5ac3 --- /dev/null +++ b/src/app/bills/services/refresh-decision.test.ts @@ -0,0 +1,52 @@ +import assert from "node:assert/strict"; +import { test } from "node:test"; +import { analysisReason } from "./refresh-decision"; + +const ANALYSED = new Date("2026-03-01T00:00:00Z"); + +test("a bill we have never seen is analyzed", () => { + assert.equal(analysisReason(undefined, "a.xml", 1, false), "new"); +}); + +test("a stored bill with no analysis is analyzed", () => { + assert.equal( + analysisReason({ source: "a.xml", billTextsCount: 1, hasAnalysis: false }, "a.xml", 1, false), + "no-analysis", + ); +}); + +test("a new bill text means the verdict is re-run", () => { + const stored = { source: "first-reading.xml", billTextsCount: 1, hasAnalysis: true, analysedAt: ANALYSED }; + assert.equal( + analysisReason(stored, "third-reading.xml", 1, false), + "source-changed", + ); +}); + +test("an extra bill text means the verdict is re-run", () => { + const stored = { source: "a.xml", billTextsCount: 1, hasAnalysis: true, analysedAt: ANALYSED }; + assert.equal(analysisReason(stored, "a.xml", 2, false), "count-changed"); +}); + +test("an unchanged bill costs no OpenAI call", () => { + const stored = { source: "a.xml", billTextsCount: 1, hasAnalysis: true, analysedAt: ANALYSED }; + assert.equal(analysisReason(stored, "a.xml", 1, false), null); +}); + +test("force re-runs an unchanged bill", () => { + const stored = { source: "a.xml", billTextsCount: 1, hasAnalysis: true, analysedAt: ANALYSED }; + assert.equal(analysisReason(stored, "a.xml", 1, true), "forced"); +}); + +test("a bill that has never had a source stays unchanged", () => { + const stored = { source: undefined, billTextsCount: 0, hasAnalysis: true, analysedAt: ANALYSED }; + assert.equal(analysisReason(stored, undefined, 0, false), null); +}); + +test("a legacy row with an analysis but no timestamp is not re-analyzed", () => { + // Rows predating `analysisGeneratedAt` have a verdict but no date. Treating + // them as unanalyzed would re-pay for every bill in the database at once. + const legacy = { source: "a.xml", billTextsCount: 1, hasAnalysis: true }; + assert.equal(analysisReason(legacy, "a.xml", 1, false), null); + assert.equal(analysisReason(legacy, "b.xml", 1, false), "source-changed"); +}); diff --git a/src/app/bills/services/refresh-decision.ts b/src/app/bills/services/refresh-decision.ts new file mode 100644 index 00000000..3852c4f1 --- /dev/null +++ b/src/app/bills/services/refresh-decision.ts @@ -0,0 +1,44 @@ +/** + * When a bill is worth spending an OpenAI call on. + * + * Kept apart from the sweep itself so it carries no server-only imports and can + * be tested directly — this decision is what stands between the refresh job and + * an unbounded OpenAI bill, and it is the logic that was previously unreachable + * (it lived inside a code path only entered for bills we had never stored, so + * "source-changed" could never be true). + */ + +/** What the sweep needs to know about a bill it already stores. */ +export type StoredState = { + source?: string; + billTextsCount?: number; + /** + * Whether a verdict is stored at all, independent of when it was generated. + * Rows written before `analysisGeneratedAt` existed have an analysis but no + * timestamp; treating those as unanalyzed would re-pay for every bill in the + * database on the first sweep. + */ + hasAnalysis: boolean; + analysedAt?: Date; +}; + +export type AnalysisReason = + | "new" + | "no-analysis" + | "source-changed" + | "count-changed" + | "forced"; + +export function analysisReason( + stored: StoredState | undefined, + apiSource: string | undefined, + apiBillTextsCount: number, + force: boolean, +): AnalysisReason | null { + if (!stored) return "new"; + if (!stored.hasAnalysis) return "no-analysis"; + if ((stored.source || null) !== (apiSource || null)) return "source-changed"; + if ((stored.billTextsCount ?? 0) !== apiBillTextsCount) return "count-changed"; + if (force) return "forced"; + return null; +} diff --git a/src/app/bills/services/refresh.ts b/src/app/bills/services/refresh.ts new file mode 100644 index 00000000..4488591a --- /dev/null +++ b/src/app/bills/services/refresh.ts @@ -0,0 +1,225 @@ +import { randomUUID } from "node:crypto"; +import { connectToDatabase } from "@/app/bills/lib/mongoose"; +import { Bill } from "@/app/bills/models/Bill"; +import { acquireLock, releaseLock } from "@/app/bills/models/JobLock"; +import { getApiBills } from "@/app/bills/server/get-api-bills"; +import { + fetchBillMarkdown, + getBillFromCivicsProjectApi, + saveBillAnalysis, + summarizeBillText, + updateBillFacts, + type ApiBillDetail, +} from "@/app/bills/services/billApi"; +import { notifyNewBillAnalysis } from "@/app/bills/services/slack-notifier"; +import { env } from "@/app/bills/env"; +import { + analysisReason, + type StoredState, +} from "@/app/bills/services/refresh-decision"; + +export { analysisReason } from "@/app/bills/services/refresh-decision"; + +export const REFRESH_LOCK = "bills-refresh"; + +/** How long a sweep may hold the lease before another process may take over. */ +const LOCK_TTL_MS = 30 * 60 * 1000; + +/** OpenAI calls one sweep is allowed to make. */ +const DEFAULT_ANALYSIS_BUDGET = 10; + +export type RefreshResult = { + scanned: number; + factsWritten: number; + analyzed: number; + /** Bills that needed analysis but did not fit in this sweep's budget. */ + deferred: number; + errors: number; + skippedReason?: "locked" | "no-database" | "no-api-bills"; +}; + +function hasValidMongoUri(): boolean { + const uri = env.MONGO_URI || ""; + return uri.startsWith("mongodb://") || uri.startsWith("mongodb+srv://"); +} + +function sourceOf(bill: ApiBillDetail): string | undefined { + return bill.source || (bill.billTexts?.[0] as { url?: string })?.url; +} + +/** + * Bring every bill in the current Parliament up to date. + * + * Two passes, deliberately unequal in cost: + * + * 1. Facts — status, stages, sponsor, genres — are refreshed for every stored + * bill on every sweep. No LLM, so this is cheap, and it is what keeps the + * detail page from showing a verdict next to a months-old status. + * 2. Analysis is re-run only for bills that are new to us or whose text has + * actually changed, and only up to `analysisBudget` per sweep. + * + * This is the job that used to not exist: analysis ran inside whichever page + * render first encountered an unseen bill, which meant a bill analyzed at first + * reading kept that verdict through every amendment. + */ +export async function refreshBills(options?: { + analysisBudget?: number; + force?: boolean; +}): Promise { + const analysisBudget = options?.analysisBudget ?? DEFAULT_ANALYSIS_BUDGET; + const force = options?.force ?? false; + const empty: RefreshResult = { + scanned: 0, + factsWritten: 0, + analyzed: 0, + deferred: 0, + errors: 0, + }; + + if (!hasValidMongoUri()) { + console.warn("[bills-refresh] No valid MONGO_URI — skipping sweep"); + return { ...empty, skippedReason: "no-database" }; + } + + await connectToDatabase(); + + const holder = randomUUID(); + if (!(await acquireLock(REFRESH_LOCK, LOCK_TTL_MS, holder))) { + console.log("[bills-refresh] Another sweep holds the lock — skipping"); + return { ...empty, skippedReason: "locked" }; + } + + const startedAt = Date.now(); + try { + const apiBills = await getApiBills(); + if (apiBills.length === 0) { + console.warn("[bills-refresh] API returned no bills — skipping sweep"); + return { ...empty, skippedReason: "no-api-bills" }; + } + + const stored = (await Bill.find({}) + .select("billId source billTextsCount summary analysisGeneratedAt") + .lean() + .exec()) as unknown as Array<{ + billId: string; + source?: string; + billTextsCount?: number; + summary?: string; + analysisGeneratedAt?: Date; + }>; + + const storedById = new Map( + stored.map((b) => [ + b.billId, + { + source: b.source, + billTextsCount: b.billTextsCount, + hasAnalysis: Boolean(b.summary?.trim()), + analysedAt: b.analysisGeneratedAt, + }, + ]), + ); + + const result: RefreshResult = { ...empty, scanned: apiBills.length }; + + for (const listBill of apiBills) { + // The list endpoint omits `billTexts` and `source`, so the per-bill + // record is what the decision is actually made on. + let detail: ApiBillDetail | null = null; + try { + detail = await getBillFromCivicsProjectApi(listBill.billID); + } catch (error) { + console.error( + `[bills-refresh] Failed to fetch ${listBill.billID}:`, + error, + ); + result.errors += 1; + continue; + } + if (!detail) continue; + + const source = sourceOf(detail); + const billTextsCount = Array.isArray(detail.billTexts) + ? detail.billTexts.length + : 0; + const state = storedById.get(detail.billID); + + const reason = analysisReason(state, source, billTextsCount, force); + + if (!reason) { + // Known bill, unchanged text: refresh the facts and move on. + if (await updateBillFacts(detail, source)) result.factsWritten += 1; + continue; + } + + if (result.analyzed >= analysisBudget) { + // Out of budget. Still take the facts — the next sweep picks up the + // analysis, and in the meantime the page shows a current status. + if (state && (await updateBillFacts(detail, source))) { + result.factsWritten += 1; + } + result.deferred += 1; + continue; + } + + if (!source) { + console.warn( + `[bills-refresh] ${detail.billID} has no bill text source — facts only`, + ); + if (state && (await updateBillFacts(detail, source))) { + result.factsWritten += 1; + } + continue; + } + + try { + const markdown = await fetchBillMarkdown(source); + if (!markdown) { + console.warn( + `[bills-refresh] ${detail.billID}: could not read bill text at ${source}`, + ); + result.errors += 1; + continue; + } + + console.log(`[bills-refresh] Analyzing ${detail.billID} (${reason})`); + // The sweep is the deliberate, budgeted caller — the organic-traffic + // cap exists to bound accidental loops, not this. + const analysis = await summarizeBillText(markdown, { bypassCap: true }); + if (analysis.isFallback) { + // Leave the stored analysis alone and try again next sweep. + console.warn( + `[bills-refresh] ${detail.billID}: degraded analysis, not persisted`, + ); + result.errors += 1; + continue; + } + + await saveBillAnalysis({ bill: detail, analysis, source }); + result.analyzed += 1; + + // Announce genuinely new analyses only, not every re-read. + if (reason === "new" || reason === "no-analysis") { + await notifyNewBillAnalysis({ + billId: detail.billID, + title: detail.title, + shortTitle: detail.shortTitle, + analysis, + }); + } + } catch (error) { + console.error(`[bills-refresh] ${detail.billID} failed:`, error); + result.errors += 1; + } + } + + console.log( + `[bills-refresh] done in ${Math.round((Date.now() - startedAt) / 1000)}s —`, + `scanned=${result.scanned} factsWritten=${result.factsWritten}`, + `analyzed=${result.analyzed} deferred=${result.deferred} errors=${result.errors}`, + ); + return result; + } finally { + await releaseLock(REFRESH_LOCK, holder); + } +} diff --git a/src/app/bills/services/social-issue-grader.ts b/src/app/bills/services/social-issue-grader.ts deleted file mode 100644 index c3939b12..00000000 --- a/src/app/bills/services/social-issue-grader.ts +++ /dev/null @@ -1,56 +0,0 @@ -import { OpenAI } from "openai"; - -export const SOCIAL_ISSUE_GRADER_PROMPT = ` -You are a policy classifier. Your sole task is to decide whether a bill is primarily a social issue. - -Definition — Social Issue (for this classifier): -A bill is a social issue if its primary purpose centers on culture, identity, values, or rights in society — including recognition/commemoration, national symbols, moral/ethical questions, language and heritage, religion, family/sex/reproduction, education content, speech/censorship, discrimination/equality, civil liberties, and community identity. - -Positive signals (any one can qualify if it is the main focus): -- Recognition/commemoration: heritage months/days, awareness days, honorary observances, national symbols (e.g., national bird/anthem/flag changes). -- Rights & identity: assisted dying, abortion, marriage/family status, gender identity/expression, LGBTQ+ rights, indigenous rights, disability rights, hate speech/hate crimes, religious freedoms. -- Culture & language: multiculturalism, official languages, curriculum content on culture/history, media/broadcast standards on content/morality. -- Civil liberties & expression: protests/assembly, press/speech regulations primarily about expression or social values. - -Negative/Non-social (unless rights/identity are the central focus): -- Core economics/fiscal: budgets, taxation, appropriations, trade, monetary policy. -- Infrastructure/operations: transportation, energy, housing supply mechanics, procurement, zoning mechanics. -- Technical/administrative: agency powers, forms, reporting, definitions not tied to values/identity. -- Environmental/health/safety mainly as regulation/operations (e.g., emissions standards, workplace safety), unless framed around rights/identity or moral controversy. - -Tie-breakers: -- Classify based on primary purpose, not incidental mentions. -- If the bill materially creates or changes an observance/day/month or declares a national symbol, classify as social issue = yes. -- If mixed, choose "no". - -Output exactly this JSON: -{ - "is_social_issue": "yes|no" -}`; - -// This defaults to false and a verdict will present to the user -export const socialIssueGrader = async (text: string): Promise => { - // Fast path when no API key - if (!process.env.OPENAI_API_KEY) { - return false; - } - - try { - const client = new OpenAI(); - const input = `${SOCIAL_ISSUE_GRADER_PROMPT}\n\nBill content:\n${text?.slice(0, 8000)}`; - const response = await client.responses.create({ model: "gpt-5", input }); - const raw = response.output_text; - try { - const parsed = JSON.parse(raw || "{}") as { is_social_issue?: string }; - return (parsed.is_social_issue || "no").toLowerCase() === "yes"; - } catch { - // Try to detect yes/no in raw - const yes = /is[_\s-]?social[_\s-]?issue\s*["':\s]*yes/i.test(raw || ""); - const no = /is[_\s-]?social[_\s-]?issue\s*["':\s]*no/i.test(raw || ""); - if (yes || no) return yes; - return false; - } - } catch { - return false; - } -}; diff --git a/src/app/bills/utils/billConverters.ts b/src/app/bills/utils/billConverters.ts index 92eccc16..59462d82 100644 --- a/src/app/bills/utils/billConverters.ts +++ b/src/app/bills/utils/billConverters.ts @@ -1,14 +1,5 @@ import type { BillDocument } from "@/app/bills/models/Bill"; import type { ApiBillDetail } from "@/app/bills/services/billApi"; -import { - summarizeBillText, - fetchBillMarkdown, - onBillNotInDatabase, - type BillAnalysis, -} from "@/app/bills/services/billApi"; -import { socialIssueGrader } from "@/app/bills/services/social-issue-grader"; -import { notifyNewBillAnalysis } from "@/app/bills/services/slack-notifier"; -import { lookupBillInDB } from "@/app/bills/server/get-bill-by-id-from-db"; // Unified bill data structure export interface UnifiedBill { @@ -47,6 +38,11 @@ export interface UnifiedBill { needs_more_info?: boolean; missing_details?: string[]; steel_man?: string; + /** When this verdict was computed, and from which bill text. */ + analysisGeneratedAt?: Date; + analysisSourceRef?: string; + /** True when no analysis exists yet — the refresh sweep has not reached it. */ + analysisPending?: boolean; } // Convert Build Canada DB bill to unified format @@ -100,13 +96,28 @@ export function fromBuildCanadaDbBill(bill: BillDocument): UnifiedBill { ? [...bill.missing_details] : undefined, steel_man: bill.steel_man, + analysisGeneratedAt: bill.analysisGeneratedAt, + analysisSourceRef: bill.analysisSourceRef, + analysisPending: !bill.analysisGeneratedAt && !bill.summary, }; } // Convert Civics Project API bill to unified format -export async function fromCivicsProjectApiBill( - bill: ApiBillDetail, -): Promise { +/** + * Shape a Civics Project API bill into the unified structure. + * + * Pure: no LLM call, no database write, no Slack post. Analysis is owned by the + * refresh sweep (`services/refresh.ts`), which runs on a schedule rather than + * inside whichever visitor happened to open an un-analyzed bill first. That + * visitor used to pay for two `gpt-5` calls at `reasoning.effort: "high"` + * inside their page render — the cost the "July 14 incident" guard was holding + * back by a single boolean. + * + * A bill the sweep has not reached yet comes back with `analysisPending: true` + * and no verdict, which the page renders as "Analysis pending" over the bill's + * real facts. + */ +export function fromCivicsProjectApiBill(bill: ApiBillDetail): UnifiedBill { const latestStageDate = bill.stages && bill.stages.length > 0 ? bill.stages[bill.stages.length - 1].date @@ -116,179 +127,11 @@ export async function fromCivicsProjectApiBill( ? bill.stages[bill.stages.length - 1].house : undefined; - let billMarkdown: string | null = null; - - const latestBillSource = - bill.source || (bill.billTexts?.[0] as { url?: string })?.url; - - if (latestBillSource) { - billMarkdown = await fetchBillMarkdown(latestBillSource); - } - - // Check if we need to regenerate summary based on source changes from Civics Project API - let analysis: BillAnalysis = { - summary: bill.header || "", - tenet_evaluations: [ - { - id: 1, - title: "Canada should aim to be the world's most prosperous country", - alignment: "neutral", - explanation: "Not analyzed", - }, - { - id: 2, - title: - "Promote economic freedom, ambition, and breaking from bureaucratic inertia", - alignment: "neutral", - explanation: "Not analyzed", - }, - { - id: 3, - title: "Drive national productivity and global competitiveness", - alignment: "neutral", - explanation: "Not analyzed", - }, - { - id: 4, - title: "Grow exports of Canadian products and resources", - alignment: "neutral", - explanation: "Not analyzed", - }, - { - id: 5, - title: "Encourage investment, innovation, and resource development", - alignment: "neutral", - explanation: "Not analyzed", - }, - { - id: 6, - title: - "Deliver better public services at lower cost (government efficiency)", - alignment: "neutral", - explanation: "Not analyzed", - }, - { - id: 7, - title: "Reform taxes to incentivize work, risk-taking, and innovation", - alignment: "neutral", - explanation: "Not analyzed", - }, - { - id: 8, - title: "Focus on large-scale prosperity, not incrementalism", - alignment: "neutral", - explanation: "Not analyzed", - }, - ], - final_judgment: "abstain", - rationale: undefined, - needs_more_info: false, - missing_details: [], - steel_man: "Not analyzed", - }; - - // LLM summarization is fail-safe: it may only run when the DB is confirmed - // reachable AND the bill is genuinely new or its source text actually - // changed. An unavailable DB (missing MONGO_URI, connection error) must - // never be treated as "bill not found" — that turns every page view into an - // OpenAI call (July 14 incident). - const dbLookup = await lookupBillInDB(bill.billID); - const existingBill = dbLookup.status === "found" ? dbLookup.bill : null; - const dbAvailable = dbLookup.status !== "unavailable"; - - if (existingBill) { - analysis = { - summary: existingBill.summary, - tenet_evaluations: - existingBill.tenet_evaluations || analysis.tenet_evaluations, - final_judgment: (() => { - const raw = String(existingBill.final_judgment || "") - .trim() - .toLowerCase(); - if (raw === "yes" || raw === "no") return raw as "yes" | "no"; - // Treat legacy "neutral" and any unknown value as "abstain" - if (raw === "abstain" || raw === "neutral") return "abstain"; - return analysis.final_judgment; - })(), - rationale: existingBill.rationale || analysis.rationale, - needs_more_info: existingBill.needs_more_info || analysis.needs_more_info, - missing_details: existingBill.missing_details || analysis.missing_details, - steel_man: existingBill.steel_man || analysis.steel_man, - }; - } - - const billTextsCount = Array.isArray(bill.billTexts) - ? bill.billTexts.length - : 0; - const sourceChanged = existingBill - ? (existingBill.source || null) !== (bill.source || null) - : false; - const countChanged = existingBill - ? existingBill.billTextsCount !== billTextsCount - : false; - const shouldRegenerate = - dbAvailable && - (dbLookup.status === "not-found" || sourceChanged || countChanged); - - let generatedNewAnalysis = false; - if (shouldRegenerate && billMarkdown) { - console.log( - `Regenerating analysis for ${bill.billID} (${ - existingBill ? "source changed" : "new bill" - })`, - ); - analysis = await summarizeBillText(billMarkdown); - generatedNewAnalysis = true; - } else if (existingBill) { - console.log( - `Using existing analysis for ${bill.billID} (source unchanged)`, - ); - } - - // Only classify if missing (new bill or classification absent). Avoid calling otherwise. - let isSocialIssueFinal: boolean = - typeof existingBill?.isSocialIssue === "boolean" - ? ((existingBill as BillDocument).isSocialIssue as boolean) - : false; - if ( - dbAvailable && - (existingBill === null || typeof existingBill.isSocialIssue !== "boolean") - ) { - isSocialIssueFinal = await socialIssueGrader( - billMarkdown || analysis.summary || bill.header || bill.title, - ); - } - - // A fallback analysis (no OpenAI key, parse failure, API error) must never - // overwrite stored data or ping Slack. - const analysisUsable = !generatedNewAnalysis || !analysis.isFallback; - - if (dbAvailable && analysisUsable) { - await onBillNotInDatabase({ - billId: bill.billID, - source: bill.source, - markdown: billMarkdown, - bill, - analysis, - billTextsCount, - isSocialIssue: isSocialIssueFinal, - }); - } - - if (generatedNewAnalysis && analysisUsable) { - await notifyNewBillAnalysis({ - billId: bill.billID, - title: bill.title, - shortTitle: bill.shortTitle, - analysis, - }); - } - return { billId: bill.billID, title: bill.title, short_title: bill.shortTitle, - summary: analysis.summary, + summary: bill.header || "", status: bill.status, stages: bill.stages ? bill.stages.map((stage) => ({ @@ -306,14 +149,8 @@ export async function fromCivicsProjectApiBill( genres: bill.genres, parliamentNumber: bill.parliamentNumber, sessionNumber: bill.sessionNumber, - fullTextMarkdown: billMarkdown, - question_period_questions: analysis.question_period_questions, - // Include analysis data - tenet_evaluations: analysis.tenet_evaluations, - final_judgment: analysis.final_judgment, - rationale: analysis.rationale, - needs_more_info: analysis.needs_more_info, - missing_details: analysis.missing_details, - steel_man: analysis.steel_man, + fullTextMarkdown: null, + final_judgment: "abstain", + analysisPending: true, }; } diff --git a/src/app/bills/utils/merge-bill.test.ts b/src/app/bills/utils/merge-bill.test.ts new file mode 100644 index 00000000..820c0c91 --- /dev/null +++ b/src/app/bills/utils/merge-bill.test.ts @@ -0,0 +1,115 @@ +import assert from "node:assert/strict"; +import { test } from "node:test"; +import { applyApiFacts, mergeBillLists } from "./merge-bill"; +import type { UnifiedBill } from "./billConverters"; +import type { BillSummary } from "../types"; + +function storedBill(overrides: Partial = {}): UnifiedBill { + return { + billId: "C-5", + title: "An Act respecting one thing", + summary: "The stored summary.", + status: "Introduced", + stages: [ + { + stage: "First Reading", + state: "Completed", + house: "House of Commons", + date: new Date("2026-01-10T00:00:00Z"), + }, + ], + final_judgment: "yes", + rationale: "The stored rationale.", + steel_man: "The stored steel man.", + sponsorParty: "Liberal", + ...overrides, + }; +} + +function apiBill(overrides: Partial = {}): UnifiedBill { + return { + billId: "C-5", + title: "An Act respecting one thing", + summary: "", + status: "Royal Assent", + stages: [ + { + stage: "First Reading", + state: "Completed", + house: "House of Commons", + date: new Date("2026-01-10T00:00:00Z"), + }, + { + stage: "Third Reading", + state: "Completed", + house: "Senate", + date: new Date("2026-06-02T00:00:00Z"), + }, + ], + final_judgment: "abstain", + analysisPending: true, + ...overrides, + }; +} + +test("applyApiFacts takes status and stages from the API, verdict from storage", () => { + const merged = applyApiFacts(storedBill(), apiBill()); + + // Facts follow the API — this is the disagreement the detail page used to show. + assert.equal(merged.status, "Royal Assent"); + assert.equal(merged.stages.length, 2); + + // The verdict stays with the stored analysis. + assert.equal(merged.final_judgment, "yes"); + assert.equal(merged.summary, "The stored summary."); + assert.equal(merged.rationale, "The stored rationale."); + assert.equal(merged.steel_man, "The stored steel man."); + assert.equal(merged.analysisPending, undefined); +}); + +test("applyApiFacts leaves the stored bill alone when the API is unavailable", () => { + const stored = storedBill(); + assert.deepEqual(applyApiFacts(stored, null), stored); +}); + +test("applyApiFacts keeps stored facts the API omits", () => { + const merged = applyApiFacts( + storedBill({ sponsorParty: "Liberal", genres: ["Finance"] }), + apiBill({ sponsorParty: undefined, genres: [], stages: [] }), + ); + assert.equal(merged.sponsorParty, "Liberal"); + assert.deepEqual(merged.genres, ["Finance"]); + // An empty stages array from the API is absence, not a bill losing its stages. + assert.equal(merged.stages.length, 1); +}); + +test("mergeBillLists keeps API order and appends database-only bills", () => { + const api: BillSummary[] = [ + { + billID: "C-5", + title: "An Act respecting one thing", + description: "", + status: "Passed", + sponsorParty: "Liberal", + sponsorName: "Someone", + chamber: "House of Commons", + introducedOn: "2026-01-10T00:00:00.000Z", + lastUpdatedOn: "2026-06-02T00:00:00.000Z", + }, + ]; + + const merged = mergeBillLists(api, [ + storedBill(), + storedBill({ billId: "S-9", title: "A Senate bill", final_judgment: "no" }), + ]); + + assert.equal(merged.length, 2); + assert.equal(merged[0].billID, "C-5"); + // API status wins, stored verdict wins. + assert.equal(merged[0].status, "Passed"); + assert.equal(merged[0].final_judgment, "yes"); + assert.equal(merged[0].summary, "The stored summary."); + + assert.equal(merged[1].billID, "S-9"); + assert.equal(merged[1].final_judgment, "no"); +}); diff --git a/src/app/bills/utils/merge-bill.ts b/src/app/bills/utils/merge-bill.ts new file mode 100644 index 00000000..f39cf180 --- /dev/null +++ b/src/app/bills/utils/merge-bill.ts @@ -0,0 +1,120 @@ +import type { BillSummary } from "@/app/bills/types"; +import type { UnifiedBill } from "@/app/bills/utils/billConverters"; + +/** + * One precedence rule, applied everywhere a bill is rendered. + * + * Facts (status, stages, sponsor, title, genres) — the Civics Project API wins. + * Verdict (summary, tenets, judgment, rationale, steel man) — the database wins. + * + * The list page used to apply this and the detail page did not, so a bill could + * read "Royal Assent" in the list and "Introduced" on its own page: the stored + * document's `status` and `stages` are frozen at whatever they were when the + * analysis was written. Both pages now go through here. + */ + +function toIsoString(value?: Date | string): string | undefined { + if (!value) return undefined; + const parsed = value instanceof Date ? value : new Date(value); + return Number.isNaN(parsed.getTime()) ? undefined : parsed.toISOString(); +} + +/** The stored verdict fields, laid over an API bill. */ +export function mergeBillSummary( + apiBill: BillSummary, + dbBill: UnifiedBill | undefined, +): BillSummary { + if (!dbBill) return apiBill; + + return { + ...dbBill, + ...apiBill, + shortTitle: dbBill.short_title || apiBill.shortTitle, + summary: dbBill.summary, + isSocialIssue: dbBill.isSocialIssue, + final_judgment: dbBill.final_judgment as BillSummary["final_judgment"], + rationale: dbBill.rationale, + needs_more_info: dbBill.needs_more_info, + missing_details: dbBill.missing_details, + genres: dbBill.genres, + parliamentNumber: dbBill.parliamentNumber, + sessionNumber: dbBill.sessionNumber, + }; +} + +/** A bill the API no longer lists (or has not listed yet), shown from storage alone. */ +export function dbBillToSummary(dbBill: UnifiedBill): BillSummary { + return { + billID: dbBill.billId, + title: dbBill.title, + shortTitle: dbBill.short_title, + stages: dbBill.stages || [], + description: dbBill.summary || "", + status: (dbBill.status as BillSummary["status"]) || "Introduced", + sponsorParty: dbBill.sponsorParty || "Unknown", + sponsorName: "Unknown", + chamber: + (dbBill.chamber as "House of Commons" | "Senate") || "House of Commons", + introducedOn: toIsoString(dbBill.introducedOn) || new Date().toISOString(), + lastUpdatedOn: toIsoString(dbBill.lastUpdatedOn) || new Date().toISOString(), + summary: dbBill.summary, + isSocialIssue: dbBill.isSocialIssue, + final_judgment: dbBill.final_judgment as BillSummary["final_judgment"], + rationale: dbBill.rationale, + needs_more_info: dbBill.needs_more_info, + missing_details: dbBill.missing_details, + genres: dbBill.genres, + parliamentNumber: dbBill.parliamentNumber, + sessionNumber: dbBill.sessionNumber, + }; +} + +/** + * Merge the full API list into the stored bills. API order is preserved; + * bills only present in the database are appended. + */ +export function mergeBillLists( + apiBills: BillSummary[], + dbBills: UnifiedBill[], +): BillSummary[] { + const dbBillsMap = new Map(dbBills.map((bill) => [bill.billId, bill])); + + const merged = apiBills.map((apiBill) => + mergeBillSummary(apiBill, dbBillsMap.get(apiBill.billID)), + ); + + const seen = new Set(merged.map((bill) => bill.billID)); + for (const [billId, dbBill] of dbBillsMap) { + if (!seen.has(billId)) merged.push(dbBillToSummary(dbBill)); + } + + return merged; +} + +/** + * The detail-page counterpart: the stored verdict, with the API's current facts + * laid over it. `apiBill` is the same unified shape the API converter produces, + * so only its factual half is read. + */ +export function applyApiFacts( + stored: UnifiedBill, + apiBill: UnifiedBill | null, +): UnifiedBill { + if (!apiBill) return stored; + + return { + ...stored, + title: apiBill.title || stored.title, + short_title: stored.short_title || apiBill.short_title, + status: apiBill.status || stored.status, + stages: apiBill.stages?.length ? apiBill.stages : stored.stages, + sponsorParty: apiBill.sponsorParty ?? stored.sponsorParty, + chamber: apiBill.chamber ?? stored.chamber, + supportedRegion: apiBill.supportedRegion ?? stored.supportedRegion, + genres: apiBill.genres?.length ? apiBill.genres : stored.genres, + introducedOn: apiBill.introducedOn ?? stored.introducedOn, + lastUpdatedOn: apiBill.lastUpdatedOn ?? stored.lastUpdatedOn, + parliamentNumber: apiBill.parliamentNumber ?? stored.parliamentNumber, + sessionNumber: apiBill.sessionNumber ?? stored.sessionNumber, + }; +} diff --git a/src/instrumentation.ts b/src/instrumentation.ts new file mode 100644 index 00000000..2c017fa3 --- /dev/null +++ b/src/instrumentation.ts @@ -0,0 +1,47 @@ +/** + * Server-process startup hook. Next.js calls `register()` once per server + * process, which is where the Builder MP refresh sweep is scheduled. + * + * This is an in-process timer rather than a platform cron because the site runs + * as a long-lived container. The sweep takes a Mongo lease before doing any + * work, so running more than one replica is safe. + * + * Off unless BILLS_REFRESH_ENABLED=true, so a local `next dev` or a one-off + * container never starts spending OpenAI calls by accident. + * + * Unrelated to `instrumentation-client.ts` at the repo root, which is PostHog's. + */ + +const DEFAULT_INTERVAL_MINUTES = 60; +/** Let the server finish booting and start serving before the first sweep. */ +const FIRST_RUN_DELAY_MS = 2 * 60 * 1000; + +export async function register() { + if (process.env.NEXT_RUNTIME !== "nodejs") return; + if (process.env.BILLS_REFRESH_ENABLED !== "true") return; + + const intervalMinutes = + Number(process.env.BILLS_REFRESH_INTERVAL_MINUTES) || + DEFAULT_INTERVAL_MINUTES; + + const runSweep = async () => { + try { + // Imported lazily so the bills service graph is not pulled into every + // server start, only into the ones that actually schedule the sweep. + const { refreshBills } = await import("@/app/bills/services/refresh"); + await refreshBills(); + } catch (error) { + // A throw here must never take the server down. + console.error("[bills-refresh] sweep threw:", error); + } + }; + + console.log( + `[bills-refresh] scheduled every ${intervalMinutes}m (first run in ${FIRST_RUN_DELAY_MS / 60000}m)`, + ); + + setTimeout(() => { + void runSweep(); + setInterval(() => void runSweep(), intervalMinutes * 60 * 1000).unref(); + }, FIRST_RUN_DELAY_MS).unref(); +}