From 53cdcb7320f70323c9811e5b4017be0e14567cff Mon Sep 17 00:00:00 2001 From: Matt Rubens <2600+mrubens@users.noreply.github.com> Date: Tue, 29 Sep 2026 16:45:19 -0400 Subject: [PATCH 1/7] Load Judgement examples into the decision model tester --- .gitattributes | 2 + .judgement/README.md | 23 + apps/docs/models.mdx | 13 + ...JudgmentDecisionTesterPage.client.test.tsx | 160 +- .../pages/JudgmentDecisionTesterPage.tsx | 178 +- .../settings/pages/JudgmentExamplePresets.tsx | 176 + .../judgment-decision-tester.test.ts | 13 + .../task-models/judgment-decision-tester.ts | 8 + apps/web/src/trpc/routers/_app.ts | 5 + apps/worker/package.json | 2 +- package.json | 1 + packages/cloud-agents/package.json | 4 +- .../scripts/prepare-judgement-presets.mjs | 69 + .../server/judgement-presets.generated.json | 3134 +++++++++++++++++ .../src/server/judgement-presets.test.ts | 48 + pnpm-lock.yaml | 15 +- 16 files changed, 3810 insertions(+), 41 deletions(-) create mode 100644 apps/web/src/components/settings/pages/JudgmentExamplePresets.tsx create mode 100644 packages/cloud-agents/scripts/prepare-judgement-presets.mjs create mode 100644 packages/cloud-agents/src/server/judgement-presets.generated.json create mode 100644 packages/cloud-agents/src/server/judgement-presets.test.ts diff --git a/.gitattributes b/.gitattributes index 9bfcb3e9c2..7822c983d5 100644 --- a/.gitattributes +++ b/.gitattributes @@ -8,3 +8,5 @@ packages/db/drizzle/meta/*_snapshot.json linguist-generated=true *.sh text eol=lf deploy/host/roomote text eol=lf deploy/scripts/roomote-deploy text eol=lf + +packages/cloud-agents/src/server/judgement-presets.generated.json linguist-generated=true diff --git a/.judgement/README.md b/.judgement/README.md index e1db32355e..59188543ca 100644 --- a/.judgement/README.md +++ b/.judgement/README.md @@ -53,3 +53,26 @@ constraints. Incomplete checks allow hook commits but fail strict checks. They do not count as successful violation detection or complete approval of a valid edit. + +## Inspect examples in the decision tester + +Admins can open **Settings → Models → Test decisions**, choose **Repository +Judgement**, and select a rule, example, evidence packet, and question variant. +Click **Load example**, then **Ask**. The state and questions remain editable. +Select three runs to inspect variation; the last ten runs remain available while +the page is open, with their original labels and confidence thresholds. + +Expected labels stay outside model input. Confidence and answer probabilities +are displayed separately. A packet answer does not establish a full checker +result: partial screens cannot approve a file, unresolved evidence still matters, +and the tester has a 20-second request timeout compared with the hook's +three-second budget. The combined acceptable-outcome question is an explicit +experiment. Normal hooks retain the standard question. + +`pnpm judgement:presets` uses Judgement's `prepareExamples` API to bundle synthetic +request inputs for the deployed tester. Run it after changing rules, fixtures, +the preparation script, or the pinned library. No credentials or inference are +needed. `pnpm judgement:presets --check` verifies exact preparation; the focused +`judgement-presets.test.ts` test detects stale source hashes and question drift. +Generated inputs are checked in so the web server does not need Git or a checkout. +Observed scores and calibration reports belong in local output or CI artifacts. diff --git a/apps/docs/models.mdx b/apps/docs/models.mdx index 0ea8fc228c..a7b58bbea3 100644 --- a/apps/docs/models.mdx +++ b/apps/docs/models.mdx @@ -406,6 +406,19 @@ answer's probabilities and latency. When Jev answers and the Roomote judgment model is also configured, it can ask both side by side. Test decisions go to the same judgment model as real ones but are not captured or shadowed. +For **Repository Judgement**, choose a rule and labeled example, select its initial +or expanded evidence, and click **Load example**. Compare the standard question +with the experimental combined acceptable-outcome question, edit the inputs, or +ask for three repeated runs. Recent runs retain their original input labels; +confidence is shown separately from answer probabilities. Expected labels are +never sent to the model. + +These are individual evidence requests. A partial screen cannot approve a file, +and the tester's 20-second timeout does not measure the commit hook's three-second +budget. Confirm a proposed improvement through the full Judgement checker before +changing a rule or its confidence threshold. + + ## Reasoning settings Some models expose reasoning controls. Roomote lets admins set reasoning levels diff --git a/apps/web/src/components/settings/pages/JudgmentDecisionTesterPage.client.test.tsx b/apps/web/src/components/settings/pages/JudgmentDecisionTesterPage.client.test.tsx index c773e7c74a..47f256a919 100644 --- a/apps/web/src/components/settings/pages/JudgmentDecisionTesterPage.client.test.tsx +++ b/apps/web/src/components/settings/pages/JudgmentDecisionTesterPage.client.test.tsx @@ -1,7 +1,8 @@ -import { fireEvent, render, screen } from '@testing-library/react'; +import { fireEvent, render, screen, waitFor } from '@testing-library/react'; const state = vi.hoisted(() => ({ mutate: vi.fn(), + presets: undefined as unknown, data: undefined as unknown, })); @@ -19,6 +20,13 @@ const catalog = { }, sampleState: { email: { subject: 'Out of office' } }, }, + { + id: 'repository-judgement', + label: 'Repository Judgement', + description: 'Rules', + sampleState: {}, + questions: {}, + }, ], targets: { configured: { available: true, label: 'Jev via OpenRouter' }, @@ -27,12 +35,15 @@ const catalog = { }; vi.mock('@tanstack/react-query', () => ({ - useQuery: () => ({ data: catalog, isPending: false }), + useQuery: (options: { presets?: boolean }) => ({ + data: options.presets ? state.presets : catalog, + isPending: false, + }), useMutation: () => ({ isPending: false, data: state.data, reset: vi.fn(), - mutate: state.mutate, + mutateAsync: state.mutate, }), })); vi.mock('@/trpc/client', () => ({ @@ -40,6 +51,7 @@ vi.mock('@/trpc/client', () => ({ taskModels: { judgment: { decisionCatalog: { queryOptions: () => ({}) }, + examplePresets: { queryOptions: () => ({ presets: true }) }, testDecision: { mutationOptions: () => ({}) }, }, }, @@ -72,9 +84,10 @@ describe('JudgmentDecisionTesterPage', () => { beforeEach(() => { state.mutate.mockClear(); state.data = undefined; + state.mutate.mockImplementation(async () => state.data ?? {}); }); - it('loads the sample state and asks the chosen models', () => { + it('loads the sample state and asks the chosen models', async () => { render(); expect( (screen.getByLabelText('State (JSON)') as HTMLTextAreaElement).value, @@ -87,13 +100,14 @@ describe('JudgmentDecisionTesterPage', () => { fireEvent.click(screen.getByRole('option', { name: 'Both, side by side' })); fireEvent.click(screen.getByRole('button', { name: 'Ask' })); - expect(state.mutate).toHaveBeenCalledWith( - expect.objectContaining({ - state: { email: { subject: 'Out of office' } }, - questions: catalog.decisions[0]!.questions, - targets: ['configured', 'roomote'], - }), - expect.anything(), + await waitFor(() => + expect(state.mutate).toHaveBeenCalledWith( + expect.objectContaining({ + state: { email: { subject: 'Out of office' } }, + questions: catalog.decisions[0]!.questions, + targets: ['configured', 'roomote'], + }), + ), ); }); @@ -107,7 +121,7 @@ describe('JudgmentDecisionTesterPage', () => { expect(state.mutate).not.toHaveBeenCalled(); }); - it('shows each model’s answer and its latency', () => { + it('shows each model’s answer and its latency', async () => { state.data = { configured: { ok: true, @@ -121,8 +135,128 @@ describe('JudgmentDecisionTesterPage', () => { }; render(); fireEvent.click(screen.getByRole('button', { name: 'Ask' })); - expect(screen.getByText(/212 ms/)).toBeInTheDocument(); + await waitFor(() => expect(screen.getByText(/212 ms/)).toBeInTheDocument()); expect(screen.getByText(/Timed out/)).toBeInTheDocument(); expect(screen.getByText('97%')).toBeInTheDocument(); }); }); + +function selectOption(label: string, option: string) { + fireEvent.pointerDown(screen.getByRole('combobox', { name: label }), { + button: 0, + pointerType: 'mouse', + }); + fireEvent.click(screen.getByRole('option', { name: option })); +} + +it('loads exact packets and experimental questions, keeps labels out of requests, and preserves repeated results', async () => { + const initial = { + rule: 'Use lowercase session.', + complete: true, + evidence: [{ kind: 'patch', text: '+Session' }], + }; + const expanded = { + ...initial, + evidence: [...initial.evidence, { kind: 'after', text: 'A Session' }], + }; + const standard = { + result: { + type: 'choice', + instructions: 'Judge the change', + criteria: { + pass: 'valid', + violation: 'invalid', + unclear: 'unknown', + not_applicable: 'irrelevant', + }, + }, + }; + const combined = { + result: { + ...standard.result, + criteria: { + pass: 'valid or irrelevant', + violation: 'invalid', + unclear: 'unknown', + }, + }, + }; + state.presets = { + examples: [ + { + ruleId: 'wording', + rule: initial.rule, + name: 'Capitalization', + expected: 'violation', + threshold: 0.85, + packets: [ + { state: initial, stage: 'initial', questionSet: 0 }, + { state: expanded, stage: 'expanded', questionSet: 0 }, + ], + }, + ], + questionSets: [{ standard, combined }], + }; + state.mutate.mockResolvedValue({ + configured: { + ok: true, + provider: 'typesafe', + model: 'jev', + answers: { + result: { + type: 'choice', + choice: 'violation', + confidence: 0.82, + probabilities: { violation: 0.9, pass: 0.08, unclear: 0.02 }, + }, + }, + invalid: [], + latencyMs: 300, + }, + }); + render(); + selectOption('Decision', 'Repository Judgement'); + fireEvent.click(screen.getByRole('button', { name: 'Load example' })); + expect( + JSON.parse( + (screen.getByLabelText('State (JSON)') as HTMLTextAreaElement).value, + ), + ).toEqual(initial); + expect( + JSON.parse( + (screen.getByLabelText('Questions (JSON)') as HTMLTextAreaElement).value, + ), + ).toEqual(standard); + selectOption('Evidence', '2 · expanded'); + selectOption('Question variant', 'Combined acceptable outcomes'); + fireEvent.click(screen.getByRole('button', { name: 'Load example' })); + selectOption('Repetitions', 'Three runs'); + fireEvent.click(screen.getByRole('button', { name: 'Ask' })); + await waitFor(() => expect(state.mutate).toHaveBeenCalledTimes(3)); + for (const [input] of state.mutate.mock.calls) { + expect(input).toEqual({ + state: expanded, + questions: combined, + targets: ['configured'], + }); + } + await waitFor(() => + expect( + screen.getByText(/confidence 82%.*below threshold/), + ).toBeInTheDocument(), + ); + expect(screen.getByText('90%')).toBeInTheDocument(); + fireEvent.change(screen.getByLabelText('State (JSON)'), { + target: { value: '{"custom":true}' }, + }); + expect(screen.getByText(/Loaded:.*edited/)).toBeInTheDocument(); + // Results retain the original packet metadata after editing the next request. + expect( + screen.getByText(/Expected: violation · Threshold: 85% · expanded/), + ).toBeInTheDocument(); + selectOption( + 'Recent runs', + '3 · wording · Capitalization · combined · run 1', + ); + expect(screen.getByText(/confidence 82%/)).toBeInTheDocument(); +}); diff --git a/apps/web/src/components/settings/pages/JudgmentDecisionTesterPage.tsx b/apps/web/src/components/settings/pages/JudgmentDecisionTesterPage.tsx index 67ab3f0f6c..1bf446501b 100644 --- a/apps/web/src/components/settings/pages/JudgmentDecisionTesterPage.tsx +++ b/apps/web/src/components/settings/pages/JudgmentDecisionTesterPage.tsx @@ -17,6 +17,10 @@ import { Textarea, } from '@/components/system'; import { useTRPC } from '@/trpc/client'; +import { + JudgmentExamplePresets, + type LoadedJudgmentExample, +} from './JudgmentExamplePresets'; type Target = 'configured' | 'roomote'; type ModelChoice = Target | 'both'; @@ -176,7 +180,20 @@ export function JudgmentDecisionTesterPage() { const [questionsText, setQuestionsText] = useState(''); const [modelChoice, setModelChoice] = useState(); const [inputError, setInputError] = useState(); - const [asked, setAsked] = useState>({}); + const [loadedExample, setLoadedExample] = useState(); + const [repeats, setRepeats] = useState(1); + const [running, setRunning] = useState(false); + const [runs, setRuns] = useState< + Array<{ + label: string; + state: unknown; + questions: Record; + results: Partial>; + example?: LoadedJudgmentExample; + edited: boolean; + }> + >([]); + const [runIndex, setRunIndex] = useState(0); const decision = useMemo( () => @@ -190,6 +207,7 @@ export function JudgmentDecisionTesterPage() { setStateText(JSON.stringify(decision.sampleState, null, 2)); setQuestionsText(JSON.stringify(decision.questions, null, 2)); test.reset(); + setLoadedExample(undefined); setInputError(undefined); // Reset only when the decision changes. // eslint-disable-next-line react-hooks/exhaustive-deps @@ -213,7 +231,7 @@ export function JudgmentDecisionTesterPage() { ? modelChoice : modelOptions[0]?.value; - const run = () => { + const run = async () => { let state: unknown; let questions: Record; try { @@ -236,18 +254,45 @@ export function JudgmentDecisionTesterPage() { return; } setInputError(undefined); - setAsked(questions); - test.mutate( - { - state: state as Record, - questions: questions as never, - targets: chosen, - }, - { onError: (error) => setInputError(error.message) }, - ); + setRunning(true); + try { + for (let index = 0; index < repeats; index++) { + const results = await test.mutateAsync({ + state: state as Record, + questions: questions as never, + targets: chosen, + }); + setRuns((previous) => + [ + { + label: `${loadedExample?.label ?? decision?.label ?? 'Custom'} · run ${index + 1}`, + state, + questions, + results: results as Partial>, + example: loadedExample, + edited: Boolean( + loadedExample && + (stateText !== loadedExample.stateText || + questionsText !== loadedExample.questionsText), + ), + }, + ...previous, + ].slice(0, 10), + ); + setRunIndex(0); + } + } catch (error) { + setInputError( + error instanceof Error ? error.message : 'The request failed.', + ); + } finally { + setRunning(false); + } }; - const results = (test.data ?? {}) as Partial>; + const selectedRun = runs[runIndex]; + const results = selectedRun?.results ?? {}; + const asked = selectedRun?.questions ?? {}; const answeredTargets = (Object.keys(results) as Target[]).filter( (t) => results[t], ); @@ -265,7 +310,7 @@ export function JudgmentDecisionTesterPage() { ) : catalog ? (
-
+