diff --git a/.gitattributes b/.gitattributes index 9bfcb3e9c2..7822c983d5 100644 --- a/.gitattributes +++ b/.gitattributes @@ -8,3 +8,5 @@ packages/db/drizzle/meta/*_snapshot.json linguist-generated=true *.sh text eol=lf deploy/host/roomote text eol=lf deploy/scripts/roomote-deploy text eol=lf + +packages/cloud-agents/src/server/judgement-presets.generated.json linguist-generated=true diff --git a/.judgement/README.md b/.judgement/README.md index e1db32355e..dd16f300aa 100644 --- a/.judgement/README.md +++ b/.judgement/README.md @@ -1,21 +1,19 @@ # Roomote rule calibration -These are labeled changes for each of the eight repository rules. Labels describe +These are labeled changes for each of the six repository rules. Labels describe what the rule requires, not what the model happened to answer. Keep cases that fail: they are regression examples, not reasons to change the expected label. | ID | Rule | Examples | | ------------- | --------------------------------------- | -------: | -| `criterion_1` | Secrets and private data | 6 | -| `criterion_2` | Worker subprocess environments | 6 | +| `criterion_2` | Worker credentials crossing into tasks | 8 | | `criterion_3` | Sanitized API errors | 6 | | `criterion_4` | Agent setup prose | 6 | -| `criterion_5` | Independent async operations | 6 | | `criterion_6` | UI and setup copy | 6 | | `criterion_7` | Lowercase ordinary nouns | 11 | | `criterion_8` | Direct descriptions of current behavior | 10 | -IDs currently follow the order in `.judgement/rules.json`. Preserve those IDs explicitly if +IDs are explicit in `.judgement/rules.json`. Keep them stable when removing or reordering rules, and update examples when changing the intended rule behavior. The `path` in each example matches the real rule's file globs. Before/after strings are synthetic source files; they are judged as diffs and are never executed. @@ -39,13 +37,7 @@ for rule design, calibration, and interpreting incomplete results. ## Coverage limits -The privacy rule explicitly allows synthetic credentials and fictional identities. -Its fixtures are therefore **all valid**. Do not relabel a placeholder as a secret -violation just to get a positive test, and never add real credentials or private -data as fixtures. These examples measure false alarms only; they do not establish -secret-detection recall or justify a new privacy threshold. - -The other rules include violations, allowed exceptions, and fixes to existing +The rules include violations, allowed exceptions, and fixes to existing violations. Rule 7 also checks identifiers, sentence starts, title case, exact UI labels, and an unrelated edit near an unchanged capitalization violation. Rule 8 also covers migration guides, release notes, supported choices, and useful security @@ -53,3 +45,52 @@ constraints. Incomplete checks allow hook commits but fail strict checks. They do not count as successful violation detection or complete approval of a valid edit. + +## Inspect examples in the decision tester + +Admins can open **Settings → Models → Test decisions**, choose **Repository +Judgement**, and select a rule, example, and evidence packet. +Click **Load example**, then **Ask**. The state and questions remain editable. +Select three runs to inspect variation; the last ten runs remain available while +the page is open, with their original labels and violation cutoffs. + +Expected labels stay outside model input. The model asks whether changed lines +need correction to satisfy the rule and returns the probability of a violation. +A probability at or above the rule's cutoff produces a finding; lower +probabilities produce no finding. A low score does not prove +that the edit is correct. Missing evidence and inference failures can still make +a check incomplete. The tester has a 20-second request timeout compared with the +hook's three-second budget, and a single packet does not establish a full checker +result. + +`pnpm judgement:presets` uses Judgement's `prepareExamples` API to bundle synthetic +request inputs for the deployed tester. Run it after changing rules, fixtures, +the preparation script, or the pinned library. No credentials or inference are +needed. `pnpm judgement:presets --check` verifies exact preparation; the focused +`judgement-presets.test.ts` test detects stale source hashes and question drift. +Generated inputs are checked in so the web server does not need Git or a checkout. +Observed scores and calibration reports belong in local output or CI artifacts. + +## Choosing cutoffs + +Use repeated runs of the labeled examples to compare missed violations and false +blocks. Check a candidate cutoff against additional examples under +`examples/validation/`; the prose rules also have confirmation examples under +`examples/confirmation/`. Preserve failing examples and their intended labels. +If valid and violating scores overlap, improve the rule or evidence before +lowering the cutoff. Recalibrate after changing the rule, model, or question. + +```sh +pnpm judgement test --examples-dir .judgement/examples/validation --repeats 5 +pnpm judgement test --examples-dir .judgement/examples/confirmation --rule criterion_7 --repeats 5 +pnpm judgement test --examples-dir .judgement/examples/confirmation --rule criterion_8 --repeats 5 +``` + +Keep rules focused on contextual requirements and exceptions that are difficult +to express as deterministic checks. General performance advice belongs in code +review. Keep observed calibration scores in local reports or CI artifacts. + +Rules 7 and 8 use a violation cutoff of 0.75. Validate the capitalization and prose +rules together with sentence starts, quoted UI labels, headings, migration and +security requirements, and edits near unchanged violations. Difficult examples +remain in the validation fixtures even when their scores fall below the cutoff. diff --git a/.judgement/examples/confirmation/criterion_7.json b/.judgement/examples/confirmation/criterion_7.json new file mode 100644 index 0000000000..3d5b25ac07 --- /dev/null +++ b/.judgement/examples/confirmation/criterion_7.json @@ -0,0 +1,117 @@ +{ + "ruleId": "criterion_7", + "examples": [ + { + "name": "Task in ordinary middle prose", + "path": "docs/confirmation.md", + "before": "Check your progress.\n", + "after": "Check your Task progress.\n", + "expected": "violation" + }, + { + "name": "Session in ordinary middle prose", + "path": "docs/confirmation.md", + "before": "Inspect the current output.\n", + "after": "Inspect the current Session output.\n", + "expected": "violation" + }, + { + "name": "Task after a full stop", + "path": "docs/confirmation.md", + "before": "Processing finished. Results follow.\n", + "after": "Processing finished. Task results follow.\n", + "expected": "pass" + }, + { + "name": "Title with both ordinary nouns", + "path": "docs/confirmation.md", + "before": "## History\n", + "after": "## Session and Task History\n", + "expected": "pass" + }, + { + "name": "Explicit quoted control label", + "path": "docs/confirmation.md", + "before": "Open settings.\n", + "after": "Select the \"New Session\" control.\n", + "expected": "pass" + }, + { + "name": "Lowercase normal plural", + "path": "docs/confirmation.md", + "before": "Review work.\n", + "after": "Review the open sessions and tasks.\n", + "expected": "pass" + }, + { + "name": "Ordinary session status", + "path": "docs/usage.md", + "before": "Check the current session status.\n", + "after": "Check the current Session status.\n", + "expected": "violation" + }, + { + "name": "Ordinary tasks list", + "path": "docs/usage.md", + "before": "Remove completed tasks from this list.\n", + "after": "Remove completed Tasks from this list.\n", + "expected": "violation" + }, + { + "name": "Task in UI description", + "path": "apps/web/src/task-hint.ts", + "before": "export const hint = \"Review task details here.\";\n", + "after": "export const hint = \"Review Task details here.\";\n", + "expected": "violation" + }, + { + "name": "Ordinary sessions search", + "path": "docs/usage.md", + "before": "Search recent sessions by project.\n", + "after": "Search recent Sessions by project.\n", + "expected": "violation" + }, + { + "name": "Capital after question mark", + "path": "docs/usage.md", + "before": "Ready? Results are available.\n", + "after": "Ready? Task results are available.\n", + "expected": "pass" + }, + { + "name": "Capital after full stop", + "path": "docs/usage.md", + "before": "Review finished. Results are available.\n", + "after": "Review finished. Session results are available.\n", + "expected": "pass" + }, + { + "name": "Quoted exact button", + "path": "docs/usage.md", + "before": "Choose the launch control.\n", + "after": "Choose the button labeled \"New Task\".\n", + "expected": "pass" + }, + { + "name": "Heading", + "path": "docs/usage.md", + "before": "## Work history\n", + "after": "## Session History\n", + "expected": "pass" + }, + { + "name": "Identifier outside prose", + "path": "apps/web/src/task-count.ts", + "before": "export const taskCount = 1;\n", + "after": "export const TaskCount = 1;\n", + "expected": "pass" + }, + { + "name": "Unchanged capital", + "path": "docs/usage.md", + "before": "View Session details.\nRefresh every minute.\n", + "after": "View Session details.\nRefresh every five minutes.\n", + "expected": "pass" + } + ] +} diff --git a/.judgement/examples/confirmation/criterion_8.json b/.judgement/examples/confirmation/criterion_8.json new file mode 100644 index 0000000000..e0c0e26f38 --- /dev/null +++ b/.judgement/examples/confirmation/criterion_8.json @@ -0,0 +1,117 @@ +{ + "ruleId": "criterion_8", + "examples": [ + { + "name": "Describe unnecessary removed path", + "path": "docs/setup.md", + "before": "Write logs to output/.\n", + "after": "Write logs to output/, unlike the previous logs/ directory.\n", + "expected": "violation" + }, + { + "name": "Unnecessary exclusive wording", + "path": "docs/setup.md", + "before": "Store themes in themes/.\n", + "after": "Store themes in themes/, the sole theme storage location.\n", + "expected": "violation" + }, + { + "name": "Document authentication requirement", + "path": "docs/setup.md", + "before": "Configure authentication.\n", + "after": "Only requests with a valid signature may update account settings.\n", + "expected": "pass" + }, + { + "name": "Necessary compatibility restriction", + "path": "docs/setup.md", + "before": "Install a supported runtime.\n", + "after": "Only Node 22 and newer support the APIs this package uses.\n", + "expected": "pass" + }, + { + "name": "Describe a change in release notes", + "path": "docs/setup.md", + "before": "## Release notes\nMaintenance changes.\n", + "after": "## Release notes\nThemes now live in themes/ instead of customizations/.\n", + "expected": "pass" + }, + { + "name": "Compare supported choices", + "path": "docs/setup.md", + "before": "Choose a storage backend.\n", + "after": "Choose local storage for a single server or object storage when several servers share uploads.\n", + "expected": "pass" + }, + { + "name": "Old path aside", + "path": "docs/usage.md", + "before": "Save exports under exports/.\n", + "after": "Save exports under exports/, unlike the former downloads/ directory.\n", + "expected": "violation" + }, + { + "name": "Historical format aside", + "path": "docs/usage.md", + "before": "Write preferences to prefs.json.\n", + "after": "Write preferences to prefs.json, replacing the older prefs.yaml approach.\n", + "expected": "violation" + }, + { + "name": "Exclusive path aside", + "path": "docs/usage.md", + "before": "Put saved views in views/.\n", + "after": "Put saved views in views/, the only location for saved views.\n", + "expected": "violation" + }, + { + "name": "Removed option aside", + "path": "docs/usage.md", + "before": "Configure the listener in listeners.json.\n", + "after": "Configure the listener in listeners.json; the retired listener.ini alternative is no longer used.\n", + "expected": "violation" + }, + { + "name": "Necessary upgrade migration", + "path": "docs/usage.md", + "before": "Upgrade the service.\n", + "after": "Before upgrading from version 1 to version 2, convert prefs.yaml to prefs.json so saved preferences remain available.\n", + "expected": "pass" + }, + { + "name": "Supported alternatives", + "path": "docs/usage.md", + "before": "Choose an output format.\n", + "after": "Use JSON for structured processing or CSV for spreadsheet imports.\n", + "expected": "pass" + }, + { + "name": "Security requirement", + "path": "docs/usage.md", + "before": "Configure access.\n", + "after": "Only repository administrators may change credential settings.\n", + "expected": "pass" + }, + { + "name": "Compatibility requirement", + "path": "docs/usage.md", + "before": "Choose a runtime.\n", + "after": "Only Node 22 or newer provides the required API.\n", + "expected": "pass" + }, + { + "name": "Release notes", + "path": "docs/usage.md", + "before": "## Release notes\nMaintenance release.\n", + "after": "## Release notes\nExports now use exports/ instead of downloads/.\n", + "expected": "pass" + }, + { + "name": "Unchanged historical aside", + "path": "docs/usage.md", + "before": "Save reports in reports/, unlike the former output/ directory.\nRun the report daily.\n", + "after": "Save reports in reports/, unlike the former output/ directory.\nRun the report weekly.\n", + "expected": "pass" + } + ] +} diff --git a/.judgement/examples/criterion_1.json b/.judgement/examples/criterion_1.json deleted file mode 100644 index ad6a578b47..0000000000 --- a/.judgement/examples/criterion_1.json +++ /dev/null @@ -1,47 +0,0 @@ -{ - "ruleId": "criterion_1", - "examples": [ - { - "name": "Clearly synthetic credential placeholder", - "path": ".env.example", - "before": "TOKEN=\n", - "after": "TOKEN=REPLACE_WITH_YOUR_TOKEN\n", - "expected": "pass" - }, - { - "name": "Server-side credential lookup", - "path": "apps/api/src/config.ts", - "before": "export const config = {};\n", - "after": "export const config = { apiKey: process.env.PROVIDER_API_KEY };\n", - "expected": "pass" - }, - { - "name": "Redacted diagnostic example", - "path": "docs/troubleshooting.md", - "before": "Example output:\n", - "after": "Example output: {\"authorization\":\"[REDACTED]\"}\n", - "expected": "pass" - }, - { - "name": "Public configuration", - "path": "src/config.ts", - "before": "export const options = {};\n", - "after": "export const options = { timeoutMs: 3000, retries: 2 };\n", - "expected": "pass" - }, - { - "name": "Unmistakably fictional fixture", - "path": "test/users.fixture.ts", - "before": "export const users = [];\n", - "after": "export const users = [{ name: \"Synthetic Test User\", email: \"test-user@example.invalid\" }];\n", - "expected": "pass" - }, - { - "name": "Remove a placeholder from an example", - "path": ".env.example", - "before": "API_KEY=REPLACE_WITH_YOUR_KEY\n", - "after": "API_KEY=\n", - "expected": "pass" - } - ] -} diff --git a/.judgement/examples/criterion_2.json b/.judgement/examples/criterion_2.json index 0512802b42..0703e8d229 100644 --- a/.judgement/examples/criterion_2.json +++ b/.judgement/examples/criterion_2.json @@ -2,10 +2,10 @@ "ruleId": "criterion_2", "examples": [ { - "name": "Spread all launcher variables", + "name": "Spread credential-bearing worker environment", "path": "apps/worker/src/run-task/launch.ts", - "before": "export function taskEnv() { return { PATH: process.env.PATH }; }\n", - "after": "export function taskEnv() { return { ...process.env }; }\n", + "before": "export function taskEnv(workerEnv: { PATH: string; WORKER_AUTH_TOKEN: string }) { return { PATH: workerEnv.PATH }; }\n", + "after": "export function taskEnv(workerEnv: { PATH: string; WORKER_AUTH_TOKEN: string }) { return { ...workerEnv }; }\n", "expected": "violation" }, { @@ -42,6 +42,20 @@ "before": "export const exampleEnv = {};\n", "after": "export const exampleEnv = { PATH: \"/synthetic/bin\", WORKER_AUTH_TOKEN: \"NON_AUTHENTICATING_TEST_PLACEHOLDER\" };\n", "expected": "pass" + }, + { + "name": "Spread an already safe environment", + "path": "apps/worker/src/run-task/launch.ts", + "before": "export function taskEnv() { return { PATH: process.env.PATH }; }\n", + "after": "export function taskEnv() { const runtime = { PATH: process.env.PATH, LANG: \"C.UTF-8\" }; return { ...runtime }; }\n", + "expected": "pass" + }, + { + "name": "Use worker credential only for trusted control-plane request", + "path": "apps/worker/src/run-task/launch.ts", + "before": "export async function report(fetch) { return fetch(\"https://control.example.test/status\"); }\n", + "after": "export async function report(fetch) { return fetch(\"https://control.example.test/status\", { headers: { Authorization: `Bearer ${process.env.WORKER_AUTH_TOKEN}` } }); }\n", + "expected": "pass" } ] } diff --git a/.judgement/examples/criterion_5.json b/.judgement/examples/criterion_5.json deleted file mode 100644 index 745a735001..0000000000 --- a/.judgement/examples/criterion_5.json +++ /dev/null @@ -1,47 +0,0 @@ -{ - "ruleId": "criterion_5", - "examples": [ - { - "name": "Serialize independent read requests", - "path": "apps/web/src/server/dashboard.ts", - "before": "export async function load(db) {\n return Promise.all([db.projects.findMany(), db.notifications.findMany()]);\n}\n", - "after": "export async function load(db) {\n const projects = await db.projects.findMany();\n const notifications = await db.notifications.findMany();\n return [projects, notifications];\n}\n", - "expected": "violation" - }, - { - "name": "Await work before unrelated early return", - "path": "apps/web/src/server/dashboard.ts", - "before": "export async function load(fetchProjects, showProjects) {\n if (!showProjects) return [];\n return fetchProjects();\n}\n", - "after": "export async function load(fetchProjects, showProjects) {\n const projects = await fetchProjects();\n if (!showProjects) return [];\n return projects;\n}\n", - "expected": "violation" - }, - { - "name": "Parallel independent reads", - "path": "apps/web/src/server/dashboard.ts", - "before": "export async function load(db) {\n const projects = await db.projects.findMany();\n const notifications = await db.notifications.findMany();\n return [projects, notifications];\n}\n", - "after": "export async function load(db) {\n return Promise.all([db.projects.findMany(), db.notifications.findMany()]);\n}\n", - "expected": "pass" - }, - { - "name": "Second query needs first result", - "path": "apps/web/src/server/dashboard.ts", - "before": "export async function load(db, slug) { return db.projects.findBySlug(slug); }\n", - "after": "export async function load(db, slug) {\n const project = await db.projects.findBySlug(slug);\n return db.tasks.findMany({ projectId: project.id });\n}\n", - "expected": "pass" - }, - { - "name": "Preserve transaction order", - "path": "apps/web/src/server/dashboard.ts", - "before": "export async function update(db, id) { return db.project.update(id); }\n", - "after": "export async function update(db, id) {\n return db.transaction(async tx => {\n await tx.project.lock(id);\n return tx.project.update(id);\n });\n}\n", - "expected": "pass" - }, - { - "name": "Explicit rate-limited operation", - "path": "apps/web/src/server/dashboard.ts", - "before": "export async function send(sendOne, ids) { return ids; }\n", - "after": "// Provider contract permits one outstanding request at a time.\nexport async function send(sendOne, ids) {\n for (const id of ids) await sendOne(id);\n}\n", - "expected": "pass" - } - ] -} diff --git a/.judgement/examples/validation/criterion_2.json b/.judgement/examples/validation/criterion_2.json new file mode 100644 index 0000000000..6e5c7dc126 --- /dev/null +++ b/.judgement/examples/validation/criterion_2.json @@ -0,0 +1,47 @@ +{ + "ruleId": "criterion_2", + "examples": [ + { + "name": "Clone credentials into task environment", + "path": "apps/worker/src/command-executor/execute.ts", + "before": "const workerEnv = { PATH: process.env.PATH, WORKER_PROXY_TOKEN: process.env.WORKER_PROXY_TOKEN };\nexport function run(exec) { return exec(\"task\", { env: { PATH: workerEnv.PATH } }); }\n", + "after": "const workerEnv = { PATH: process.env.PATH, WORKER_PROXY_TOKEN: process.env.WORKER_PROXY_TOKEN };\nexport function run(exec) { return exec(\"task\", { env: Object.assign({}, workerEnv) }); }\n", + "expected": "violation" + }, + { + "name": "Pass proxy authentication to subprocess", + "path": "apps/worker/src/command-executor/execute.ts", + "before": "export function run(exec) { return exec(\"task\", {env:{PATH: process.env.PATH}}); }\n", + "after": "export function run(exec) { return exec(\"task\", {env:{PATH: process.env.PATH, WORKER_PROXY_TOKEN: process.env.WORKER_PROXY_TOKEN}}); }\n", + "expected": "violation" + }, + { + "name": "Allow named locale variable", + "path": "apps/worker/src/command-executor/execute.ts", + "before": "export const childEnv = {};\n", + "after": "export const childEnv = { PATH: process.env.PATH, LC_ALL: \"C.UTF-8\" };\n", + "expected": "pass" + }, + { + "name": "Strip secret while retaining explicit runtime contract", + "path": "apps/worker/src/command-executor/execute.ts", + "before": "export function env() { return {}; }\n", + "after": "// The nested service requires this non-sensitive timezone setting.\nexport function env() { return { PATH: process.env.PATH, TZ: \"UTC\" }; }\n", + "expected": "pass" + }, + { + "name": "Forward explicitly task-scoped credential", + "path": "apps/worker/src/run-task/launch.ts", + "before": "export function taskEnv(taskCredentials) { return {}; }\n", + "after": "// This API key was issued for the task and is intended for its subprocesses.\nexport function taskEnv(taskCredentials) { return { API_KEY: taskCredentials.apiKey }; }\n", + "expected": "pass" + }, + { + "name": "Rename worker credential while forwarding to task", + "path": "apps/worker/src/run-task/launch.ts", + "before": "export function taskEnv() { return { PATH: process.env.PATH }; }\n", + "after": "export function taskEnv() { const token = process.env.WORKER_AUTH_TOKEN; return { PATH: process.env.PATH, SERVICE_TOKEN: token }; }\n", + "expected": "violation" + } + ] +} diff --git a/.judgement/examples/validation/criterion_3.json b/.judgement/examples/validation/criterion_3.json new file mode 100644 index 0000000000..e197c37971 --- /dev/null +++ b/.judgement/examples/validation/criterion_3.json @@ -0,0 +1,33 @@ +{ + "ruleId": "criterion_3", + "examples": [ + { + "name": "Expose error through a response header", + "path": "apps/api/src/handlers/report.ts", + "before": "export function fail(c, err) { return c.text(\"Request failed\", 500); }\n", + "after": "export function fail(c, err) { c.header(\"X-Debug-Error\", err.stack); return c.text(\"Request failed\", 500); }\n", + "expected": "violation" + }, + { + "name": "Stringify a caught error for the caller", + "path": "apps/api/src/handlers/report.ts", + "before": "export function fail(c, err) { return c.text(\"Report unavailable\", 500); }\n", + "after": "export function fail(c, err) { return c.text(String(err), 500); }\n", + "expected": "violation" + }, + { + "name": "Known safe authentication error", + "path": "apps/api/src/handlers/report.ts", + "before": "export function fail(c) { return c.json({}, 401); }\n", + "after": "export function fail(c) { return c.json({error:\"Sign in to continue.\"}, 401); }\n", + "expected": "pass" + }, + { + "name": "Bounded diagnostic with generic response", + "path": "apps/api/src/handlers/report.ts", + "before": "export function fail(c, log, err) { return c.json({error:err.message}, 500); }\n", + "after": "export function fail(c, log, err) { log.error({code:\"REPORT_FAILED\"}); return c.json({error:\"Report unavailable. Try again later.\"},500); }\n", + "expected": "pass" + } + ] +} diff --git a/.judgement/examples/validation/criterion_4.json b/.judgement/examples/validation/criterion_4.json new file mode 100644 index 0000000000..5092ea0ce3 --- /dev/null +++ b/.judgement/examples/validation/criterion_4.json @@ -0,0 +1,33 @@ +{ + "ruleId": "criterion_4", + "examples": [ + { + "name": "Locate an internal setup control", + "path": "packages/cloud-agents/src/server/onboarding-prompt.ts", + "before": "export const reply = \"Connect your calendar to schedule events.\";\n", + "after": "export const reply = \"Use the authorization button inside the calendar configuration panel.\";\n", + "expected": "violation" + }, + { + "name": "Solicit a secret in an agent reply", + "path": "packages/cloud-agents/src/server/onboarding-prompt.ts", + "before": "export const reply = \"Authorize access securely.\";\n", + "after": "export const reply = \"Send me your service password in your next chat message.\";\n", + "expected": "violation" + }, + { + "name": "Explain an access requirement", + "path": "packages/cloud-agents/src/server/onboarding-prompt.ts", + "before": "export const reply = \"More access needed.\";\n", + "after": "export const reply = \"Grant calendar access so I can find a time for your meeting.\";\n", + "expected": "pass" + }, + { + "name": "An internal schema description", + "path": "packages/cloud-agents/src/server/onboarding-prompt.ts", + "before": "export const toolDescription = \"Configure calendar.\";\n", + "after": "// Internal tool schema, not agent prose.\nexport const toolDescription = \"Display the calendar authorization dialog and its Confirm button.\";\n", + "expected": "pass" + } + ] +} diff --git a/.judgement/examples/validation/criterion_6.json b/.judgement/examples/validation/criterion_6.json new file mode 100644 index 0000000000..69629d72f2 --- /dev/null +++ b/.judgement/examples/validation/criterion_6.json @@ -0,0 +1,33 @@ +{ + "ruleId": "criterion_6", + "examples": [ + { + "name": "Expose queue plumbing in descriptive text", + "path": "apps/web/src/components/notifications.ts", + "before": "export const description = \"Receive a summary when work finishes.\";\n", + "after": "export const description = \"A cron controller enqueues payloads and our internal worker deserializes database event rows.\";\n", + "expected": "violation" + }, + { + "name": "Describe internal dispatch in task copy", + "path": "apps/web/src/components/notifications.ts", + "before": "export const taskDescription = \"Review new issues.\";\n", + "after": "export const taskDescription = \"This task invokes a schema compiler and polls internal dispatcher state.\";\n", + "expected": "violation" + }, + { + "name": "A necessary configuration consequence", + "path": "apps/web/src/components/notifications.ts", + "before": "export const help = \"Notifications\";\n", + "after": "export const help = \"Turning this off stops email notifications for new comments.\";\n", + "expected": "pass" + }, + { + "name": "An actionable permission label", + "path": "apps/web/src/components/notifications.ts", + "before": "export const help = \"Repository access\";\n", + "after": "export const help = \"Read-only access lets your agent review code without changing files.\";\n", + "expected": "pass" + } + ] +} diff --git a/.judgement/examples/validation/criterion_7.json b/.judgement/examples/validation/criterion_7.json new file mode 100644 index 0000000000..f9b0b2f453 --- /dev/null +++ b/.judgement/examples/validation/criterion_7.json @@ -0,0 +1,96 @@ +{ + "ruleId": "criterion_7", + "examples": [ + { + "name": "Mid-sentence plural task", + "path": "docs/tasks.md", + "before": "Review the remaining work.\n", + "after": "Review the remaining Tasks before continuing.\n", + "expected": "violation" + }, + { + "name": "Mid-sentence singular session", + "path": "docs/tasks.md", + "before": "Open the work history.\n", + "after": "Open the Session history.\n", + "expected": "violation" + }, + { + "name": "Task at the beginning of a sentence", + "path": "docs/tasks.md", + "before": "View the result here.\n", + "after": "Task results appear here.\n", + "expected": "pass" + }, + { + "name": "Quoted visible UI text", + "path": "docs/tasks.md", + "before": "Choose the history view.\n", + "after": "Choose \"Session History\" from the menu.\n", + "expected": "pass" + }, + { + "name": "Heading in title case", + "path": "docs/tasks.md", + "before": "## History\nDetails are below.\n", + "after": "## Task History\nDetails are below.\n", + "expected": "pass" + }, + { + "name": "Ordinary session noun", + "path": "docs/guide.md", + "before": "Read the guidance below.\n", + "after": "Inspect the Session timeline before retrying.\n", + "expected": "violation" + }, + { + "name": "Ordinary plural sessions", + "path": "docs/guide.md", + "before": "Read the guidance below.\n", + "after": "You can search recent Sessions by name.\n", + "expected": "violation" + }, + { + "name": "Ordinary task noun", + "path": "docs/guide.md", + "before": "Read the guidance below.\n", + "after": "The Task output appears below.\n", + "expected": "violation" + }, + { + "name": "Ordinary plural tasks", + "path": "docs/guide.md", + "before": "Read the guidance below.\n", + "after": "Archive completed Tasks after review.\n", + "expected": "violation" + }, + { + "name": "Quoted control with context", + "path": "docs/guide.md", + "before": "Read the guidance below.\n", + "after": "Select the control labeled \"Task Queue\".\n", + "expected": "pass" + }, + { + "name": "Title case heading", + "path": "docs/guide.md", + "before": "Read the guidance below.\n", + "after": "## Session Timeline\nInspect recent activity here.\n", + "expected": "pass" + }, + { + "name": "Sentence start plural", + "path": "docs/guide.md", + "before": "Read the guidance below.\n", + "after": "Sessions remain available after completion.\n", + "expected": "pass" + }, + { + "name": "Code identifier", + "path": "apps/web/src/types/task-summary.ts", + "before": "Read the guidance below.\n", + "after": "export type TaskSummary = { sessionCount: number };\n", + "expected": "pass" + } + ] +} diff --git a/.judgement/examples/validation/criterion_8.json b/.judgement/examples/validation/criterion_8.json new file mode 100644 index 0000000000..85e9c5393e --- /dev/null +++ b/.judgement/examples/validation/criterion_8.json @@ -0,0 +1,96 @@ +{ + "ruleId": "criterion_8", + "examples": [ + { + "name": "Historical aside in an ordinary instruction", + "path": "docs/setup.md", + "before": "Save your filter in preferences.json.\n", + "after": "Save your filter in preferences.json, replacing the earlier filters.yaml convention.\n", + "expected": "violation" + }, + { + "name": "Unnecessary sole-location aside", + "path": "docs/setup.md", + "before": "Put templates under templates/.\n", + "after": "Put templates under templates/, the only directory for templates.\n", + "expected": "violation" + }, + { + "name": "An actionable security restriction", + "path": "docs/setup.md", + "before": "Configure the integration.\n", + "after": "Only signed webhook requests are accepted.\n", + "expected": "pass" + }, + { + "name": "An explicit migration guide", + "path": "docs/setup.md", + "before": "## Migration to version 4\nUpdate configuration.\n", + "after": "## Migration to version 4\nMove filters from filters.yaml to preferences.json before upgrading.\n", + "expected": "pass" + }, + { + "name": "Clearly quoted writing examples", + "path": "docs/setup.md", + "before": "## Writing guidance\nKeep instructions direct.\n", + "after": "## Writing guidance\nAvoid: \"templates/ is the only directory for templates.\"\nPrefer: \"Put templates under templates/.\"\n", + "expected": "pass" + }, + { + "name": "Unnecessary old cache path", + "path": "docs/guide.md", + "before": "Read the guidance below.\n", + "after": "Store cache entries in cache/, unlike the retired tmp-cache/ path.\n", + "expected": "violation" + }, + { + "name": "Unnecessary prior format", + "path": "docs/guide.md", + "before": "Read the guidance below.\n", + "after": "Define settings in settings.json, replacing the former config.ini convention.\n", + "expected": "violation" + }, + { + "name": "Unnecessary exclusivity", + "path": "docs/guide.md", + "before": "Read the guidance below.\n", + "after": "Save snippets under snippets/, the sole snippet directory.\n", + "expected": "violation" + }, + { + "name": "Migration without heading", + "path": "docs/guide.md", + "before": "Read the guidance below.\n", + "after": "When upgrading from version 2, move config.ini values into settings.json before starting version 3.\n", + "expected": "pass" + }, + { + "name": "Two supported version requirements", + "path": "docs/guide.md", + "before": "Read the guidance below.\n", + "after": "Version 2 reads config.ini; version 3 reads settings.json. Choose the format for the version you run.\n", + "expected": "pass" + }, + { + "name": "Security restriction", + "path": "docs/guide.md", + "before": "Read the guidance below.\n", + "after": "Only administrators may rotate signing keys.\n", + "expected": "pass" + }, + { + "name": "Release notes", + "path": "docs/guide.md", + "before": "Read the guidance below.\n", + "after": "## Release notes\nCache entries now live in cache/ instead of tmp-cache/.\n", + "expected": "pass" + }, + { + "name": "Quoted bad writing", + "path": "docs/guide.md", + "before": "Read the guidance below.\n", + "after": "## Writing examples\nAvoid: \"snippets/ is the sole snippet directory.\"\nPrefer: \"Save snippets under snippets/.\"\n", + "expected": "pass" + } + ] +} diff --git a/.judgement/rules.json b/.judgement/rules.json index bfc11c6d14..b4fdfdfb57 100644 --- a/.judgement/rules.json +++ b/.judgement/rules.json @@ -1,19 +1,15 @@ { "criteria": [ { - "rule": "Do not add real secrets, credentials, session tokens, private keys, customer or PII data, private deployment details, or full environment dumps to repository files. Use placeholders, redaction, or server-side/environment-managed access instead. Tests, fixtures, examples, and documentation may contain only unmistakably synthetic, non-sensitive values that cannot authenticate, identify a private person or customer, or reveal private deployment data; public identifiers and non-sensitive configuration are allowed.", - "files": ["**/*"], - "threshold": 0.96 - }, - { - "rule": "Construct child-process and task environments from explicit worker-safe allowlists; never spread launcher process.env or pass worker-only auth or proxy credentials into task subprocesses, and scrub captured secrets before launch. Intentionally propagated non-sensitive runtime variables are allowed only when explicitly named and required by a nested sandbox or service contract; tests, fixtures, and placeholders are allowed when they contain no real secret or broad inherited environment.", + "rule": "Do not expose worker-only authentication or proxy credentials through environments passed to task-controlled code. Judge whether the change carries those credentials across the worker-to-task boundary. Non-sensitive runtime variables, credentials explicitly intended for task use, and clearly synthetic test values are allowed. Keeping worker credentials inside trusted worker operations is allowed.", "files": [ "apps/worker/src/env/**/*.ts", "apps/worker/src/run-task/**/*.ts", "apps/worker/src/sandbox-server/**/*.ts", "apps/worker/src/command-executor/**/*.ts" ], - "threshold": 0.95 + "threshold": 0.75, + "id": "criterion_2" }, { "rule": "User-facing or API responses must not expose raw exception messages, stack traces, SQL queries, config or environment values, file paths, credentials, or other server internals; return an intentionally sanitized error while logging only bounded diagnostics server-side. Intentionally safe validation, not-found, authentication, provider, or health messages, authenticated operator diagnostics designed for that audience, redacted or sanitized errors, and test or fixture assertions are allowed; do not pass through raw exception objects or unsanitized internal details.", @@ -22,7 +18,8 @@ "apps/api/src/middleware/**/*.ts", "packages/sdk/src/server/**/*.ts" ], - "threshold": 0.95 + "threshold": 0.8, + "id": "criterion_3" }, { "rule": "In user-visible setup or agent prose, state the user's goal, needed capability, outcome, or decision in ordinary language; do not name, locate, or instruct the user to interact with internal cards, rails, dialogs, panels, buttons, presets, setup steps, or state labels, and do not ask the user to paste credentials in chat. Trusted UI descriptions, internal tool schemas, comments, tests or fixtures, and trusted public documentation that needs to explain controls are allowed, but user-facing setup prose must not expose those control mechanics.", @@ -32,27 +29,26 @@ "packages/cloud-agents/src/system-prompt.ts", "apps/worker/src/run-task/**/*prompt*.ts" ], - "threshold": 0.94 - }, - { - "rule": "When async operations are independent, start them before awaiting and await them together; defer awaits needed only after a branch or early return, and do not serialize independent network or database work. Truly dependent, ordered, rate-limited, transactionally coupled, or error-semantics-sensitive steps may remain sequential, as may intentional small operations and test or fixture code.", - "files": ["apps/web/src/**/*.ts", "apps/web/src/**/*.tsx"], - "threshold": 0.97 + "threshold": 0.85, + "id": "criterion_4" }, { "rule": "In user-facing UI, task, and setup copy, communicate user value, current state, or available actions rather than implementation details or feature mechanics. Avoid labels or descriptions such as 'default off' and detailed explanations of how a feature works in descriptive task or setup text. Necessary control labels, status, accessibility, error or recovery, permission, configuration-consequence, and security copy are allowed, as are tests or fixtures and trusted public documentation.", "files": ["apps/web/src/**/*.ts", "apps/web/src/**/*.tsx"], - "threshold": 0.95 + "threshold": 0.85, + "id": "criterion_6" }, { "rule": "In user-facing prose, treat \"session\" and \"task\" as ordinary nouns rather than proper nouns; use lowercase \"session\" and \"task\" except at sentence beginnings, in title case, or when quoting an exact UI label.", "files": ["**/*"], - "threshold": 0.85 + "threshold": 0.75, + "id": "criterion_7" }, { - "rule": "Release notes, migration instructions, security or compatibility requirements, and quoted writing examples are exempt from this rule. In ordinary explanatory and instructional prose, state the current location, behavior, or action directly. Do not add unnecessary asides about previous iterations or removed alternatives, or unnecessary exclusivity such as \"the only policy location\". Comparisons between supported options are allowed when they help the reader choose.", + "rule": "Release notes, migration instructions, security or compatibility requirements, and quoted writing examples are exempt from this rule. In ordinary explanatory and instructional prose, state the current location, behavior, or action directly. Do not add unnecessary asides about previous iterations or removed alternatives, or unnecessary exclusivity such as \"the only policy location\". Comparisons between supported options are allowed when they help the reader choose. A statement that restricts allowed protocols, credentials, access, or data handling is a security requirement, not unnecessary exclusivity.", "files": ["**/*"], - "threshold": 0.85 + "threshold": 0.75, + "id": "criterion_8" } ] } diff --git a/apps/docs/models.mdx b/apps/docs/models.mdx index 0ea8fc228c..d4736de45d 100644 --- a/apps/docs/models.mdx +++ b/apps/docs/models.mdx @@ -406,6 +406,17 @@ answer's probabilities and latency. When Jev answers and the Roomote judgment model is also configured, it can ask both side by side. Test decisions go to the same judgment model as real ones but are not captured or shadowed. +For **Repository Judgement**, choose a rule and labeled example, select its initial +or expanded evidence, and click **Load example**. Edit the inputs or ask for three +repeated runs. Each answer shows the probability of a rule violation and whether +it reaches the rule's cutoff. Recent runs retain their original inputs and labels. +Expected labels are never sent to the model. + +These are individual evidence requests. A partial screen cannot approve a file, +and the tester's 20-second timeout does not measure the commit hook's three-second +budget. Confirm a proposed improvement through the full Judgement checker before +changing a rule or its violation cutoff. + ## Reasoning settings Some models expose reasoning controls. Roomote lets admins set reasoning levels diff --git a/apps/web/src/components/settings/pages/JudgmentDecisionTesterPage.client.test.tsx b/apps/web/src/components/settings/pages/JudgmentDecisionTesterPage.client.test.tsx index c773e7c74a..8b58a2cf0e 100644 --- a/apps/web/src/components/settings/pages/JudgmentDecisionTesterPage.client.test.tsx +++ b/apps/web/src/components/settings/pages/JudgmentDecisionTesterPage.client.test.tsx @@ -1,7 +1,8 @@ -import { fireEvent, render, screen } from '@testing-library/react'; +import { fireEvent, render, screen, waitFor } from '@testing-library/react'; const state = vi.hoisted(() => ({ mutate: vi.fn(), + presets: undefined as unknown, data: undefined as unknown, })); @@ -19,6 +20,13 @@ const catalog = { }, sampleState: { email: { subject: 'Out of office' } }, }, + { + id: 'repository-judgement', + label: 'Repository Judgement', + description: 'Rules', + sampleState: {}, + questions: {}, + }, ], targets: { configured: { available: true, label: 'Jev via OpenRouter' }, @@ -27,12 +35,15 @@ const catalog = { }; vi.mock('@tanstack/react-query', () => ({ - useQuery: () => ({ data: catalog, isPending: false }), + useQuery: (options: { presets?: boolean }) => ({ + data: options.presets ? state.presets : catalog, + isPending: false, + }), useMutation: () => ({ isPending: false, data: state.data, reset: vi.fn(), - mutate: state.mutate, + mutateAsync: state.mutate, }), })); vi.mock('@/trpc/client', () => ({ @@ -40,6 +51,7 @@ vi.mock('@/trpc/client', () => ({ taskModels: { judgment: { decisionCatalog: { queryOptions: () => ({}) }, + examplePresets: { queryOptions: () => ({ presets: true }) }, testDecision: { mutationOptions: () => ({}) }, }, }, @@ -72,9 +84,10 @@ describe('JudgmentDecisionTesterPage', () => { beforeEach(() => { state.mutate.mockClear(); state.data = undefined; + state.mutate.mockImplementation(async () => state.data ?? {}); }); - it('loads the sample state and asks the chosen models', () => { + it('loads the sample state and asks the chosen models', async () => { render(); expect( (screen.getByLabelText('State (JSON)') as HTMLTextAreaElement).value, @@ -87,13 +100,14 @@ describe('JudgmentDecisionTesterPage', () => { fireEvent.click(screen.getByRole('option', { name: 'Both, side by side' })); fireEvent.click(screen.getByRole('button', { name: 'Ask' })); - expect(state.mutate).toHaveBeenCalledWith( - expect.objectContaining({ - state: { email: { subject: 'Out of office' } }, - questions: catalog.decisions[0]!.questions, - targets: ['configured', 'roomote'], - }), - expect.anything(), + await waitFor(() => + expect(state.mutate).toHaveBeenCalledWith( + expect.objectContaining({ + state: { email: { subject: 'Out of office' } }, + questions: catalog.decisions[0]!.questions, + targets: ['configured', 'roomote'], + }), + ), ); }); @@ -107,7 +121,7 @@ describe('JudgmentDecisionTesterPage', () => { expect(state.mutate).not.toHaveBeenCalled(); }); - it('shows each model’s answer and its latency', () => { + it('shows each model’s answer and its latency', async () => { state.data = { configured: { ok: true, @@ -121,8 +135,108 @@ describe('JudgmentDecisionTesterPage', () => { }; render(); fireEvent.click(screen.getByRole('button', { name: 'Ask' })); - expect(screen.getByText(/212 ms/)).toBeInTheDocument(); + await waitFor(() => expect(screen.getByText(/212 ms/)).toBeInTheDocument()); expect(screen.getByText(/Timed out/)).toBeInTheDocument(); expect(screen.getByText('97%')).toBeInTheDocument(); }); }); + +function selectOption(label: string, option: string) { + fireEvent.pointerDown(screen.getByRole('combobox', { name: label }), { + button: 0, + pointerType: 'mouse', + }); + fireEvent.click(screen.getByRole('option', { name: option })); +} + +it('loads exact packets and production questions, keeps labels out of requests, and preserves repeated results', async () => { + const initial = { + rule: 'Use lowercase session.', + complete: true, + evidence: [{ kind: 'patch', text: '+Session' }], + }; + const expanded = { + ...initial, + evidence: [...initial.evidence, { kind: 'after', text: 'A Session' }], + }; + const standard = { + result: { + type: 'noul', + instructions: 'Does this change violate the rule?', + criteria: { true: 'violates', false: 'complies or is inapplicable' }, + }, + }; + state.presets = { + examples: [ + { + ruleId: 'wording', + rule: initial.rule, + name: 'Capitalization', + expected: 'violation', + threshold: 0.85, + packets: [ + { state: initial, stage: 'initial', questionSet: 0 }, + { state: expanded, stage: 'expanded', questionSet: 0 }, + ], + }, + ], + questionSets: [standard], + }; + state.mutate.mockResolvedValue({ + configured: { + ok: true, + provider: 'typesafe', + model: 'jev', + answers: { + result: { + type: 'noul', + noul: 0.82, + }, + }, + invalid: [], + latencyMs: 300, + }, + }); + render(); + selectOption('Decision', 'Repository Judgement'); + fireEvent.click(screen.getByRole('button', { name: 'Load example' })); + expect( + JSON.parse( + (screen.getByLabelText('State (JSON)') as HTMLTextAreaElement).value, + ), + ).toEqual(initial); + expect( + JSON.parse( + (screen.getByLabelText('Questions (JSON)') as HTMLTextAreaElement).value, + ), + ).toEqual(standard); + selectOption('Evidence', '2 · expanded'); + expect( + screen.queryByRole('combobox', { name: 'Question variant' }), + ).not.toBeInTheDocument(); + fireEvent.click(screen.getByRole('button', { name: 'Load example' })); + selectOption('Repetitions', 'Three runs'); + fireEvent.click(screen.getByRole('button', { name: 'Ask' })); + await waitFor(() => expect(state.mutate).toHaveBeenCalledTimes(3)); + for (const [input] of state.mutate.mock.calls) { + expect(input).toEqual({ + state: expanded, + questions: standard, + targets: ['configured'], + }); + } + await waitFor(() => + expect(screen.getByText(/violation probability 82%/)).toBeInTheDocument(), + ); + expect(screen.getByText('82%')).toBeInTheDocument(); + fireEvent.change(screen.getByLabelText('State (JSON)'), { + target: { value: '{"custom":true}' }, + }); + expect(screen.getByText(/Loaded:.*edited/)).toBeInTheDocument(); + // Results retain the original packet metadata after editing the next request. + expect( + screen.getByText(/Expected: violation · Violation cutoff: 85% · expanded/), + ).toBeInTheDocument(); + selectOption('Recent runs', '3 · wording · Capitalization · run 1'); + expect(screen.getByText(/violation probability 82%/)).toBeInTheDocument(); +}); diff --git a/apps/web/src/components/settings/pages/JudgmentDecisionTesterPage.tsx b/apps/web/src/components/settings/pages/JudgmentDecisionTesterPage.tsx index 67ab3f0f6c..336139aba2 100644 --- a/apps/web/src/components/settings/pages/JudgmentDecisionTesterPage.tsx +++ b/apps/web/src/components/settings/pages/JudgmentDecisionTesterPage.tsx @@ -17,6 +17,10 @@ import { Textarea, } from '@/components/system'; import { useTRPC } from '@/trpc/client'; +import { + JudgmentExamplePresets, + type LoadedJudgmentExample, +} from './JudgmentExamplePresets'; type Target = 'configured' | 'roomote'; type ModelChoice = Target | 'both'; @@ -176,7 +180,20 @@ export function JudgmentDecisionTesterPage() { const [questionsText, setQuestionsText] = useState(''); const [modelChoice, setModelChoice] = useState(); const [inputError, setInputError] = useState(); - const [asked, setAsked] = useState>({}); + const [loadedExample, setLoadedExample] = useState(); + const [repeats, setRepeats] = useState(1); + const [running, setRunning] = useState(false); + const [runs, setRuns] = useState< + Array<{ + label: string; + state: unknown; + questions: Record; + results: Partial>; + example?: LoadedJudgmentExample; + edited: boolean; + }> + >([]); + const [runIndex, setRunIndex] = useState(0); const decision = useMemo( () => @@ -190,6 +207,7 @@ export function JudgmentDecisionTesterPage() { setStateText(JSON.stringify(decision.sampleState, null, 2)); setQuestionsText(JSON.stringify(decision.questions, null, 2)); test.reset(); + setLoadedExample(undefined); setInputError(undefined); // Reset only when the decision changes. // eslint-disable-next-line react-hooks/exhaustive-deps @@ -213,7 +231,7 @@ export function JudgmentDecisionTesterPage() { ? modelChoice : modelOptions[0]?.value; - const run = () => { + const run = async () => { let state: unknown; let questions: Record; try { @@ -236,18 +254,45 @@ export function JudgmentDecisionTesterPage() { return; } setInputError(undefined); - setAsked(questions); - test.mutate( - { - state: state as Record, - questions: questions as never, - targets: chosen, - }, - { onError: (error) => setInputError(error.message) }, - ); + setRunning(true); + try { + for (let index = 0; index < repeats; index++) { + const results = await test.mutateAsync({ + state: state as Record, + questions: questions as never, + targets: chosen, + }); + setRuns((previous) => + [ + { + label: `${loadedExample?.label ?? decision?.label ?? 'Custom'} · run ${index + 1}`, + state, + questions, + results: results as Partial>, + example: loadedExample, + edited: Boolean( + loadedExample && + (stateText !== loadedExample.stateText || + questionsText !== loadedExample.questionsText), + ), + }, + ...previous, + ].slice(0, 10), + ); + setRunIndex(0); + } + } catch (error) { + setInputError( + error instanceof Error ? error.message : 'The request failed.', + ); + } finally { + setRunning(false); + } }; - const results = (test.data ?? {}) as Partial>; + const selectedRun = runs[runIndex]; + const results = selectedRun?.results ?? {}; + const asked = selectedRun?.questions ?? {}; const answeredTargets = (Object.keys(results) as Target[]).filter( (t) => results[t], ); @@ -265,7 +310,7 @@ export function JudgmentDecisionTesterPage() { ) : catalog ? (
-
+