From 06a64111400ffc954b13b7934d2281f8a81a4293 Mon Sep 17 00:00:00 2001 From: Matt Rubens <2600+mrubens@users.noreply.github.com> Date: Tue, 29 Sep 2026 15:42:36 -0400 Subject: [PATCH 1/3] Add calibration fixtures for repository judgement rules --- .judgement/.gitignore | 1 + .judgement/README.md | 92 ++++++++++++++++ .judgement/calibration-results.md | 95 +++++++++++++++++ .judgement/examples/criterion_1.json | 47 +++++++++ .judgement/examples/criterion_2.json | 47 +++++++++ .judgement/examples/criterion_3.json | 47 +++++++++ .judgement/examples/criterion_4.json | 47 +++++++++ .judgement/examples/criterion_5.json | 47 +++++++++ .judgement/examples/criterion_6.json | 47 +++++++++ .judgement/examples/criterion_7.json | 82 +++++++++++++++ .judgement/run-calibration.mjs | 151 +++++++++++++++++++++++++++ 11 files changed, 703 insertions(+) create mode 100644 .judgement/.gitignore create mode 100644 .judgement/README.md create mode 100644 .judgement/calibration-results.md create mode 100644 .judgement/examples/criterion_1.json create mode 100644 .judgement/examples/criterion_2.json create mode 100644 .judgement/examples/criterion_3.json create mode 100644 .judgement/examples/criterion_4.json create mode 100644 .judgement/examples/criterion_5.json create mode 100644 .judgement/examples/criterion_6.json create mode 100644 .judgement/examples/criterion_7.json create mode 100644 .judgement/run-calibration.mjs diff --git a/.judgement/.gitignore b/.judgement/.gitignore new file mode 100644 index 0000000000..fbca225379 --- /dev/null +++ b/.judgement/.gitignore @@ -0,0 +1 @@ +results/ diff --git a/.judgement/README.md b/.judgement/README.md new file mode 100644 index 0000000000..62c6d1c94f --- /dev/null +++ b/.judgement/README.md @@ -0,0 +1,92 @@ +# Roomote rule calibration + +These are labeled changes for each of the seven repository rules. Labels describe +what the rule requires, not what the model happened to answer. Keep cases that +fail: they are regression examples, not reasons to change the expected label. + +| ID | Rule | Examples | +| ------------- | ------------------------------ | -------: | +| `criterion_1` | Secrets and private data | 6 | +| `criterion_2` | Worker subprocess environments | 6 | +| `criterion_3` | Sanitized API errors | 6 | +| `criterion_4` | Agent setup prose | 6 | +| `criterion_5` | Independent async operations | 6 | +| `criterion_6` | UI and setup copy | 6 | +| `criterion_7` | Lowercase ordinary nouns | 11 | + +IDs currently follow the order in `JUDGE.json`. Preserve those IDs explicitly if +reordering rules, and update examples when changing the intended rule behavior. +The `path` in each example matches the real rule's file globs. Before/after strings +are synthetic source files; they are judged as diffs and are never executed. + +## Run locally + +Roomote currently pins Judgement 0.1.5, which predates the calibration API. Use a +local checkout of [Judgement's calibration branch](https://github.com/RooCodeInc/judgement/pull/1) +until the new package is released and Roomote upgrades. Install that checkout's +dependencies first. From the Roomote root: + +```sh +# Validate the fixtures and plan checks without inference. +pnpm exec tsx .judgement/run-calibration.mjs \ + --judgement-source ../judgement --dry-run --repeats 1 + +# Run all rules at their configured thresholds, three times per example. +pnpm exec dotenvx run --quiet -f .env.local -- \ + pnpm exec tsx .judgement/run-calibration.mjs \ + --judgement-source ../judgement --output .judgement/results/current.json + +# Compare thresholds for one rule. +pnpm exec dotenvx run --quiet -f .env.local -- \ + pnpm exec tsx .judgement/run-calibration.mjs \ + --judgement-source ../judgement --rule criterion_2 \ + --thresholds 0.8,0.85,0.9 --output .judgement/results/worker-sweep.json +``` + +Replace `../judgement` with the checkout path. Once the installed package exports +`calibrate`, that flag can be omitted. The runner uses Roomote's existing inference +adapter, model settings, and server credentials. Live runs make paid inference +requests. They run only when explicitly invoked, never during a commit hook. + +The runner copies the working-tree `JUDGE.json` into a disposable +`.judgement/rules.json` for the new harness. This is a temporary bridge for +Roomote's pinned hook; it does not add legacy lookup to Judgement. Migrate the real +policy and remove this copy step when upgrading Roomote to the new package. +Neither calibration nor threshold comparisons modify the real policy or index. + +Checks use two concurrent fixtures, a three-second deadline, and no verdict cache. +Each example can make more than one model request if context needs expanding. +Use `--repeats ` to change repetitions and `--examples-dir ` to run a separate +held-out set. Save tuning and held-out sets separately; the current set is a small +initial calibration suite, not independent proof of generalization. + +Output includes each score and context expansion, plus caught violations, false +blocks, complete valid passes, incomplete results, and timing. Optional JSON output +includes policy and fixture hashes. `results/` is ignored by Git. + +Exit codes: + +- `0`: every observed result matches its label, or a dry run finished. +- `1`: at least one result is wrong or incomplete, or inference failed. +- `2`: setup, input, or configuration failed. +- `130`: interrupted. + +For a sweep, exit `0` requires every candidate's results to match, even when the +harness recommends one of the candidates. Inspect the table before adopting a +threshold. A recommendation can still leave valid examples incomplete. + +## Coverage limits + +The privacy rule explicitly allows synthetic credentials and fictional identities. +Its fixtures are therefore **all valid**. Do not relabel a placeholder as a secret +violation just to get a positive test, and never add real credentials or private +data as fixtures. These examples measure false alarms only; they do not establish +secret-detection recall or justify a new privacy threshold. + +The other rules include violations, allowed exceptions, and fixes to existing +violations. Rule 7 also checks identifiers, sentence starts, title case, exact UI +labels, and an unrelated edit near an unchanged capitalization violation. + +See [the initial results](calibration-results.md) for observed gaps. Incomplete +checks allow hook commits but fail strict checks, so they are not successful +violation detection or complete approval of a valid edit. diff --git a/.judgement/calibration-results.md b/.judgement/calibration-results.md new file mode 100644 index 0000000000..e361b3ec71 --- /dev/null +++ b/.judgement/calibration-results.md @@ -0,0 +1,95 @@ +# Initial calibration results + +This suite exposes real gaps; it does not certify that the current rules catch +all violations. No production rule text or thresholds were changed. + +## Setup + +- 47 labeled examples, three repetitions each: 141 checks at current thresholds. +- Existing Roomote inference adapter and local model configuration; no verdict cache. +- Full Judgement checks, including context expansion, two concurrent fixtures and a three-second deadline. +- Judgement calibration implementation: `8c54991817bc42106c505721c007c1aa645206e7`. +- Policy SHA-256: `28e10de1f28460924c471d8abd03337ae1b47083edb5679fed129ab5884c07ba`. +- The runner returned exit 1 because incomplete results do not satisfy the expected labels. +- No operational failures or confidently incorrect passes were observed. + +## Current thresholds + +Each numerator/denominator counts runs, not unique examples. “Valid passes” means +a complete checker pass; the incomplete columns show the remaining cases. + +| Rule | Threshold | Violations caught | False blocks | Valid passes | Incomplete violations | Incomplete valid | Mean check ms | +| ----------- | --------: | ----------------: | -----------: | -----------: | --------------------: | ---------------: | ------------: | +| criterion_1 | 0.96 | Not tested | 0/18 | 12/18 | 0 | 6 | 763 | +| criterion_2 | 0.95 | 3/6 | 0/12 | 0/12 | 3 | 12 | 587 | +| criterion_3 | 0.95 | 7/9 | 0/9 | 9/9 | 2 | 0 | 419 | +| criterion_4 | 0.94 | 6/6 | 0/12 | 6/12 | 0 | 6 | 357 | +| criterion_5 | 0.97 | 3/6 | 0/12 | 3/12 | 3 | 9 | 518 | +| criterion_6 | 0.95 | 6/6 | 0/12 | 9/12 | 0 | 3 | 438 | +| criterion_7 | 0.85 | 9/9 | 0/24 | 15/24 | 0 | 9 | 418 | + +Overall: **34/42 violating runs blocked, 0/99 valid runs falsely blocked, +54/99 valid runs fully passed, and 53 checks incomplete**. Those incomplete +checks comprise eight violations and 45 valid changes. The hook permits incomplete +checks; a strict check rejects them. Neither outcome is equivalent to a correct +classification. Timing includes Git setup within each check and context expansion, +but excludes disposable-fixture preparation; it is not whole-commit latency. + +A preceding run on the same examples caught 33/42 violations and passed 55/99 +valid runs. This variation is another reason to repeat evaluation and avoid +treating an exact confidence value as a guarantee. + +## What needs work + +- **Privacy:** no real private data was used. There are no positive secret cases, + so detection recall is untested. Public config and environment-managed credential + lookup were valid but incomplete at 0.96. +- **Worker environments:** spreading all launcher variables was blocked. Explicitly + forwarding worker authentication remained incomplete, as did all valid exceptions. +- **API errors:** raw messages and stacks were blocked. Returning the whole exception + object was inconsistent at 0.95. +- **Agent setup prose:** both violations were blocked. Allowed internal tool schemas + and trusted UI descriptions remained incomplete. +- **Async operations:** serial independent queries were blocked. Moving a read before + an unrelated early return was missed, with low-confidence `pass` answers. Dependent + queries, transactions and explicit rate limits also remained incomplete. Lowering + a threshold cannot repair a wrong outcome; test clearer or narrower rules. +- **UI copy:** both violations were blocked. Necessary permission consequences remained + incomplete despite being allowed. +- **Capitalization:** all three violating examples were blocked. Title-case headings + and unrelated edits near unchanged violations remained incomplete; code identifiers + and sentence-start cases can vary between complete and incomplete. + +## Candidate threshold comparisons + +Each candidate was tested with three repetitions per example through the same full +checker. These are tuning results on the original examples, not held-out validation. + +| Rule | Candidate | Violations caught | False blocks | Valid passes | Incomplete valid | +| ----------- | --------: | ----------------: | -----------: | -----------: | ---------------: | +| criterion_2 | 0.80 | 6/6 | 0/12 | 12/12 | 0 | +| criterion_2 | 0.85 | 6/6 | 0/12 | 10/12 | 2 | +| criterion_2 | 0.90 | 6/6 | 0/12 | 5/12 | 7 | +| criterion_3 | 0.80 | 9/9 | 0/9 | 9/9 | 0 | +| criterion_3 | 0.85 | 9/9 | 0/9 | 9/9 | 0 | +| criterion_3 | 0.90 | 9/9 | 0/9 | 9/9 | 0 | + +The harness selected **0.80 for worker environments** and **0.90 for API errors**. +Those candidates caught every labeled violation with complete passes for every +valid run in their respective sweeps. All 108 sweep checks completed without +operational failures. They are candidates for a separate policy change after +adding held-out examples, especially close allowed exceptions. + +The worker sweep exited 1 because its other candidates still had incomplete +valid runs. The API-error sweep exited 0 because every candidate matched every +expected result. The threshold recommendation and suite exit code answer different +questions. + +## Reproduce and extend + +Use the commands in [README.md](README.md). Reports include per-run answers and +confidence values plus hashes of the exact policy and fixture input. Keep reports +under the ignored `results/` directory. Add independently labeled held-out cases +before selecting new thresholds, then rerun after rule, prompt, model, or backend +changes. Small examples cannot establish coverage for large commits or missing +supporting context; add representative cases for those separately. diff --git a/.judgement/examples/criterion_1.json b/.judgement/examples/criterion_1.json new file mode 100644 index 0000000000..ad6a578b47 --- /dev/null +++ b/.judgement/examples/criterion_1.json @@ -0,0 +1,47 @@ +{ + "ruleId": "criterion_1", + "examples": [ + { + "name": "Clearly synthetic credential placeholder", + "path": ".env.example", + "before": "TOKEN=\n", + "after": "TOKEN=REPLACE_WITH_YOUR_TOKEN\n", + "expected": "pass" + }, + { + "name": "Server-side credential lookup", + "path": "apps/api/src/config.ts", + "before": "export const config = {};\n", + "after": "export const config = { apiKey: process.env.PROVIDER_API_KEY };\n", + "expected": "pass" + }, + { + "name": "Redacted diagnostic example", + "path": "docs/troubleshooting.md", + "before": "Example output:\n", + "after": "Example output: {\"authorization\":\"[REDACTED]\"}\n", + "expected": "pass" + }, + { + "name": "Public configuration", + "path": "src/config.ts", + "before": "export const options = {};\n", + "after": "export const options = { timeoutMs: 3000, retries: 2 };\n", + "expected": "pass" + }, + { + "name": "Unmistakably fictional fixture", + "path": "test/users.fixture.ts", + "before": "export const users = [];\n", + "after": "export const users = [{ name: \"Synthetic Test User\", email: \"test-user@example.invalid\" }];\n", + "expected": "pass" + }, + { + "name": "Remove a placeholder from an example", + "path": ".env.example", + "before": "API_KEY=REPLACE_WITH_YOUR_KEY\n", + "after": "API_KEY=\n", + "expected": "pass" + } + ] +} diff --git a/.judgement/examples/criterion_2.json b/.judgement/examples/criterion_2.json new file mode 100644 index 0000000000..0512802b42 --- /dev/null +++ b/.judgement/examples/criterion_2.json @@ -0,0 +1,47 @@ +{ + "ruleId": "criterion_2", + "examples": [ + { + "name": "Spread all launcher variables", + "path": "apps/worker/src/run-task/launch.ts", + "before": "export function taskEnv() { return { PATH: process.env.PATH }; }\n", + "after": "export function taskEnv() { return { ...process.env }; }\n", + "expected": "violation" + }, + { + "name": "Forward worker authentication to task", + "path": "apps/worker/src/run-task/launch.ts", + "before": "export function taskEnv() { return { PATH: process.env.PATH }; }\n", + "after": "export function taskEnv() { return { PATH: process.env.PATH, WORKER_AUTH_TOKEN: process.env.WORKER_AUTH_TOKEN }; }\n", + "expected": "violation" + }, + { + "name": "Explicit non-sensitive allowlist", + "path": "apps/worker/src/run-task/launch.ts", + "before": "export function taskEnv() { return {}; }\n", + "after": "export function taskEnv() { return { PATH: process.env.PATH, LANG: process.env.LANG }; }\n", + "expected": "pass" + }, + { + "name": "Named nested runtime contract", + "path": "apps/worker/src/run-task/launch.ts", + "before": "export function taskEnv() { return { PATH: process.env.PATH }; }\n", + "after": "// The nested Node service requires NODE_OPTIONS to set its heap limit.\nexport function taskEnv() { return { PATH: process.env.PATH, NODE_OPTIONS: \"--max-old-space-size=512\" }; }\n", + "expected": "pass" + }, + { + "name": "Scrub captured worker token before launch", + "path": "apps/worker/src/run-task/launch.ts", + "before": "export function launch(spawn, binary, capturedEnv) { return spawn(binary, [], { env: capturedEnv }); }\n", + "after": "export function launch(spawn, binary, capturedEnv) {\n const env = { PATH: capturedEnv.PATH, LANG: capturedEnv.LANG };\n return spawn(binary, [], { env });\n}\n", + "expected": "pass" + }, + { + "name": "Synthetic fixture with no inherited environment", + "path": "apps/worker/src/run-task/launch.test.ts", + "before": "export const exampleEnv = {};\n", + "after": "export const exampleEnv = { PATH: \"/synthetic/bin\", WORKER_AUTH_TOKEN: \"NON_AUTHENTICATING_TEST_PLACEHOLDER\" };\n", + "expected": "pass" + } + ] +} diff --git a/.judgement/examples/criterion_3.json b/.judgement/examples/criterion_3.json new file mode 100644 index 0000000000..9d0e1a6a09 --- /dev/null +++ b/.judgement/examples/criterion_3.json @@ -0,0 +1,47 @@ +{ + "ruleId": "criterion_3", + "examples": [ + { + "name": "Return raw exception message", + "path": "apps/api/src/handlers/project.ts", + "before": "export function failure(c, error) { return c.json({ error: \"Unable to load the project.\" }, 500); }\n", + "after": "export function failure(c, error) { return c.json({ error: error.message }, 500); }\n", + "expected": "violation" + }, + { + "name": "Return stack trace", + "path": "apps/api/src/handlers/project.ts", + "before": "export function failure(c, error) { return c.json({ error: \"Unable to load the project.\" }, 500); }\n", + "after": "export function failure(c, error) { return c.json({ error: error.stack }, 500); }\n", + "expected": "violation" + }, + { + "name": "Return full exception object", + "path": "apps/api/src/handlers/project.ts", + "before": "export function failure(c, error) { return c.json({ error: \"Request failed.\" }, 500); }\n", + "after": "export function failure(c, error) { return c.json({ error }, 500); }\n", + "expected": "violation" + }, + { + "name": "Sanitized failure response", + "path": "apps/api/src/handlers/project.ts", + "before": "export function failure(c, error) { return c.json({ error: error.message }, 500); }\n", + "after": "export function failure(c, error) { return c.json({ error: \"Unable to load the project. Try again.\" }, 500); }\n", + "expected": "pass" + }, + { + "name": "Safe not-found response", + "path": "apps/api/src/handlers/project.ts", + "before": "export function missing(c) { return c.json({}, 404); }\n", + "after": "export function missing(c) { return c.json({ error: \"Project not found.\" }, 404); }\n", + "expected": "pass" + }, + { + "name": "Safe validation message", + "path": "apps/api/src/handlers/project.ts", + "before": "export function invalid(c) { return c.json({}, 400); }\n", + "after": "export function invalid(c) { return c.json({ error: \"Enter a valid project name.\" }, 400); }\n", + "expected": "pass" + } + ] +} diff --git a/.judgement/examples/criterion_4.json b/.judgement/examples/criterion_4.json new file mode 100644 index 0000000000..b0d5911404 --- /dev/null +++ b/.judgement/examples/criterion_4.json @@ -0,0 +1,47 @@ +{ + "ruleId": "criterion_4", + "examples": [ + { + "name": "Tell user to operate an internal card", + "path": "packages/cloud-agents/src/server/setup-prompt.ts", + "before": "export const setupReply = \"Connect a repository so I can review your code.\";\n", + "after": "export const setupReply = \"Click the Configure button in the setup card on the right rail.\";\n", + "expected": "violation" + }, + { + "name": "Ask for credentials in chat", + "path": "packages/cloud-agents/src/server/setup-prompt.ts", + "before": "export const setupReply = \"Connect your account securely to continue.\";\n", + "after": "export const setupReply = \"Paste your API key into this chat so I can connect your account.\";\n", + "expected": "violation" + }, + { + "name": "State needed capability", + "path": "packages/cloud-agents/src/server/setup-prompt.ts", + "before": "export const setupReply = \"I need access.\";\n", + "after": "export const setupReply = \"Connect a repository so I can review your code.\";\n", + "expected": "pass" + }, + { + "name": "State user outcome", + "path": "packages/cloud-agents/src/server/setup-prompt.ts", + "before": "export const setupReply = \"Configuration complete.\";\n", + "after": "export const setupReply = \"I can now review pull requests in your repository.\";\n", + "expected": "pass" + }, + { + "name": "Internal tool schema may describe controls", + "path": "packages/cloud-agents/src/server/setup-prompt.ts", + "before": "export const toolDescription = \"Configure access.\";\n", + "after": "// Internal tool schema, never shown as agent prose.\nexport const toolDescription = \"Render the trusted repository configuration card with its Connect button.\";\n", + "expected": "pass" + }, + { + "name": "Trusted UI description may describe controls", + "path": "packages/cloud-agents/src/server/setup-prompt.ts", + "before": "export const trustedUiDescription = \"Connection settings\";\n", + "after": "// Trusted UI description for the setup renderer; not agent prose.\nexport const trustedUiDescription = \"The Connect button opens the repository permissions dialog.\";\n", + "expected": "pass" + } + ] +} diff --git a/.judgement/examples/criterion_5.json b/.judgement/examples/criterion_5.json new file mode 100644 index 0000000000..745a735001 --- /dev/null +++ b/.judgement/examples/criterion_5.json @@ -0,0 +1,47 @@ +{ + "ruleId": "criterion_5", + "examples": [ + { + "name": "Serialize independent read requests", + "path": "apps/web/src/server/dashboard.ts", + "before": "export async function load(db) {\n return Promise.all([db.projects.findMany(), db.notifications.findMany()]);\n}\n", + "after": "export async function load(db) {\n const projects = await db.projects.findMany();\n const notifications = await db.notifications.findMany();\n return [projects, notifications];\n}\n", + "expected": "violation" + }, + { + "name": "Await work before unrelated early return", + "path": "apps/web/src/server/dashboard.ts", + "before": "export async function load(fetchProjects, showProjects) {\n if (!showProjects) return [];\n return fetchProjects();\n}\n", + "after": "export async function load(fetchProjects, showProjects) {\n const projects = await fetchProjects();\n if (!showProjects) return [];\n return projects;\n}\n", + "expected": "violation" + }, + { + "name": "Parallel independent reads", + "path": "apps/web/src/server/dashboard.ts", + "before": "export async function load(db) {\n const projects = await db.projects.findMany();\n const notifications = await db.notifications.findMany();\n return [projects, notifications];\n}\n", + "after": "export async function load(db) {\n return Promise.all([db.projects.findMany(), db.notifications.findMany()]);\n}\n", + "expected": "pass" + }, + { + "name": "Second query needs first result", + "path": "apps/web/src/server/dashboard.ts", + "before": "export async function load(db, slug) { return db.projects.findBySlug(slug); }\n", + "after": "export async function load(db, slug) {\n const project = await db.projects.findBySlug(slug);\n return db.tasks.findMany({ projectId: project.id });\n}\n", + "expected": "pass" + }, + { + "name": "Preserve transaction order", + "path": "apps/web/src/server/dashboard.ts", + "before": "export async function update(db, id) { return db.project.update(id); }\n", + "after": "export async function update(db, id) {\n return db.transaction(async tx => {\n await tx.project.lock(id);\n return tx.project.update(id);\n });\n}\n", + "expected": "pass" + }, + { + "name": "Explicit rate-limited operation", + "path": "apps/web/src/server/dashboard.ts", + "before": "export async function send(sendOne, ids) { return ids; }\n", + "after": "// Provider contract permits one outstanding request at a time.\nexport async function send(sendOne, ids) {\n for (const id of ids) await sendOne(id);\n}\n", + "expected": "pass" + } + ] +} diff --git a/.judgement/examples/criterion_6.json b/.judgement/examples/criterion_6.json new file mode 100644 index 0000000000..0fc35991ab --- /dev/null +++ b/.judgement/examples/criterion_6.json @@ -0,0 +1,47 @@ +{ + "ruleId": "criterion_6", + "examples": [ + { + "name": "Describe implementation in setup copy", + "path": "apps/web/src/components/setup-copy.ts", + "before": "export const description = \"Get updates when work is ready.\";\n", + "after": "export const description = \"This feature is default off and uses a polling worker to materialize event records in a Redis queue.\";\n", + "expected": "violation" + }, + { + "name": "Expose internal mechanism in task description", + "path": "apps/web/src/components/setup-copy.ts", + "before": "export const description = \"Review your latest code changes.\";\n", + "after": "export const description = \"This task hydrates the worker state, dispatches an internal tool schema, and serializes the result into our event envelope.\";\n", + "expected": "violation" + }, + { + "name": "Explain user value", + "path": "apps/web/src/components/setup-copy.ts", + "before": "export const description = \"Configure notifications.\";\n", + "after": "export const description = \"Get updates when your work is ready.\";\n", + "expected": "pass" + }, + { + "name": "Necessary permission consequence", + "path": "apps/web/src/components/setup-copy.ts", + "before": "export const permissionHelp = \"Connect your repository.\";\n", + "after": "export const permissionHelp = \"Allow read access so your agent can review code. It cannot push changes without write access.\";\n", + "expected": "pass" + }, + { + "name": "Actionable recovery message", + "path": "apps/web/src/components/setup-copy.ts", + "before": "export const errorMessage = \"Failed.\";\n", + "after": "export const errorMessage = \"Connection expired. Reconnect your account to continue.\";\n", + "expected": "pass" + }, + { + "name": "Necessary security copy", + "path": "apps/web/src/components/setup-copy.ts", + "before": "export const securityHelp = \"Account settings\";\n", + "after": "export const securityHelp = \"Revoking this token immediately stops access for apps that use it.\";\n", + "expected": "pass" + } + ] +} diff --git a/.judgement/examples/criterion_7.json b/.judgement/examples/criterion_7.json new file mode 100644 index 0000000000..6dc258fe8f --- /dev/null +++ b/.judgement/examples/criterion_7.json @@ -0,0 +1,82 @@ +{ + "ruleId": "criterion_7", + "examples": [ + { + "name": "Mid-sentence session capital", + "path": "docs/guide.md", + "before": "Review the remaining session checks before release.\n", + "after": "Review the remaining Session checks before release.\n", + "expected": "violation" + }, + { + "name": "Mid-sentence task capital", + "path": "docs/guide.md", + "before": "You can resume the task from the sidebar.\n", + "after": "You can resume the Task from the sidebar.\n", + "expected": "violation" + }, + { + "name": "UI prose with capitalized ordinary noun", + "path": "apps/web/src/components/status-copy.ts", + "before": "export const message = \"The task is ready to resume.\";\n", + "after": "export const message = \"The Task is ready to resume.\";\n", + "expected": "violation" + }, + { + "name": "Lowercase ordinary nouns", + "path": "docs/guide.md", + "before": "Review the remaining checks.\n", + "after": "Review the remaining session and task checks.\n", + "expected": "pass" + }, + { + "name": "Sentence begins with session", + "path": "docs/guide.md", + "before": "History is available.\n", + "after": "Session history is available.\n", + "expected": "pass" + }, + { + "name": "Sentence begins with task", + "path": "docs/guide.md", + "before": "Checks are complete.\n", + "after": "Task checks are complete.\n", + "expected": "pass" + }, + { + "name": "Title case heading", + "path": "docs/guide.md", + "before": "## Release Checks\n", + "after": "## Task and Session Checks\n", + "expected": "pass" + }, + { + "name": "Exact quoted UI label", + "path": "docs/guide.md", + "before": "Click the button to continue.\n", + "after": "Click the exact UI label \"New Task\" to continue.\n", + "expected": "pass" + }, + { + "name": "Code identifier", + "path": "src/types.ts", + "before": "export type RecordId = string;\n", + "after": "export type TaskId = string;\n", + "expected": "pass" + }, + { + "name": "Fix prior capitalization", + "path": "docs/guide.md", + "before": "Open the Task to review its Session history.\n", + "after": "Open the task to review its session history.\n", + "expected": "pass" + }, + { + "name": "Do not attribute unchanged violation to new diff", + "path": "docs/guide.md", + "before": "Open the Task to review history.\nWait for completion.\n", + "after": "Open the Task to review history.\nWait for completion before closing.\n", + "expected": "pass" + } + ] +} diff --git a/.judgement/run-calibration.mjs b/.judgement/run-calibration.mjs new file mode 100644 index 0000000000..489e2fe76d --- /dev/null +++ b/.judgement/run-calibration.mjs @@ -0,0 +1,151 @@ +// Run with tsx so the existing Roomote inference adapter can load TypeScript. +import { createHash } from 'node:crypto'; +import { mkdtemp, mkdir, readFile, rm, writeFile } from 'node:fs/promises'; +import { createRequire } from 'node:module'; +import { tmpdir } from 'node:os'; +import { dirname, join, resolve } from 'node:path'; +import { pathToFileURL } from 'node:url'; +import { parseArgs } from 'node:util'; + +async function main() { + const { values } = parseArgs({ + options: { + cwd: { type: 'string' }, + rule: { type: 'string' }, + repeats: { type: 'string' }, + thresholds: { type: 'string' }, + 'examples-dir': { type: 'string' }, + 'judgement-source': { type: 'string' }, + output: { type: 'string' }, + 'dry-run': { type: 'boolean' }, + help: { type: 'boolean' }, + }, + }); + if (values.help) { + console.log( + 'Run from the Roomote root with dotenvx and tsx. Options: --rule , --repeats , --thresholds , --examples-dir , --judgement-source , --output , --dry-run. See .judgement/README.md.', + ); + return 0; + } + const root = resolve(values.cwd ?? '.'); + const require = createRequire( + join(root, 'packages/cloud-agents/package.json'), + ); + const modulePath = values['judgement-source'] + ? resolve(values['judgement-source'], 'src/index.js') + : require.resolve('@roo-code/judgement'); + const judgement = await import(pathToFileURL(modulePath).href); + if (typeof judgement.calibrate !== 'function') { + throw new Error( + 'The installed Judgement version has no calibration API. Use --judgement-source /path/to/judgement with the calibration branch.', + ); + } + // Roomote still pins 0.1.5. Stage its policy in the new harness layout without + // changing the real policy or index. Remove this bridge when upgrading the hook. + const policyText = await readFile(join(root, 'JUDGE.json'), 'utf8'); + const policy = judgement.parsePolicy(policyText); + const criteria = policy.criteria.filter( + (rule) => !values.rule || rule.id === values.rule, + ); + if (!criteria.length) throw new Error('Unknown rule ID.'); + if (values.thresholds?.split(',').some((value) => !value.trim())) { + throw new Error('Thresholds must be comma-separated numbers.'); + } + const thresholds = values.thresholds?.split(',').map(Number); + const examplesDir = resolve( + root, + values['examples-dir'] ?? '.judgement/examples', + ); + const fixtures = await Promise.all( + criteria.map(async (rule) => { + const path = join(examplesDir, `${rule.id}.json`); + return { rule, path, text: await readFile(path, 'utf8') }; + }), + ); + const controller = new AbortController(); + const interrupt = () => controller.abort(); + process.once('SIGINT', interrupt); + const cwd = await mkdtemp(join(tmpdir(), 'roomote-calibration-')); + const reports = []; + try { + await mkdir(join(cwd, '.judgement')); + await writeFile(join(cwd, '.judgement/rules.json'), policyText); + const adapter = values['dry-run'] + ? null + : await import( + pathToFileURL( + join(root, 'packages/cloud-agents/src/server/judge-file.ts'), + ).href + ); + for (const [index, { rule, text }] of fixtures.entries()) { + const path = join(cwd, '.judgement', `fixture-${index}.json`); + await writeFile(path, text); + const report = await judgement.calibrate({ + cwd, + ruleId: rule.id, + examplesPath: path, + repeats: Number(values.repeats ?? 3), + thresholds: thresholds ?? [rule.threshold], + concurrency: 2, + deadlineMs: 3000, + dryRun: values['dry-run'], + signal: controller.signal, + evaluate: async (request, signal) => { + signal.throwIfAborted(); + const answer = await adapter.evaluateRepositoryJudgement(request); + if (!answer) throw new Error('Roomote evaluator unavailable'); + return answer; + }, + }); + reports.push(report); + console.log(judgement.formatCalibrationReport(report)); + } + if (values.output) { + const output = resolve(root, values.output); + await mkdir(dirname(output), { recursive: true }); + const hash = (text) => createHash('sha256').update(text).digest('hex'); + await writeFile( + output, + JSON.stringify( + { + recordedAt: new Date().toISOString(), + policySha256: hash(policyText), + fixtures: Object.fromEntries( + fixtures.map(({ rule, text }) => [rule.id, hash(text)]), + ), + reports, + }, + null, + 2, + ) + '\n', + ); + } + // This suite checks expected behavior, including complete passes for valid + // changes. A threshold recommendation alone can still leave valid cases incomplete. + return values['dry-run'] || + reports.every((report) => + report.results.every( + (run) => !run.operationalFailure && run.status === run.expected, + ), + ) + ? 0 + : 1; + } finally { + process.removeListener('SIGINT', interrupt); + await rm(cwd, { recursive: true, force: true }); + } +} + +// Inference settings open shared database pools. Exit after cleanup and reporting. +main() + .then((code) => process.exit(code)) + .catch((error) => { + console.error( + error.name === 'AbortError' + ? 'Calibration interrupted.' + : 'Calibration failed. Check the rule, fixtures, Judgement source, and local inference setup.', + ); + if (error.message?.startsWith('The installed Judgement version')) + console.error(error.message); + process.exit(error.name === 'AbortError' ? 130 : 2); + }); From 3182431ce466af480e10f8ac475b015e24ce564c Mon Sep 17 00:00:00 2001 From: Matt Rubens <2600+mrubens@users.noreply.github.com> Date: Tue, 29 Sep 2026 15:52:48 -0400 Subject: [PATCH 2/3] Keep rule examples in Roomote and move tooling to Judgement --- .judgement/README.md | 64 ++----------- .judgement/calibration-results.md | 95 ------------------- .judgement/run-calibration.mjs | 151 ------------------------------ 3 files changed, 7 insertions(+), 303 deletions(-) delete mode 100644 .judgement/calibration-results.md delete mode 100644 .judgement/run-calibration.mjs diff --git a/.judgement/README.md b/.judgement/README.md index 62c6d1c94f..31f13866b9 100644 --- a/.judgement/README.md +++ b/.judgement/README.md @@ -19,61 +19,12 @@ reordering rules, and update examples when changing the intended rule behavior. The `path` in each example matches the real rule's file globs. Before/after strings are synthetic source files; they are judged as diffs and are never executed. -## Run locally +## Running examples -Roomote currently pins Judgement 0.1.5, which predates the calibration API. Use a -local checkout of [Judgement's calibration branch](https://github.com/RooCodeInc/judgement/pull/1) -until the new package is released and Roomote upgrades. Install that checkout's -dependencies first. From the Roomote root: - -```sh -# Validate the fixtures and plan checks without inference. -pnpm exec tsx .judgement/run-calibration.mjs \ - --judgement-source ../judgement --dry-run --repeats 1 - -# Run all rules at their configured thresholds, three times per example. -pnpm exec dotenvx run --quiet -f .env.local -- \ - pnpm exec tsx .judgement/run-calibration.mjs \ - --judgement-source ../judgement --output .judgement/results/current.json - -# Compare thresholds for one rule. -pnpm exec dotenvx run --quiet -f .env.local -- \ - pnpm exec tsx .judgement/run-calibration.mjs \ - --judgement-source ../judgement --rule criterion_2 \ - --thresholds 0.8,0.85,0.9 --output .judgement/results/worker-sweep.json -``` - -Replace `../judgement` with the checkout path. Once the installed package exports -`calibrate`, that flag can be omitted. The runner uses Roomote's existing inference -adapter, model settings, and server credentials. Live runs make paid inference -requests. They run only when explicitly invoked, never during a commit hook. - -The runner copies the working-tree `JUDGE.json` into a disposable -`.judgement/rules.json` for the new harness. This is a temporary bridge for -Roomote's pinned hook; it does not add legacy lookup to Judgement. Migrate the real -policy and remove this copy step when upgrading Roomote to the new package. -Neither calibration nor threshold comparisons modify the real policy or index. - -Checks use two concurrent fixtures, a three-second deadline, and no verdict cache. -Each example can make more than one model request if context needs expanding. -Use `--repeats ` to change repetitions and `--examples-dir ` to run a separate -held-out set. Save tuning and held-out sets separately; the current set is a small -initial calibration suite, not independent proof of generalization. - -Output includes each score and context expansion, plus caught violations, false -blocks, complete valid passes, incomplete results, and timing. Optional JSON output -includes policy and fixture hashes. `results/` is ignored by Git. - -Exit codes: - -- `0`: every observed result matches its label, or a dry run finished. -- `1`: at least one result is wrong or incomplete, or inference failed. -- `2`: setup, input, or configuration failed. -- `130`: interrupted. - -For a sweep, exit `0` requires every candidate's results to match, even when the -harness recommends one of the candidates. Inspect the table before adopting a -threshold. A recommendation can still leave valid examples incomplete. +Use Judgement's shared test and calibration commands once Roomote upgrades to the +release containing them. Roomote currently pins 0.1.5. The command documentation +and implementation belong in [Judgement](https://github.com/RooCodeInc/judgement). +Keep generated reports local; `results/` is ignored by Git. ## Coverage limits @@ -87,6 +38,5 @@ The other rules include violations, allowed exceptions, and fixes to existing violations. Rule 7 also checks identifiers, sentence starts, title case, exact UI labels, and an unrelated edit near an unchanged capitalization violation. -See [the initial results](calibration-results.md) for observed gaps. Incomplete -checks allow hook commits but fail strict checks, so they are not successful -violation detection or complete approval of a valid edit. +Incomplete checks allow hook commits but fail strict checks. They do not count as +successful violation detection or complete approval of a valid edit. diff --git a/.judgement/calibration-results.md b/.judgement/calibration-results.md deleted file mode 100644 index e361b3ec71..0000000000 --- a/.judgement/calibration-results.md +++ /dev/null @@ -1,95 +0,0 @@ -# Initial calibration results - -This suite exposes real gaps; it does not certify that the current rules catch -all violations. No production rule text or thresholds were changed. - -## Setup - -- 47 labeled examples, three repetitions each: 141 checks at current thresholds. -- Existing Roomote inference adapter and local model configuration; no verdict cache. -- Full Judgement checks, including context expansion, two concurrent fixtures and a three-second deadline. -- Judgement calibration implementation: `8c54991817bc42106c505721c007c1aa645206e7`. -- Policy SHA-256: `28e10de1f28460924c471d8abd03337ae1b47083edb5679fed129ab5884c07ba`. -- The runner returned exit 1 because incomplete results do not satisfy the expected labels. -- No operational failures or confidently incorrect passes were observed. - -## Current thresholds - -Each numerator/denominator counts runs, not unique examples. “Valid passes” means -a complete checker pass; the incomplete columns show the remaining cases. - -| Rule | Threshold | Violations caught | False blocks | Valid passes | Incomplete violations | Incomplete valid | Mean check ms | -| ----------- | --------: | ----------------: | -----------: | -----------: | --------------------: | ---------------: | ------------: | -| criterion_1 | 0.96 | Not tested | 0/18 | 12/18 | 0 | 6 | 763 | -| criterion_2 | 0.95 | 3/6 | 0/12 | 0/12 | 3 | 12 | 587 | -| criterion_3 | 0.95 | 7/9 | 0/9 | 9/9 | 2 | 0 | 419 | -| criterion_4 | 0.94 | 6/6 | 0/12 | 6/12 | 0 | 6 | 357 | -| criterion_5 | 0.97 | 3/6 | 0/12 | 3/12 | 3 | 9 | 518 | -| criterion_6 | 0.95 | 6/6 | 0/12 | 9/12 | 0 | 3 | 438 | -| criterion_7 | 0.85 | 9/9 | 0/24 | 15/24 | 0 | 9 | 418 | - -Overall: **34/42 violating runs blocked, 0/99 valid runs falsely blocked, -54/99 valid runs fully passed, and 53 checks incomplete**. Those incomplete -checks comprise eight violations and 45 valid changes. The hook permits incomplete -checks; a strict check rejects them. Neither outcome is equivalent to a correct -classification. Timing includes Git setup within each check and context expansion, -but excludes disposable-fixture preparation; it is not whole-commit latency. - -A preceding run on the same examples caught 33/42 violations and passed 55/99 -valid runs. This variation is another reason to repeat evaluation and avoid -treating an exact confidence value as a guarantee. - -## What needs work - -- **Privacy:** no real private data was used. There are no positive secret cases, - so detection recall is untested. Public config and environment-managed credential - lookup were valid but incomplete at 0.96. -- **Worker environments:** spreading all launcher variables was blocked. Explicitly - forwarding worker authentication remained incomplete, as did all valid exceptions. -- **API errors:** raw messages and stacks were blocked. Returning the whole exception - object was inconsistent at 0.95. -- **Agent setup prose:** both violations were blocked. Allowed internal tool schemas - and trusted UI descriptions remained incomplete. -- **Async operations:** serial independent queries were blocked. Moving a read before - an unrelated early return was missed, with low-confidence `pass` answers. Dependent - queries, transactions and explicit rate limits also remained incomplete. Lowering - a threshold cannot repair a wrong outcome; test clearer or narrower rules. -- **UI copy:** both violations were blocked. Necessary permission consequences remained - incomplete despite being allowed. -- **Capitalization:** all three violating examples were blocked. Title-case headings - and unrelated edits near unchanged violations remained incomplete; code identifiers - and sentence-start cases can vary between complete and incomplete. - -## Candidate threshold comparisons - -Each candidate was tested with three repetitions per example through the same full -checker. These are tuning results on the original examples, not held-out validation. - -| Rule | Candidate | Violations caught | False blocks | Valid passes | Incomplete valid | -| ----------- | --------: | ----------------: | -----------: | -----------: | ---------------: | -| criterion_2 | 0.80 | 6/6 | 0/12 | 12/12 | 0 | -| criterion_2 | 0.85 | 6/6 | 0/12 | 10/12 | 2 | -| criterion_2 | 0.90 | 6/6 | 0/12 | 5/12 | 7 | -| criterion_3 | 0.80 | 9/9 | 0/9 | 9/9 | 0 | -| criterion_3 | 0.85 | 9/9 | 0/9 | 9/9 | 0 | -| criterion_3 | 0.90 | 9/9 | 0/9 | 9/9 | 0 | - -The harness selected **0.80 for worker environments** and **0.90 for API errors**. -Those candidates caught every labeled violation with complete passes for every -valid run in their respective sweeps. All 108 sweep checks completed without -operational failures. They are candidates for a separate policy change after -adding held-out examples, especially close allowed exceptions. - -The worker sweep exited 1 because its other candidates still had incomplete -valid runs. The API-error sweep exited 0 because every candidate matched every -expected result. The threshold recommendation and suite exit code answer different -questions. - -## Reproduce and extend - -Use the commands in [README.md](README.md). Reports include per-run answers and -confidence values plus hashes of the exact policy and fixture input. Keep reports -under the ignored `results/` directory. Add independently labeled held-out cases -before selecting new thresholds, then rerun after rule, prompt, model, or backend -changes. Small examples cannot establish coverage for large commits or missing -supporting context; add representative cases for those separately. diff --git a/.judgement/run-calibration.mjs b/.judgement/run-calibration.mjs deleted file mode 100644 index 489e2fe76d..0000000000 --- a/.judgement/run-calibration.mjs +++ /dev/null @@ -1,151 +0,0 @@ -// Run with tsx so the existing Roomote inference adapter can load TypeScript. -import { createHash } from 'node:crypto'; -import { mkdtemp, mkdir, readFile, rm, writeFile } from 'node:fs/promises'; -import { createRequire } from 'node:module'; -import { tmpdir } from 'node:os'; -import { dirname, join, resolve } from 'node:path'; -import { pathToFileURL } from 'node:url'; -import { parseArgs } from 'node:util'; - -async function main() { - const { values } = parseArgs({ - options: { - cwd: { type: 'string' }, - rule: { type: 'string' }, - repeats: { type: 'string' }, - thresholds: { type: 'string' }, - 'examples-dir': { type: 'string' }, - 'judgement-source': { type: 'string' }, - output: { type: 'string' }, - 'dry-run': { type: 'boolean' }, - help: { type: 'boolean' }, - }, - }); - if (values.help) { - console.log( - 'Run from the Roomote root with dotenvx and tsx. Options: --rule , --repeats , --thresholds , --examples-dir , --judgement-source , --output , --dry-run. See .judgement/README.md.', - ); - return 0; - } - const root = resolve(values.cwd ?? '.'); - const require = createRequire( - join(root, 'packages/cloud-agents/package.json'), - ); - const modulePath = values['judgement-source'] - ? resolve(values['judgement-source'], 'src/index.js') - : require.resolve('@roo-code/judgement'); - const judgement = await import(pathToFileURL(modulePath).href); - if (typeof judgement.calibrate !== 'function') { - throw new Error( - 'The installed Judgement version has no calibration API. Use --judgement-source /path/to/judgement with the calibration branch.', - ); - } - // Roomote still pins 0.1.5. Stage its policy in the new harness layout without - // changing the real policy or index. Remove this bridge when upgrading the hook. - const policyText = await readFile(join(root, 'JUDGE.json'), 'utf8'); - const policy = judgement.parsePolicy(policyText); - const criteria = policy.criteria.filter( - (rule) => !values.rule || rule.id === values.rule, - ); - if (!criteria.length) throw new Error('Unknown rule ID.'); - if (values.thresholds?.split(',').some((value) => !value.trim())) { - throw new Error('Thresholds must be comma-separated numbers.'); - } - const thresholds = values.thresholds?.split(',').map(Number); - const examplesDir = resolve( - root, - values['examples-dir'] ?? '.judgement/examples', - ); - const fixtures = await Promise.all( - criteria.map(async (rule) => { - const path = join(examplesDir, `${rule.id}.json`); - return { rule, path, text: await readFile(path, 'utf8') }; - }), - ); - const controller = new AbortController(); - const interrupt = () => controller.abort(); - process.once('SIGINT', interrupt); - const cwd = await mkdtemp(join(tmpdir(), 'roomote-calibration-')); - const reports = []; - try { - await mkdir(join(cwd, '.judgement')); - await writeFile(join(cwd, '.judgement/rules.json'), policyText); - const adapter = values['dry-run'] - ? null - : await import( - pathToFileURL( - join(root, 'packages/cloud-agents/src/server/judge-file.ts'), - ).href - ); - for (const [index, { rule, text }] of fixtures.entries()) { - const path = join(cwd, '.judgement', `fixture-${index}.json`); - await writeFile(path, text); - const report = await judgement.calibrate({ - cwd, - ruleId: rule.id, - examplesPath: path, - repeats: Number(values.repeats ?? 3), - thresholds: thresholds ?? [rule.threshold], - concurrency: 2, - deadlineMs: 3000, - dryRun: values['dry-run'], - signal: controller.signal, - evaluate: async (request, signal) => { - signal.throwIfAborted(); - const answer = await adapter.evaluateRepositoryJudgement(request); - if (!answer) throw new Error('Roomote evaluator unavailable'); - return answer; - }, - }); - reports.push(report); - console.log(judgement.formatCalibrationReport(report)); - } - if (values.output) { - const output = resolve(root, values.output); - await mkdir(dirname(output), { recursive: true }); - const hash = (text) => createHash('sha256').update(text).digest('hex'); - await writeFile( - output, - JSON.stringify( - { - recordedAt: new Date().toISOString(), - policySha256: hash(policyText), - fixtures: Object.fromEntries( - fixtures.map(({ rule, text }) => [rule.id, hash(text)]), - ), - reports, - }, - null, - 2, - ) + '\n', - ); - } - // This suite checks expected behavior, including complete passes for valid - // changes. A threshold recommendation alone can still leave valid cases incomplete. - return values['dry-run'] || - reports.every((report) => - report.results.every( - (run) => !run.operationalFailure && run.status === run.expected, - ), - ) - ? 0 - : 1; - } finally { - process.removeListener('SIGINT', interrupt); - await rm(cwd, { recursive: true, force: true }); - } -} - -// Inference settings open shared database pools. Exit after cleanup and reporting. -main() - .then((code) => process.exit(code)) - .catch((error) => { - console.error( - error.name === 'AbortError' - ? 'Calibration interrupted.' - : 'Calibration failed. Check the rule, fixtures, Judgement source, and local inference setup.', - ); - if (error.message?.startsWith('The installed Judgement version')) - console.error(error.message); - process.exit(error.name === 'AbortError' ? 130 : 2); - }); From 71d21e7de6fcfaa4a1269fc6018038b97f602821 Mon Sep 17 00:00:00 2001 From: Matt Rubens <2600+mrubens@users.noreply.github.com> Date: Tue, 29 Sep 2026 16:09:41 -0400 Subject: [PATCH 3/3] Upgrade Judgement to 0.2.0 and wire shared rule testing --- .judgement/README.md | 47 +++++--- .judgement/examples/criterion_8.json | 75 ++++++++++++ JUDGE.json => .judgement/rules.json | 5 + LOCAL_DEVELOPMENT.md | 45 +++++--- apps/docs/tasks.mdx | 2 +- apps/worker/package.json | 2 +- .../__tests__/judgement-check.test.ts | 7 +- packages/cloud-agents/package.json | 2 +- .../server/__tests__/judgement-local.test.ts | 64 +++++++++- .../src/server/judgement-local-cli.ts | 109 +++++++++++++++++- .../src/server/judgement-local.ts | 53 ++++++--- packages/types/src/judge-policy.ts | 2 +- pnpm-lock.yaml | 14 +-- pnpm-workspace.yaml | 2 +- 14 files changed, 353 insertions(+), 76 deletions(-) create mode 100644 .judgement/examples/criterion_8.json rename JUDGE.json => .judgement/rules.json (88%) diff --git a/.judgement/README.md b/.judgement/README.md index 31f13866b9..e1db32355e 100644 --- a/.judgement/README.md +++ b/.judgement/README.md @@ -1,30 +1,41 @@ # Roomote rule calibration -These are labeled changes for each of the seven repository rules. Labels describe +These are labeled changes for each of the eight repository rules. Labels describe what the rule requires, not what the model happened to answer. Keep cases that fail: they are regression examples, not reasons to change the expected label. -| ID | Rule | Examples | -| ------------- | ------------------------------ | -------: | -| `criterion_1` | Secrets and private data | 6 | -| `criterion_2` | Worker subprocess environments | 6 | -| `criterion_3` | Sanitized API errors | 6 | -| `criterion_4` | Agent setup prose | 6 | -| `criterion_5` | Independent async operations | 6 | -| `criterion_6` | UI and setup copy | 6 | -| `criterion_7` | Lowercase ordinary nouns | 11 | - -IDs currently follow the order in `JUDGE.json`. Preserve those IDs explicitly if +| ID | Rule | Examples | +| ------------- | --------------------------------------- | -------: | +| `criterion_1` | Secrets and private data | 6 | +| `criterion_2` | Worker subprocess environments | 6 | +| `criterion_3` | Sanitized API errors | 6 | +| `criterion_4` | Agent setup prose | 6 | +| `criterion_5` | Independent async operations | 6 | +| `criterion_6` | UI and setup copy | 6 | +| `criterion_7` | Lowercase ordinary nouns | 11 | +| `criterion_8` | Direct descriptions of current behavior | 10 | + +IDs currently follow the order in `.judgement/rules.json`. Preserve those IDs explicitly if reordering rules, and update examples when changing the intended rule behavior. The `path` in each example matches the real rule's file globs. Before/after strings are synthetic source files; they are judged as diffs and are never executed. ## Running examples -Use Judgement's shared test and calibration commands once Roomote upgrades to the -release containing them. Roomote currently pins 0.1.5. The command documentation -and implementation belong in [Judgement](https://github.com/RooCodeInc/judgement). -Keep generated reports local; `results/` is ignored by Git. +Run from the repository root: + +```sh +pnpm judgement test --dry-run +pnpm judgement test --rule criterion_8 +pnpm judgement calibrate --rule criterion_8 --thresholds 0.8,0.85,0.9 +pnpm --silent judgement test --format json > .judgement/results/latest.json +``` + +Create `.judgement/results/` before redirecting output there. The commands use +Roomote's existing model settings and server credentials through the shared +Judgement library. Keep generated reports local; `results/` is ignored by Git. +See [Judgement's agent guide](https://github.com/RooCodeInc/judgement/blob/main/docs/agents.md) +for rule design, calibration, and interpreting incomplete results. ## Coverage limits @@ -36,7 +47,9 @@ secret-detection recall or justify a new privacy threshold. The other rules include violations, allowed exceptions, and fixes to existing violations. Rule 7 also checks identifiers, sentence starts, title case, exact UI -labels, and an unrelated edit near an unchanged capitalization violation. +labels, and an unrelated edit near an unchanged capitalization violation. Rule 8 +also covers migration guides, release notes, supported choices, and useful security +constraints. Incomplete checks allow hook commits but fail strict checks. They do not count as successful violation detection or complete approval of a valid edit. diff --git a/.judgement/examples/criterion_8.json b/.judgement/examples/criterion_8.json new file mode 100644 index 0000000000..d4c43b4e10 --- /dev/null +++ b/.judgement/examples/criterion_8.json @@ -0,0 +1,75 @@ +{ + "ruleId": "criterion_8", + "examples": [ + { + "name": "Unnecessary exclusivity in a save instruction", + "path": "docs/setup.md", + "before": "Save rules in `.judgement/rules.json`.\n", + "after": "Save rules in `.judgement/rules.json`, the only policy location.\n", + "expected": "violation" + }, + { + "name": "Past implementation aside in setup guide", + "path": "docs/setup.md", + "before": "Open Settings to connect your account.\n", + "after": "Open Settings to connect your account. We used to put this in the sidebar before the redesign.\n", + "expected": "violation" + }, + { + "name": "Removed alternative in ordinary setup instructions", + "path": "docs/setup.md", + "before": "Save the configuration in `config.json`.\n", + "after": "Save the configuration in `config.json`, not the old YAML file we no longer use.\n", + "expected": "violation" + }, + { + "name": "Direct save instruction", + "path": "docs/setup.md", + "before": "Create a rules file.\n", + "after": "Save rules in `.judgement/rules.json`.\n", + "expected": "pass" + }, + { + "name": "Direct setup action", + "path": "docs/setup.md", + "before": "Configure access.\n", + "after": "Open Settings to connect your account.\n", + "expected": "pass" + }, + { + "name": "Migration guide requires old and new paths", + "path": "docs/migration.md", + "before": "# Upgrade from version 1\nUpdate your configuration.\n", + "after": "# Upgrade from version 1\nMove your existing `settings.yml` to `config.json` before starting version 2.\n", + "expected": "pass" + }, + { + "name": "Release notes describe the change", + "path": "CHANGELOG.md", + "before": "## Version 2\nImprovements.\n", + "after": "## Version 2\nConfiguration now lives in `config.json`; the version 1 YAML format is no longer accepted.\n", + "expected": "pass" + }, + { + "name": "Supported choices with practical tradeoffs", + "path": "docs/setup.md", + "before": "Choose a storage option.\n", + "after": "Choose local storage for offline access or shared storage to collaborate across devices.\n", + "expected": "pass" + }, + { + "name": "Security restriction is actionable", + "path": "docs/setup.md", + "before": "Configure the callback URL.\n", + "after": "Only HTTPS callback URLs are accepted to protect credentials in transit.\n", + "expected": "pass" + }, + { + "name": "Explicit bad and good examples", + "path": "docs/writing.md", + "before": "## Writing examples\nUse direct instructions.\n", + "after": "## Writing examples\nAvoid: \"Save rules here, the only policy location.\"\nPrefer: \"Save rules here.\"\n", + "expected": "pass" + } + ] +} diff --git a/JUDGE.json b/.judgement/rules.json similarity index 88% rename from JUDGE.json rename to .judgement/rules.json index 623e66f637..bfc11c6d14 100644 --- a/JUDGE.json +++ b/.judgement/rules.json @@ -48,6 +48,11 @@ "rule": "In user-facing prose, treat \"session\" and \"task\" as ordinary nouns rather than proper nouns; use lowercase \"session\" and \"task\" except at sentence beginnings, in title case, or when quoting an exact UI label.", "files": ["**/*"], "threshold": 0.85 + }, + { + "rule": "Release notes, migration instructions, security or compatibility requirements, and quoted writing examples are exempt from this rule. In ordinary explanatory and instructional prose, state the current location, behavior, or action directly. Do not add unnecessary asides about previous iterations or removed alternatives, or unnecessary exclusivity such as \"the only policy location\". Comparisons between supported options are allowed when they help the reader choose.", + "files": ["**/*"], + "threshold": 0.85 } ] } diff --git a/LOCAL_DEVELOPMENT.md b/LOCAL_DEVELOPMENT.md index 1cc689d9ff..3bf92a5f5a 100644 --- a/LOCAL_DEVELOPMENT.md +++ b/LOCAL_DEVELOPMENT.md @@ -6,14 +6,14 @@ checkout. If you want to operate Roomote for a team, start with the public ## Prerequisites -| Tool | Version | Notes | -| ---- | ------- | ----- | -| macOS | Latest | This guide is optimized for macOS. Apple Silicon runs the task worker natively (no amd64 emulation). | -| Docker Desktop | Latest | Provides Docker Engine and Compose for local services. | -| mise | Latest | Manages the repository toolchain. | -| ngrok | Latest | Optional, only when `R_PUBLIC_URL` uses an ngrok domain. | -| OpenCode CLI | Latest | Required to run tasks and server-side model calls. Install with `npm install -g opencode-ai`. | -| Python 3 | Latest | Required by the server-side OpenCode CLI helper. Install with `brew install python` if `python3` is missing. | +| Tool | Version | Notes | +| -------------- | ------- | ------------------------------------------------------------------------------------------------------------ | +| macOS | Latest | This guide is optimized for macOS. Apple Silicon runs the task worker natively (no amd64 emulation). | +| Docker Desktop | Latest | Provides Docker Engine and Compose for local services. | +| mise | Latest | Manages the repository toolchain. | +| ngrok | Latest | Optional, only when `R_PUBLIC_URL` uses an ngrok domain. | +| OpenCode CLI | Latest | Required to run tasks and server-side model calls. Install with `npm install -g opencode-ai`. | +| Python 3 | Latest | Required by the server-side OpenCode CLI helper. Install with `brew install python` if `python3` is missing. | The dev toolchain runs on Node.js 24.13.1, pinned in `.tool-versions`, `.nvmrc`, and `.node-version`. The `package.json` `engines` field accepts @@ -142,14 +142,14 @@ the `roomote-worker:local` image exists. Common options: -| Option | Default | Description | -| ------ | ------- | ----------- | -| `--reset` | `false` | Reset the database before starting services. | -| `--verbose` | `false` | Enable verbose output with stdio logging. | -| `--skip-worker-release-build` | `false` | Reuse the existing worker artifact from `./releases`. | -| `--use-release` | `false` | Use GitHub worker releases instead of the local worker build. | -| `--worker-release-channel` | `stable` | GitHub worker release channel for `--use-release`. | -| `--worker-release-version` | unset | Optional GitHub worker release version pin. | +| Option | Default | Description | +| ----------------------------- | -------- | ------------------------------------------------------------- | +| `--reset` | `false` | Reset the database before starting services. | +| `--verbose` | `false` | Enable verbose output with stdio logging. | +| `--skip-worker-release-build` | `false` | Reuse the existing worker artifact from `./releases`. | +| `--use-release` | `false` | Use GitHub worker releases instead of the local worker build. | +| `--worker-release-channel` | `stable` | GitHub worker release channel for `--use-release`. | +| `--worker-release-version` | unset | Optional GitHub worker release version pin. | Examples: @@ -292,7 +292,6 @@ curl -X GET "http://localhost:13001/trpc/auth.me" \ -H "Content-Type: application/json" | jq ``` - ### Checking staged changes with Judgement The pre-commit hook runs Judgement after `lint-staged`, so it checks the formatted @@ -303,7 +302,7 @@ the worker-installed wrapper runs Judgement once through task inference, even when the proxy is unavailable. Run `pnpm judgement --verbose` from the Roomote checkout to check staged changes -against `JUDGE.json`. The command loads `.env.local` and uses the same deployment +against `.judgement/rules.json`. The command loads `.env.local` and uses the same deployment judgment-model settings and encrypted provider credentials as the local API. The local database must be running. Select Jev through TypeSafe, OpenRouter, or Vercel AI Gateway in **Settings → Models**; `R_JUDGMENT_MODEL` overrides that choice. @@ -317,3 +316,13 @@ or `--format json` for the report as JSON. Checks are not cached because the configured inference backend can change. Managed task hooks use the same API inference settings through a worker-owned local proxy and retain their three-second deadline. + +Run `pnpm judgement test` to evaluate the labeled examples under +`.judgement/examples/` at their configured thresholds. Use `--rule criterion_8` +to select a rule and `--dry-run` to validate inputs without inference. To compare +thresholds, run `pnpm judgement calibrate --rule criterion_8 --thresholds 0.8,0.85,0.9` +or use `--all` to include every rule. Judgement owns the test and calibration +runner; Roomote supplies its inference adapter. Live runs make model requests. +Generated reports belong in local output or CI artifacts. See +[the agent guide](https://github.com/RooCodeInc/judgement/blob/main/docs/agents.md) +for labeling examples and interpreting results. diff --git a/apps/docs/tasks.mdx b/apps/docs/tasks.mdx index cc03d08ec3..65aeae425d 100644 --- a/apps/docs/tasks.mdx +++ b/apps/docs/tasks.mdx @@ -137,7 +137,7 @@ runs the Roomote dashboard. Roomote installs [Judgement](https://github.com/RooCodeInc/judgement) as a Git pre-commit check in task repositories, including repositories checked out during -a task. Add a root `JUDGE.json` to define your business rules: +a task. Add `.judgement/rules.json` to define your business rules: ```json { diff --git a/apps/worker/package.json b/apps/worker/package.json index 726771528c..4f97ccf72f 100644 --- a/apps/worker/package.json +++ b/apps/worker/package.json @@ -20,7 +20,7 @@ "dependencies": { "@hono/node-server": "2.0.10", "@modelcontextprotocol/sdk": "^1.29.0", - "@roo-code/judgement": "0.1.5", + "@roo-code/judgement": "0.2.0", "@roomote/auth": "workspace:^", "@roomote/cloud-agents": "workspace:^", "@roomote/communication": "workspace:^", diff --git a/apps/worker/src/run-task/__tests__/judgement-check.test.ts b/apps/worker/src/run-task/__tests__/judgement-check.test.ts index d0f4641f2c..aba444b075 100644 --- a/apps/worker/src/run-task/__tests__/judgement-check.test.ts +++ b/apps/worker/src/run-task/__tests__/judgement-check.test.ts @@ -1,5 +1,5 @@ import { execFile } from 'node:child_process'; -import { mkdtemp, rm, writeFile } from 'node:fs/promises'; +import { mkdtemp, mkdir, rm, writeFile } from 'node:fs/promises'; import { tmpdir } from 'node:os'; import { join } from 'node:path'; import { promisify } from 'node:util'; @@ -44,8 +44,9 @@ beforeEach(async () => { env.R_JUDGEMENT_GATEWAY_URL = proxy.endpoint; cwd = await mkdtemp(join(tmpdir(), 'roomote-judgement-')); await exec('git', ['init', '-q'], { cwd }); + await mkdir(join(cwd, '.judgement')); await writeFile( - join(cwd, 'JUDGE.json'), + join(cwd, '.judgement/rules.json'), JSON.stringify({ criteria: [{ rule: 'Use sentence case' }] }), ); await writeFile(join(cwd, 'example.ts'), 'changed'); @@ -95,7 +96,7 @@ it('reports unavailable models and missing runtime credentials as incomplete', a expect(mutate).not.toHaveBeenCalled(); }); it('does not send a model request without a repository policy', async () => { - await exec('git', ['rm', '-f', 'JUDGE.json'], { cwd }); + await exec('git', ['rm', '-f', '.judgement/rules.json'], { cwd }); expect((await checkRepositoryJudgement(env, cwd)).status).toBe('pass'); expect(mutate).not.toHaveBeenCalled(); }); diff --git a/packages/cloud-agents/package.json b/packages/cloud-agents/package.json index b3a5ce24b8..570c37d5ac 100644 --- a/packages/cloud-agents/package.json +++ b/packages/cloud-agents/package.json @@ -98,7 +98,7 @@ "@aws-sdk/client-s3": "3.1000.0", "@modelcontextprotocol/sdk": "^1.29.0", "@opencode-ai/sdk": "1.18.30", - "@roo-code/judgement": "0.1.5", + "@roo-code/judgement": "0.2.0", "@roomote/ado": "workspace:^", "@roomote/auth": "workspace:^", "@roomote/bitbucket": "workspace:^", diff --git a/packages/cloud-agents/src/server/__tests__/judgement-local.test.ts b/packages/cloud-agents/src/server/__tests__/judgement-local.test.ts index 931dd0033d..60fbc0010a 100644 --- a/packages/cloud-agents/src/server/__tests__/judgement-local.test.ts +++ b/packages/cloud-agents/src/server/__tests__/judgement-local.test.ts @@ -1,4 +1,4 @@ -import { mkdtemp, rm, writeFile } from 'node:fs/promises'; +import { mkdtemp, mkdir, rm, writeFile } from 'node:fs/promises'; import { tmpdir } from 'node:os'; import { join } from 'node:path'; import { execFile } from 'node:child_process'; @@ -15,8 +15,9 @@ beforeEach(async () => { .mockResolvedValue({ outcome: 'violation', confidence: 0.99 }); cwd = await mkdtemp(join(tmpdir(), 'judgement-local-')); await exec('git', ['init', '-q'], { cwd }); + await mkdir(join(cwd, '.judgement')); await writeFile( - join(cwd, 'JUDGE.json'), + join(cwd, '.judgement/rules.json'), JSON.stringify({ criteria: [{ rule: 'Use lowercase session.' }] }), ); await writeFile(join(cwd, 'example.md'), 'new Session checks'); @@ -54,3 +55,62 @@ it('does not load inference for a dry run', async () => { await checkLocalRepositoryJudgement({ cwd, dryRun: true }); expect(evaluate).not.toHaveBeenCalled(); }); + +it('uses the same Roomote evaluator for the shared rule example suite', async () => { + const { testRules } = await import('@roo-code/judgement'); + const { evaluateLocalRepositoryJudgement } = + await import('../judgement-local'); + await mkdir(join(cwd, '.judgement/examples')); + await writeFile( + join(cwd, '.judgement/examples/criterion_1.json'), + JSON.stringify({ + ruleId: 'criterion_1', + examples: [ + { + name: 'capital', + before: 'session', + after: 'Session', + expected: 'violation', + }, + ], + }), + ); + const report = await testRules({ + cwd, + repeats: 1, + evaluate: evaluateLocalRepositoryJudgement, + }); + expect(report.status).toBe('pass'); + expect(evaluate).toHaveBeenCalled(); + expect(report.reports[0]?.results[0]?.answers[0]?.confidence).toBe(0.99); +}); + +it('does not expose backend errors in example suite reports', async () => { + const { testRules } = await import('@roo-code/judgement'); + const { evaluateLocalRepositoryJudgement } = + await import('../judgement-local'); + await mkdir(join(cwd, '.judgement/examples')); + await writeFile( + join(cwd, '.judgement/examples/criterion_1.json'), + JSON.stringify({ + ruleId: 'criterion_1', + examples: [ + { + name: 'capital', + before: 'session', + after: 'Session', + expected: 'violation', + }, + ], + }), + ); + evaluate.mockRejectedValue(new Error('secret provider details')); + const report = await testRules({ + cwd, + repeats: 1, + evaluate: evaluateLocalRepositoryJudgement, + }); + expect(report.status).toBe('fail'); + expect(report.reports[0]?.results[0]?.operationalFailure).toBe(true); + expect(JSON.stringify(report)).not.toContain('secret provider details'); +}); diff --git a/packages/cloud-agents/src/server/judgement-local-cli.ts b/packages/cloud-agents/src/server/judgement-local-cli.ts index 2e6a7006e9..6739b35926 100644 --- a/packages/cloud-agents/src/server/judgement-local-cli.ts +++ b/packages/cloud-agents/src/server/judgement-local-cli.ts @@ -1,11 +1,29 @@ import { parseArgs } from 'node:util'; -import { exitCode, formatReport } from '@roo-code/judgement'; -import { checkLocalRepositoryJudgement } from './judgement-local'; +import { + exitCode, + formatReport, + testRules, + calibrateRules, + formatExampleSuite, + exampleSuiteExitCode, +} from '@roo-code/judgement'; +import { + checkLocalRepositoryJudgement, + evaluateLocalRepositoryJudgement, +} from './judgement-local'; async function main() { - const { values } = parseArgs({ + const { values, positionals } = parseArgs({ + allowPositionals: true, options: { cwd: { type: 'string' }, + all: { type: 'boolean' }, + rule: { type: 'string' }, + examples: { type: 'string' }, + 'examples-dir': { type: 'string' }, + repeats: { type: 'string' }, + thresholds: { type: 'string' }, + concurrency: { type: 'string' }, staged: { type: 'boolean' }, verbose: { type: 'boolean', short: 'v' }, hook: { type: 'boolean' }, @@ -21,10 +39,87 @@ async function main() { }); if (values.help) { console.log( - 'pnpm judgement [--verbose] [--dry-run] [--format json] [--hook] [--base --head ] [--timeout-ms ] [--advisory]\nUses Roomote Settings > Models and server-side provider credentials. Cache is disabled because deployment inference settings can change.', + 'pnpm judgement [--verbose] [--dry-run] [--format json] [--hook] [--base --head ] [--timeout-ms ] [--advisory]\npnpm judgement test [--rule ] [--dry-run] [--format json]\npnpm judgement calibrate (--rule | --all) [--thresholds 0.8,0.85,0.9] [--format json]\nUses Roomote Settings > Models and server-side provider credentials. Cache is disabled because deployment inference settings can change.', ); return 0; } + const command = positionals[0] ?? 'check'; + if ( + positionals.length > 1 || + !['check', 'test', 'calibrate'].includes(command) + ) { + throw new Error('Invalid command.'); + } + if (command !== 'check') { + if ( + values.staged || + values.base || + values.head || + values.hook || + values.advisory || + (values.all && values.rule) || + (values.examples && values['examples-dir']) || + (command === 'test' && values.thresholds !== undefined) || + (command === 'calibrate' && !values.rule && !values.all) || + values.thresholds?.split(',').some((value) => !value.trim()) || + (values.format && !['text', 'json'].includes(values.format)) + ) { + throw new Error('Invalid example command arguments.'); + } + const controller = new AbortController(); + const interrupt = () => controller.abort(); + process.once('SIGINT', interrupt); + try { + const options = { + cwd: values.cwd, + ruleId: values.rule, + examplesPath: values.examples, + examplesDirectory: values['examples-dir'], + repeats: + values.repeats === undefined ? undefined : Number(values.repeats), + concurrency: + values.concurrency === undefined + ? undefined + : Number(values.concurrency), + deadlineMs: + values['timeout-ms'] === undefined + ? undefined + : Number(values['timeout-ms']), + dryRun: values['dry-run'], + signal: controller.signal, + evaluate: evaluateLocalRepositoryJudgement, + onDiagnostic: values.verbose + ? (message: string) => console.error(`judgement: ${message}`) + : undefined, + }; + const report = + command === 'test' + ? await testRules(options) + : await calibrateRules({ + ...options, + thresholds: values.thresholds?.split(',').map(Number), + }); + console.log( + values.format === 'json' + ? JSON.stringify(report) + : formatExampleSuite(report), + ); + return exampleSuiteExitCode(report); + } finally { + process.removeListener('SIGINT', interrupt); + } + } + if ( + values.all || + values.rule || + values.examples || + values['examples-dir'] || + values.repeats || + values.thresholds || + values.concurrency + ) { + throw new Error('Example options require test or calibrate.'); + } if ( (values.staged && values.base) || (values.head && !values.base) || @@ -60,7 +155,11 @@ async function main() { // exit after reporting rather than waiting for the server pools to idle out. void main() .then((code) => process.exit(code)) - .catch(() => { + .catch((error: unknown) => { + if (error instanceof Error && error.name === 'AbortError') { + console.error('judgement: interrupted.'); + process.exit(130); + } console.error( 'judgement: failed to start. Check arguments, .env.local, and local Roomote configuration.', ); diff --git a/packages/cloud-agents/src/server/judgement-local.ts b/packages/cloud-agents/src/server/judgement-local.ts index 2e2aff6d05..831160378d 100644 --- a/packages/cloud-agents/src/server/judgement-local.ts +++ b/packages/cloud-agents/src/server/judgement-local.ts @@ -1,6 +1,34 @@ -import { check, type CheckOptions } from '@roo-code/judgement'; +import { + check, + type CheckOptions, + type JudgeRequest, +} from '@roo-code/judgement'; + +/** Developer checks and example suites share Roomote's inference configuration. */ +export async function evaluateLocalRepositoryJudgement( + request: JudgeRequest, + signal: AbortSignal, +) { + let answer; + try { + // Dry runs and repositories with no applicable rules need no database. + const { evaluateRepositoryJudgement } = await import('./judge-file'); + signal.throwIfAborted(); + answer = await evaluateRepositoryJudgement(request); + signal.throwIfAborted(); + } catch { + throw new Error( + 'Roomote inference could not complete the check. Verify the local database, deployment settings, and provider access.', + ); + } + if (!answer) { + throw new Error( + 'No supported Judgement backend is configured. Select Jev in Roomote Settings > Models and configure its provider credentials. R_JUDGMENT_MODEL overrides that selection.', + ); + } + return answer; +} -/** Developer checks share the API's deployment settings and credential resolver. */ export async function checkLocalRepositoryJudgement(options: CheckOptions) { const failures = new Set(); const report = await check({ @@ -8,23 +36,10 @@ export async function checkLocalRepositoryJudgement(options: CheckOptions) { cache: false, evaluate: async (request, signal) => { try { - // Dry runs and repositories with no applicable rules need no database. - const { evaluateRepositoryJudgement } = await import('./judge-file'); - signal.throwIfAborted(); - const answer = await evaluateRepositoryJudgement(request); - signal.throwIfAborted(); - if (!answer) { - failures.add( - 'No supported Judgement backend is configured. Select Jev in Roomote Settings > Models and configure its provider credentials. R_JUDGMENT_MODEL overrides that selection.', - ); - throw new Error('Backend unavailable'); - } - return answer; - } catch { - if (!failures.size) - failures.add( - 'Roomote inference could not complete the check. Verify the local database, deployment settings, and provider access.', - ); + return await evaluateLocalRepositoryJudgement(request, signal); + } catch (error) { + // The shared adapter only throws the safe configuration messages above. + failures.add((error as Error).message); throw new Error('Roomote inference unavailable'); } }, diff --git a/packages/types/src/judge-policy.ts b/packages/types/src/judge-policy.ts index efc2064719..18212fa676 100644 --- a/packages/types/src/judge-policy.ts +++ b/packages/types/src/judge-policy.ts @@ -1,6 +1,6 @@ import { z } from 'zod'; -export const JUDGE_POLICY_FILE_NAME = 'JUDGE.json'; +export const JUDGE_POLICY_FILE_NAME = '.judgement/rules.json'; export const JUDGE_DEFAULT_THRESHOLD = 0.85; export const JUDGE_MAX_FILE_CONTEXT_BYTES = 48_000; export const JUDGE_MAX_PATCH_CONTEXT_BYTES = 32_000; diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 3331613fa7..7dc571bfbc 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -942,8 +942,8 @@ importers: specifier: ^1.29.0 version: 1.29.0(@cfworker/json-schema@4.1.1)(zod@3.25.76) '@roo-code/judgement': - specifier: 0.1.5 - version: 0.1.5 + specifier: 0.2.0 + version: 0.2.0 '@roomote/auth': specifier: workspace:^ version: link:../../packages/auth @@ -1147,8 +1147,8 @@ importers: specifier: 1.18.30 version: 1.18.30 '@roo-code/judgement': - specifier: 0.1.5 - version: 0.1.5 + specifier: 0.2.0 + version: 0.2.0 '@roomote/ado': specifier: workspace:^ version: link:../ado @@ -5101,8 +5101,8 @@ packages: cpu: [x64] os: [win32] - '@roo-code/judgement@0.1.5': - resolution: {integrity: sha512-1WB83Vw6MIn5v7vS1feuX5MefVzz/zKxLgt91gejhxuLWUv+spDB95a/s6BGETl0kfQg5dZOAUs9utQsdpkljA==} + '@roo-code/judgement@0.2.0': + resolution: {integrity: sha512-DXPbL+pEcY0DnSBdUIcB+8+5wDdiuZzQa1C+eH1I7YfliKtGL39S5KxMkrFO3u6gpmzDaqVXAD+18DEhA0UTAQ==} engines: {node: '>=22.20.0'} hasBin: true @@ -15587,7 +15587,7 @@ snapshots: '@rollup/rollup-win32-x64-msvc@4.61.1': optional: true - '@roo-code/judgement@0.1.5': + '@roo-code/judgement@0.2.0': dependencies: picomatch: 4.0.7 diff --git a/pnpm-workspace.yaml b/pnpm-workspace.yaml index 10b03a453a..3902f00d08 100644 --- a/pnpm-workspace.yaml +++ b/pnpm-workspace.yaml @@ -6,7 +6,7 @@ minimumReleaseAge: 10080 # lockfile-affecting changes while it finishes maturing. minimumReleaseAgeExclude: # First-party release reviewed with the Judgement integration. - - "@roo-code/judgement@0.1.5" + - "@roo-code/judgement@0.2.0" - jose # Allow the patched releases while they finish maturing. - brace-expansion