diff --git a/.judgement/.gitignore b/.judgement/.gitignore new file mode 100644 index 0000000000..fbca225379 --- /dev/null +++ b/.judgement/.gitignore @@ -0,0 +1 @@ +results/ diff --git a/.judgement/README.md b/.judgement/README.md new file mode 100644 index 0000000000..e1db32355e --- /dev/null +++ b/.judgement/README.md @@ -0,0 +1,55 @@ +# Roomote rule calibration + +These are labeled changes for each of the eight repository rules. Labels describe +what the rule requires, not what the model happened to answer. Keep cases that +fail: they are regression examples, not reasons to change the expected label. + +| ID | Rule | Examples | +| ------------- | --------------------------------------- | -------: | +| `criterion_1` | Secrets and private data | 6 | +| `criterion_2` | Worker subprocess environments | 6 | +| `criterion_3` | Sanitized API errors | 6 | +| `criterion_4` | Agent setup prose | 6 | +| `criterion_5` | Independent async operations | 6 | +| `criterion_6` | UI and setup copy | 6 | +| `criterion_7` | Lowercase ordinary nouns | 11 | +| `criterion_8` | Direct descriptions of current behavior | 10 | + +IDs currently follow the order in `.judgement/rules.json`. Preserve those IDs explicitly if +reordering rules, and update examples when changing the intended rule behavior. +The `path` in each example matches the real rule's file globs. Before/after strings +are synthetic source files; they are judged as diffs and are never executed. + +## Running examples + +Run from the repository root: + +```sh +pnpm judgement test --dry-run +pnpm judgement test --rule criterion_8 +pnpm judgement calibrate --rule criterion_8 --thresholds 0.8,0.85,0.9 +pnpm --silent judgement test --format json > .judgement/results/latest.json +``` + +Create `.judgement/results/` before redirecting output there. The commands use +Roomote's existing model settings and server credentials through the shared +Judgement library. Keep generated reports local; `results/` is ignored by Git. +See [Judgement's agent guide](https://github.com/RooCodeInc/judgement/blob/main/docs/agents.md) +for rule design, calibration, and interpreting incomplete results. + +## Coverage limits + +The privacy rule explicitly allows synthetic credentials and fictional identities. +Its fixtures are therefore **all valid**. Do not relabel a placeholder as a secret +violation just to get a positive test, and never add real credentials or private +data as fixtures. These examples measure false alarms only; they do not establish +secret-detection recall or justify a new privacy threshold. + +The other rules include violations, allowed exceptions, and fixes to existing +violations. Rule 7 also checks identifiers, sentence starts, title case, exact UI +labels, and an unrelated edit near an unchanged capitalization violation. Rule 8 +also covers migration guides, release notes, supported choices, and useful security +constraints. + +Incomplete checks allow hook commits but fail strict checks. They do not count as +successful violation detection or complete approval of a valid edit. diff --git a/.judgement/examples/criterion_1.json b/.judgement/examples/criterion_1.json new file mode 100644 index 0000000000..ad6a578b47 --- /dev/null +++ b/.judgement/examples/criterion_1.json @@ -0,0 +1,47 @@ +{ + "ruleId": "criterion_1", + "examples": [ + { + "name": "Clearly synthetic credential placeholder", + "path": ".env.example", + "before": "TOKEN=\n", + "after": "TOKEN=REPLACE_WITH_YOUR_TOKEN\n", + "expected": "pass" + }, + { + "name": "Server-side credential lookup", + "path": "apps/api/src/config.ts", + "before": "export const config = {};\n", + "after": "export const config = { apiKey: process.env.PROVIDER_API_KEY };\n", + "expected": "pass" + }, + { + "name": "Redacted diagnostic example", + "path": "docs/troubleshooting.md", + "before": "Example output:\n", + "after": "Example output: {\"authorization\":\"[REDACTED]\"}\n", + "expected": "pass" + }, + { + "name": "Public configuration", + "path": "src/config.ts", + "before": "export const options = {};\n", + "after": "export const options = { timeoutMs: 3000, retries: 2 };\n", + "expected": "pass" + }, + { + "name": "Unmistakably fictional fixture", + "path": "test/users.fixture.ts", + "before": "export const users = [];\n", + "after": "export const users = [{ name: \"Synthetic Test User\", email: \"test-user@example.invalid\" }];\n", + "expected": "pass" + }, + { + "name": "Remove a placeholder from an example", + "path": ".env.example", + "before": "API_KEY=REPLACE_WITH_YOUR_KEY\n", + "after": "API_KEY=\n", + "expected": "pass" + } + ] +} diff --git a/.judgement/examples/criterion_2.json b/.judgement/examples/criterion_2.json new file mode 100644 index 0000000000..0512802b42 --- /dev/null +++ b/.judgement/examples/criterion_2.json @@ -0,0 +1,47 @@ +{ + "ruleId": "criterion_2", + "examples": [ + { + "name": "Spread all launcher variables", + "path": "apps/worker/src/run-task/launch.ts", + "before": "export function taskEnv() { return { PATH: process.env.PATH }; }\n", + "after": "export function taskEnv() { return { ...process.env }; }\n", + "expected": "violation" + }, + { + "name": "Forward worker authentication to task", + "path": "apps/worker/src/run-task/launch.ts", + "before": "export function taskEnv() { return { PATH: process.env.PATH }; }\n", + "after": "export function taskEnv() { return { PATH: process.env.PATH, WORKER_AUTH_TOKEN: process.env.WORKER_AUTH_TOKEN }; }\n", + "expected": "violation" + }, + { + "name": "Explicit non-sensitive allowlist", + "path": "apps/worker/src/run-task/launch.ts", + "before": "export function taskEnv() { return {}; }\n", + "after": "export function taskEnv() { return { PATH: process.env.PATH, LANG: process.env.LANG }; }\n", + "expected": "pass" + }, + { + "name": "Named nested runtime contract", + "path": "apps/worker/src/run-task/launch.ts", + "before": "export function taskEnv() { return { PATH: process.env.PATH }; }\n", + "after": "// The nested Node service requires NODE_OPTIONS to set its heap limit.\nexport function taskEnv() { return { PATH: process.env.PATH, NODE_OPTIONS: \"--max-old-space-size=512\" }; }\n", + "expected": "pass" + }, + { + "name": "Scrub captured worker token before launch", + "path": "apps/worker/src/run-task/launch.ts", + "before": "export function launch(spawn, binary, capturedEnv) { return spawn(binary, [], { env: capturedEnv }); }\n", + "after": "export function launch(spawn, binary, capturedEnv) {\n const env = { PATH: capturedEnv.PATH, LANG: capturedEnv.LANG };\n return spawn(binary, [], { env });\n}\n", + "expected": "pass" + }, + { + "name": "Synthetic fixture with no inherited environment", + "path": "apps/worker/src/run-task/launch.test.ts", + "before": "export const exampleEnv = {};\n", + "after": "export const exampleEnv = { PATH: \"/synthetic/bin\", WORKER_AUTH_TOKEN: \"NON_AUTHENTICATING_TEST_PLACEHOLDER\" };\n", + "expected": "pass" + } + ] +} diff --git a/.judgement/examples/criterion_3.json b/.judgement/examples/criterion_3.json new file mode 100644 index 0000000000..9d0e1a6a09 --- /dev/null +++ b/.judgement/examples/criterion_3.json @@ -0,0 +1,47 @@ +{ + "ruleId": "criterion_3", + "examples": [ + { + "name": "Return raw exception message", + "path": "apps/api/src/handlers/project.ts", + "before": "export function failure(c, error) { return c.json({ error: \"Unable to load the project.\" }, 500); }\n", + "after": "export function failure(c, error) { return c.json({ error: error.message }, 500); }\n", + "expected": "violation" + }, + { + "name": "Return stack trace", + "path": "apps/api/src/handlers/project.ts", + "before": "export function failure(c, error) { return c.json({ error: \"Unable to load the project.\" }, 500); }\n", + "after": "export function failure(c, error) { return c.json({ error: error.stack }, 500); }\n", + "expected": "violation" + }, + { + "name": "Return full exception object", + "path": "apps/api/src/handlers/project.ts", + "before": "export function failure(c, error) { return c.json({ error: \"Request failed.\" }, 500); }\n", + "after": "export function failure(c, error) { return c.json({ error }, 500); }\n", + "expected": "violation" + }, + { + "name": "Sanitized failure response", + "path": "apps/api/src/handlers/project.ts", + "before": "export function failure(c, error) { return c.json({ error: error.message }, 500); }\n", + "after": "export function failure(c, error) { return c.json({ error: \"Unable to load the project. Try again.\" }, 500); }\n", + "expected": "pass" + }, + { + "name": "Safe not-found response", + "path": "apps/api/src/handlers/project.ts", + "before": "export function missing(c) { return c.json({}, 404); }\n", + "after": "export function missing(c) { return c.json({ error: \"Project not found.\" }, 404); }\n", + "expected": "pass" + }, + { + "name": "Safe validation message", + "path": "apps/api/src/handlers/project.ts", + "before": "export function invalid(c) { return c.json({}, 400); }\n", + "after": "export function invalid(c) { return c.json({ error: \"Enter a valid project name.\" }, 400); }\n", + "expected": "pass" + } + ] +} diff --git a/.judgement/examples/criterion_4.json b/.judgement/examples/criterion_4.json new file mode 100644 index 0000000000..b0d5911404 --- /dev/null +++ b/.judgement/examples/criterion_4.json @@ -0,0 +1,47 @@ +{ + "ruleId": "criterion_4", + "examples": [ + { + "name": "Tell user to operate an internal card", + "path": "packages/cloud-agents/src/server/setup-prompt.ts", + "before": "export const setupReply = \"Connect a repository so I can review your code.\";\n", + "after": "export const setupReply = \"Click the Configure button in the setup card on the right rail.\";\n", + "expected": "violation" + }, + { + "name": "Ask for credentials in chat", + "path": "packages/cloud-agents/src/server/setup-prompt.ts", + "before": "export const setupReply = \"Connect your account securely to continue.\";\n", + "after": "export const setupReply = \"Paste your API key into this chat so I can connect your account.\";\n", + "expected": "violation" + }, + { + "name": "State needed capability", + "path": "packages/cloud-agents/src/server/setup-prompt.ts", + "before": "export const setupReply = \"I need access.\";\n", + "after": "export const setupReply = \"Connect a repository so I can review your code.\";\n", + "expected": "pass" + }, + { + "name": "State user outcome", + "path": "packages/cloud-agents/src/server/setup-prompt.ts", + "before": "export const setupReply = \"Configuration complete.\";\n", + "after": "export const setupReply = \"I can now review pull requests in your repository.\";\n", + "expected": "pass" + }, + { + "name": "Internal tool schema may describe controls", + "path": "packages/cloud-agents/src/server/setup-prompt.ts", + "before": "export const toolDescription = \"Configure access.\";\n", + "after": "// Internal tool schema, never shown as agent prose.\nexport const toolDescription = \"Render the trusted repository configuration card with its Connect button.\";\n", + "expected": "pass" + }, + { + "name": "Trusted UI description may describe controls", + "path": "packages/cloud-agents/src/server/setup-prompt.ts", + "before": "export const trustedUiDescription = \"Connection settings\";\n", + "after": "// Trusted UI description for the setup renderer; not agent prose.\nexport const trustedUiDescription = \"The Connect button opens the repository permissions dialog.\";\n", + "expected": "pass" + } + ] +} diff --git a/.judgement/examples/criterion_5.json b/.judgement/examples/criterion_5.json new file mode 100644 index 0000000000..745a735001 --- /dev/null +++ b/.judgement/examples/criterion_5.json @@ -0,0 +1,47 @@ +{ + "ruleId": "criterion_5", + "examples": [ + { + "name": "Serialize independent read requests", + "path": "apps/web/src/server/dashboard.ts", + "before": "export async function load(db) {\n return Promise.all([db.projects.findMany(), db.notifications.findMany()]);\n}\n", + "after": "export async function load(db) {\n const projects = await db.projects.findMany();\n const notifications = await db.notifications.findMany();\n return [projects, notifications];\n}\n", + "expected": "violation" + }, + { + "name": "Await work before unrelated early return", + "path": "apps/web/src/server/dashboard.ts", + "before": "export async function load(fetchProjects, showProjects) {\n if (!showProjects) return [];\n return fetchProjects();\n}\n", + "after": "export async function load(fetchProjects, showProjects) {\n const projects = await fetchProjects();\n if (!showProjects) return [];\n return projects;\n}\n", + "expected": "violation" + }, + { + "name": "Parallel independent reads", + "path": "apps/web/src/server/dashboard.ts", + "before": "export async function load(db) {\n const projects = await db.projects.findMany();\n const notifications = await db.notifications.findMany();\n return [projects, notifications];\n}\n", + "after": "export async function load(db) {\n return Promise.all([db.projects.findMany(), db.notifications.findMany()]);\n}\n", + "expected": "pass" + }, + { + "name": "Second query needs first result", + "path": "apps/web/src/server/dashboard.ts", + "before": "export async function load(db, slug) { return db.projects.findBySlug(slug); }\n", + "after": "export async function load(db, slug) {\n const project = await db.projects.findBySlug(slug);\n return db.tasks.findMany({ projectId: project.id });\n}\n", + "expected": "pass" + }, + { + "name": "Preserve transaction order", + "path": "apps/web/src/server/dashboard.ts", + "before": "export async function update(db, id) { return db.project.update(id); }\n", + "after": "export async function update(db, id) {\n return db.transaction(async tx => {\n await tx.project.lock(id);\n return tx.project.update(id);\n });\n}\n", + "expected": "pass" + }, + { + "name": "Explicit rate-limited operation", + "path": "apps/web/src/server/dashboard.ts", + "before": "export async function send(sendOne, ids) { return ids; }\n", + "after": "// Provider contract permits one outstanding request at a time.\nexport async function send(sendOne, ids) {\n for (const id of ids) await sendOne(id);\n}\n", + "expected": "pass" + } + ] +} diff --git a/.judgement/examples/criterion_6.json b/.judgement/examples/criterion_6.json new file mode 100644 index 0000000000..0fc35991ab --- /dev/null +++ b/.judgement/examples/criterion_6.json @@ -0,0 +1,47 @@ +{ + "ruleId": "criterion_6", + "examples": [ + { + "name": "Describe implementation in setup copy", + "path": "apps/web/src/components/setup-copy.ts", + "before": "export const description = \"Get updates when work is ready.\";\n", + "after": "export const description = \"This feature is default off and uses a polling worker to materialize event records in a Redis queue.\";\n", + "expected": "violation" + }, + { + "name": "Expose internal mechanism in task description", + "path": "apps/web/src/components/setup-copy.ts", + "before": "export const description = \"Review your latest code changes.\";\n", + "after": "export const description = \"This task hydrates the worker state, dispatches an internal tool schema, and serializes the result into our event envelope.\";\n", + "expected": "violation" + }, + { + "name": "Explain user value", + "path": "apps/web/src/components/setup-copy.ts", + "before": "export const description = \"Configure notifications.\";\n", + "after": "export const description = \"Get updates when your work is ready.\";\n", + "expected": "pass" + }, + { + "name": "Necessary permission consequence", + "path": "apps/web/src/components/setup-copy.ts", + "before": "export const permissionHelp = \"Connect your repository.\";\n", + "after": "export const permissionHelp = \"Allow read access so your agent can review code. It cannot push changes without write access.\";\n", + "expected": "pass" + }, + { + "name": "Actionable recovery message", + "path": "apps/web/src/components/setup-copy.ts", + "before": "export const errorMessage = \"Failed.\";\n", + "after": "export const errorMessage = \"Connection expired. Reconnect your account to continue.\";\n", + "expected": "pass" + }, + { + "name": "Necessary security copy", + "path": "apps/web/src/components/setup-copy.ts", + "before": "export const securityHelp = \"Account settings\";\n", + "after": "export const securityHelp = \"Revoking this token immediately stops access for apps that use it.\";\n", + "expected": "pass" + } + ] +} diff --git a/.judgement/examples/criterion_7.json b/.judgement/examples/criterion_7.json new file mode 100644 index 0000000000..6dc258fe8f --- /dev/null +++ b/.judgement/examples/criterion_7.json @@ -0,0 +1,82 @@ +{ + "ruleId": "criterion_7", + "examples": [ + { + "name": "Mid-sentence session capital", + "path": "docs/guide.md", + "before": "Review the remaining session checks before release.\n", + "after": "Review the remaining Session checks before release.\n", + "expected": "violation" + }, + { + "name": "Mid-sentence task capital", + "path": "docs/guide.md", + "before": "You can resume the task from the sidebar.\n", + "after": "You can resume the Task from the sidebar.\n", + "expected": "violation" + }, + { + "name": "UI prose with capitalized ordinary noun", + "path": "apps/web/src/components/status-copy.ts", + "before": "export const message = \"The task is ready to resume.\";\n", + "after": "export const message = \"The Task is ready to resume.\";\n", + "expected": "violation" + }, + { + "name": "Lowercase ordinary nouns", + "path": "docs/guide.md", + "before": "Review the remaining checks.\n", + "after": "Review the remaining session and task checks.\n", + "expected": "pass" + }, + { + "name": "Sentence begins with session", + "path": "docs/guide.md", + "before": "History is available.\n", + "after": "Session history is available.\n", + "expected": "pass" + }, + { + "name": "Sentence begins with task", + "path": "docs/guide.md", + "before": "Checks are complete.\n", + "after": "Task checks are complete.\n", + "expected": "pass" + }, + { + "name": "Title case heading", + "path": "docs/guide.md", + "before": "## Release Checks\n", + "after": "## Task and Session Checks\n", + "expected": "pass" + }, + { + "name": "Exact quoted UI label", + "path": "docs/guide.md", + "before": "Click the button to continue.\n", + "after": "Click the exact UI label \"New Task\" to continue.\n", + "expected": "pass" + }, + { + "name": "Code identifier", + "path": "src/types.ts", + "before": "export type RecordId = string;\n", + "after": "export type TaskId = string;\n", + "expected": "pass" + }, + { + "name": "Fix prior capitalization", + "path": "docs/guide.md", + "before": "Open the Task to review its Session history.\n", + "after": "Open the task to review its session history.\n", + "expected": "pass" + }, + { + "name": "Do not attribute unchanged violation to new diff", + "path": "docs/guide.md", + "before": "Open the Task to review history.\nWait for completion.\n", + "after": "Open the Task to review history.\nWait for completion before closing.\n", + "expected": "pass" + } + ] +} diff --git a/.judgement/examples/criterion_8.json b/.judgement/examples/criterion_8.json new file mode 100644 index 0000000000..d4c43b4e10 --- /dev/null +++ b/.judgement/examples/criterion_8.json @@ -0,0 +1,75 @@ +{ + "ruleId": "criterion_8", + "examples": [ + { + "name": "Unnecessary exclusivity in a save instruction", + "path": "docs/setup.md", + "before": "Save rules in `.judgement/rules.json`.\n", + "after": "Save rules in `.judgement/rules.json`, the only policy location.\n", + "expected": "violation" + }, + { + "name": "Past implementation aside in setup guide", + "path": "docs/setup.md", + "before": "Open Settings to connect your account.\n", + "after": "Open Settings to connect your account. We used to put this in the sidebar before the redesign.\n", + "expected": "violation" + }, + { + "name": "Removed alternative in ordinary setup instructions", + "path": "docs/setup.md", + "before": "Save the configuration in `config.json`.\n", + "after": "Save the configuration in `config.json`, not the old YAML file we no longer use.\n", + "expected": "violation" + }, + { + "name": "Direct save instruction", + "path": "docs/setup.md", + "before": "Create a rules file.\n", + "after": "Save rules in `.judgement/rules.json`.\n", + "expected": "pass" + }, + { + "name": "Direct setup action", + "path": "docs/setup.md", + "before": "Configure access.\n", + "after": "Open Settings to connect your account.\n", + "expected": "pass" + }, + { + "name": "Migration guide requires old and new paths", + "path": "docs/migration.md", + "before": "# Upgrade from version 1\nUpdate your configuration.\n", + "after": "# Upgrade from version 1\nMove your existing `settings.yml` to `config.json` before starting version 2.\n", + "expected": "pass" + }, + { + "name": "Release notes describe the change", + "path": "CHANGELOG.md", + "before": "## Version 2\nImprovements.\n", + "after": "## Version 2\nConfiguration now lives in `config.json`; the version 1 YAML format is no longer accepted.\n", + "expected": "pass" + }, + { + "name": "Supported choices with practical tradeoffs", + "path": "docs/setup.md", + "before": "Choose a storage option.\n", + "after": "Choose local storage for offline access or shared storage to collaborate across devices.\n", + "expected": "pass" + }, + { + "name": "Security restriction is actionable", + "path": "docs/setup.md", + "before": "Configure the callback URL.\n", + "after": "Only HTTPS callback URLs are accepted to protect credentials in transit.\n", + "expected": "pass" + }, + { + "name": "Explicit bad and good examples", + "path": "docs/writing.md", + "before": "## Writing examples\nUse direct instructions.\n", + "after": "## Writing examples\nAvoid: \"Save rules here, the only policy location.\"\nPrefer: \"Save rules here.\"\n", + "expected": "pass" + } + ] +} diff --git a/JUDGE.json b/.judgement/rules.json similarity index 88% rename from JUDGE.json rename to .judgement/rules.json index 623e66f637..bfc11c6d14 100644 --- a/JUDGE.json +++ b/.judgement/rules.json @@ -48,6 +48,11 @@ "rule": "In user-facing prose, treat \"session\" and \"task\" as ordinary nouns rather than proper nouns; use lowercase \"session\" and \"task\" except at sentence beginnings, in title case, or when quoting an exact UI label.", "files": ["**/*"], "threshold": 0.85 + }, + { + "rule": "Release notes, migration instructions, security or compatibility requirements, and quoted writing examples are exempt from this rule. In ordinary explanatory and instructional prose, state the current location, behavior, or action directly. Do not add unnecessary asides about previous iterations or removed alternatives, or unnecessary exclusivity such as \"the only policy location\". Comparisons between supported options are allowed when they help the reader choose.", + "files": ["**/*"], + "threshold": 0.85 } ] } diff --git a/LOCAL_DEVELOPMENT.md b/LOCAL_DEVELOPMENT.md index 1cc689d9ff..3bf92a5f5a 100644 --- a/LOCAL_DEVELOPMENT.md +++ b/LOCAL_DEVELOPMENT.md @@ -6,14 +6,14 @@ checkout. If you want to operate Roomote for a team, start with the public ## Prerequisites -| Tool | Version | Notes | -| ---- | ------- | ----- | -| macOS | Latest | This guide is optimized for macOS. Apple Silicon runs the task worker natively (no amd64 emulation). | -| Docker Desktop | Latest | Provides Docker Engine and Compose for local services. | -| mise | Latest | Manages the repository toolchain. | -| ngrok | Latest | Optional, only when `R_PUBLIC_URL` uses an ngrok domain. | -| OpenCode CLI | Latest | Required to run tasks and server-side model calls. Install with `npm install -g opencode-ai`. | -| Python 3 | Latest | Required by the server-side OpenCode CLI helper. Install with `brew install python` if `python3` is missing. | +| Tool | Version | Notes | +| -------------- | ------- | ------------------------------------------------------------------------------------------------------------ | +| macOS | Latest | This guide is optimized for macOS. Apple Silicon runs the task worker natively (no amd64 emulation). | +| Docker Desktop | Latest | Provides Docker Engine and Compose for local services. | +| mise | Latest | Manages the repository toolchain. | +| ngrok | Latest | Optional, only when `R_PUBLIC_URL` uses an ngrok domain. | +| OpenCode CLI | Latest | Required to run tasks and server-side model calls. Install with `npm install -g opencode-ai`. | +| Python 3 | Latest | Required by the server-side OpenCode CLI helper. Install with `brew install python` if `python3` is missing. | The dev toolchain runs on Node.js 24.13.1, pinned in `.tool-versions`, `.nvmrc`, and `.node-version`. The `package.json` `engines` field accepts @@ -142,14 +142,14 @@ the `roomote-worker:local` image exists. Common options: -| Option | Default | Description | -| ------ | ------- | ----------- | -| `--reset` | `false` | Reset the database before starting services. | -| `--verbose` | `false` | Enable verbose output with stdio logging. | -| `--skip-worker-release-build` | `false` | Reuse the existing worker artifact from `./releases`. | -| `--use-release` | `false` | Use GitHub worker releases instead of the local worker build. | -| `--worker-release-channel` | `stable` | GitHub worker release channel for `--use-release`. | -| `--worker-release-version` | unset | Optional GitHub worker release version pin. | +| Option | Default | Description | +| ----------------------------- | -------- | ------------------------------------------------------------- | +| `--reset` | `false` | Reset the database before starting services. | +| `--verbose` | `false` | Enable verbose output with stdio logging. | +| `--skip-worker-release-build` | `false` | Reuse the existing worker artifact from `./releases`. | +| `--use-release` | `false` | Use GitHub worker releases instead of the local worker build. | +| `--worker-release-channel` | `stable` | GitHub worker release channel for `--use-release`. | +| `--worker-release-version` | unset | Optional GitHub worker release version pin. | Examples: @@ -292,7 +292,6 @@ curl -X GET "http://localhost:13001/trpc/auth.me" \ -H "Content-Type: application/json" | jq ``` - ### Checking staged changes with Judgement The pre-commit hook runs Judgement after `lint-staged`, so it checks the formatted @@ -303,7 +302,7 @@ the worker-installed wrapper runs Judgement once through task inference, even when the proxy is unavailable. Run `pnpm judgement --verbose` from the Roomote checkout to check staged changes -against `JUDGE.json`. The command loads `.env.local` and uses the same deployment +against `.judgement/rules.json`. The command loads `.env.local` and uses the same deployment judgment-model settings and encrypted provider credentials as the local API. The local database must be running. Select Jev through TypeSafe, OpenRouter, or Vercel AI Gateway in **Settings → Models**; `R_JUDGMENT_MODEL` overrides that choice. @@ -317,3 +316,13 @@ or `--format json` for the report as JSON. Checks are not cached because the configured inference backend can change. Managed task hooks use the same API inference settings through a worker-owned local proxy and retain their three-second deadline. + +Run `pnpm judgement test` to evaluate the labeled examples under +`.judgement/examples/` at their configured thresholds. Use `--rule criterion_8` +to select a rule and `--dry-run` to validate inputs without inference. To compare +thresholds, run `pnpm judgement calibrate --rule criterion_8 --thresholds 0.8,0.85,0.9` +or use `--all` to include every rule. Judgement owns the test and calibration +runner; Roomote supplies its inference adapter. Live runs make model requests. +Generated reports belong in local output or CI artifacts. See +[the agent guide](https://github.com/RooCodeInc/judgement/blob/main/docs/agents.md) +for labeling examples and interpreting results. diff --git a/apps/docs/tasks.mdx b/apps/docs/tasks.mdx index cc03d08ec3..65aeae425d 100644 --- a/apps/docs/tasks.mdx +++ b/apps/docs/tasks.mdx @@ -137,7 +137,7 @@ runs the Roomote dashboard. Roomote installs [Judgement](https://github.com/RooCodeInc/judgement) as a Git pre-commit check in task repositories, including repositories checked out during -a task. Add a root `JUDGE.json` to define your business rules: +a task. Add `.judgement/rules.json` to define your business rules: ```json { diff --git a/apps/worker/package.json b/apps/worker/package.json index 726771528c..4f97ccf72f 100644 --- a/apps/worker/package.json +++ b/apps/worker/package.json @@ -20,7 +20,7 @@ "dependencies": { "@hono/node-server": "2.0.10", "@modelcontextprotocol/sdk": "^1.29.0", - "@roo-code/judgement": "0.1.5", + "@roo-code/judgement": "0.2.0", "@roomote/auth": "workspace:^", "@roomote/cloud-agents": "workspace:^", "@roomote/communication": "workspace:^", diff --git a/apps/worker/src/run-task/__tests__/judgement-check.test.ts b/apps/worker/src/run-task/__tests__/judgement-check.test.ts index d0f4641f2c..aba444b075 100644 --- a/apps/worker/src/run-task/__tests__/judgement-check.test.ts +++ b/apps/worker/src/run-task/__tests__/judgement-check.test.ts @@ -1,5 +1,5 @@ import { execFile } from 'node:child_process'; -import { mkdtemp, rm, writeFile } from 'node:fs/promises'; +import { mkdtemp, mkdir, rm, writeFile } from 'node:fs/promises'; import { tmpdir } from 'node:os'; import { join } from 'node:path'; import { promisify } from 'node:util'; @@ -44,8 +44,9 @@ beforeEach(async () => { env.R_JUDGEMENT_GATEWAY_URL = proxy.endpoint; cwd = await mkdtemp(join(tmpdir(), 'roomote-judgement-')); await exec('git', ['init', '-q'], { cwd }); + await mkdir(join(cwd, '.judgement')); await writeFile( - join(cwd, 'JUDGE.json'), + join(cwd, '.judgement/rules.json'), JSON.stringify({ criteria: [{ rule: 'Use sentence case' }] }), ); await writeFile(join(cwd, 'example.ts'), 'changed'); @@ -95,7 +96,7 @@ it('reports unavailable models and missing runtime credentials as incomplete', a expect(mutate).not.toHaveBeenCalled(); }); it('does not send a model request without a repository policy', async () => { - await exec('git', ['rm', '-f', 'JUDGE.json'], { cwd }); + await exec('git', ['rm', '-f', '.judgement/rules.json'], { cwd }); expect((await checkRepositoryJudgement(env, cwd)).status).toBe('pass'); expect(mutate).not.toHaveBeenCalled(); }); diff --git a/packages/cloud-agents/package.json b/packages/cloud-agents/package.json index b3a5ce24b8..570c37d5ac 100644 --- a/packages/cloud-agents/package.json +++ b/packages/cloud-agents/package.json @@ -98,7 +98,7 @@ "@aws-sdk/client-s3": "3.1000.0", "@modelcontextprotocol/sdk": "^1.29.0", "@opencode-ai/sdk": "1.18.30", - "@roo-code/judgement": "0.1.5", + "@roo-code/judgement": "0.2.0", "@roomote/ado": "workspace:^", "@roomote/auth": "workspace:^", "@roomote/bitbucket": "workspace:^", diff --git a/packages/cloud-agents/src/server/__tests__/judgement-local.test.ts b/packages/cloud-agents/src/server/__tests__/judgement-local.test.ts index 931dd0033d..60fbc0010a 100644 --- a/packages/cloud-agents/src/server/__tests__/judgement-local.test.ts +++ b/packages/cloud-agents/src/server/__tests__/judgement-local.test.ts @@ -1,4 +1,4 @@ -import { mkdtemp, rm, writeFile } from 'node:fs/promises'; +import { mkdtemp, mkdir, rm, writeFile } from 'node:fs/promises'; import { tmpdir } from 'node:os'; import { join } from 'node:path'; import { execFile } from 'node:child_process'; @@ -15,8 +15,9 @@ beforeEach(async () => { .mockResolvedValue({ outcome: 'violation', confidence: 0.99 }); cwd = await mkdtemp(join(tmpdir(), 'judgement-local-')); await exec('git', ['init', '-q'], { cwd }); + await mkdir(join(cwd, '.judgement')); await writeFile( - join(cwd, 'JUDGE.json'), + join(cwd, '.judgement/rules.json'), JSON.stringify({ criteria: [{ rule: 'Use lowercase session.' }] }), ); await writeFile(join(cwd, 'example.md'), 'new Session checks'); @@ -54,3 +55,62 @@ it('does not load inference for a dry run', async () => { await checkLocalRepositoryJudgement({ cwd, dryRun: true }); expect(evaluate).not.toHaveBeenCalled(); }); + +it('uses the same Roomote evaluator for the shared rule example suite', async () => { + const { testRules } = await import('@roo-code/judgement'); + const { evaluateLocalRepositoryJudgement } = + await import('../judgement-local'); + await mkdir(join(cwd, '.judgement/examples')); + await writeFile( + join(cwd, '.judgement/examples/criterion_1.json'), + JSON.stringify({ + ruleId: 'criterion_1', + examples: [ + { + name: 'capital', + before: 'session', + after: 'Session', + expected: 'violation', + }, + ], + }), + ); + const report = await testRules({ + cwd, + repeats: 1, + evaluate: evaluateLocalRepositoryJudgement, + }); + expect(report.status).toBe('pass'); + expect(evaluate).toHaveBeenCalled(); + expect(report.reports[0]?.results[0]?.answers[0]?.confidence).toBe(0.99); +}); + +it('does not expose backend errors in example suite reports', async () => { + const { testRules } = await import('@roo-code/judgement'); + const { evaluateLocalRepositoryJudgement } = + await import('../judgement-local'); + await mkdir(join(cwd, '.judgement/examples')); + await writeFile( + join(cwd, '.judgement/examples/criterion_1.json'), + JSON.stringify({ + ruleId: 'criterion_1', + examples: [ + { + name: 'capital', + before: 'session', + after: 'Session', + expected: 'violation', + }, + ], + }), + ); + evaluate.mockRejectedValue(new Error('secret provider details')); + const report = await testRules({ + cwd, + repeats: 1, + evaluate: evaluateLocalRepositoryJudgement, + }); + expect(report.status).toBe('fail'); + expect(report.reports[0]?.results[0]?.operationalFailure).toBe(true); + expect(JSON.stringify(report)).not.toContain('secret provider details'); +}); diff --git a/packages/cloud-agents/src/server/judgement-local-cli.ts b/packages/cloud-agents/src/server/judgement-local-cli.ts index 2e6a7006e9..6739b35926 100644 --- a/packages/cloud-agents/src/server/judgement-local-cli.ts +++ b/packages/cloud-agents/src/server/judgement-local-cli.ts @@ -1,11 +1,29 @@ import { parseArgs } from 'node:util'; -import { exitCode, formatReport } from '@roo-code/judgement'; -import { checkLocalRepositoryJudgement } from './judgement-local'; +import { + exitCode, + formatReport, + testRules, + calibrateRules, + formatExampleSuite, + exampleSuiteExitCode, +} from '@roo-code/judgement'; +import { + checkLocalRepositoryJudgement, + evaluateLocalRepositoryJudgement, +} from './judgement-local'; async function main() { - const { values } = parseArgs({ + const { values, positionals } = parseArgs({ + allowPositionals: true, options: { cwd: { type: 'string' }, + all: { type: 'boolean' }, + rule: { type: 'string' }, + examples: { type: 'string' }, + 'examples-dir': { type: 'string' }, + repeats: { type: 'string' }, + thresholds: { type: 'string' }, + concurrency: { type: 'string' }, staged: { type: 'boolean' }, verbose: { type: 'boolean', short: 'v' }, hook: { type: 'boolean' }, @@ -21,10 +39,87 @@ async function main() { }); if (values.help) { console.log( - 'pnpm judgement [--verbose] [--dry-run] [--format json] [--hook] [--base --head ] [--timeout-ms ] [--advisory]\nUses Roomote Settings > Models and server-side provider credentials. Cache is disabled because deployment inference settings can change.', + 'pnpm judgement [--verbose] [--dry-run] [--format json] [--hook] [--base --head ] [--timeout-ms ] [--advisory]\npnpm judgement test [--rule ] [--dry-run] [--format json]\npnpm judgement calibrate (--rule | --all) [--thresholds 0.8,0.85,0.9] [--format json]\nUses Roomote Settings > Models and server-side provider credentials. Cache is disabled because deployment inference settings can change.', ); return 0; } + const command = positionals[0] ?? 'check'; + if ( + positionals.length > 1 || + !['check', 'test', 'calibrate'].includes(command) + ) { + throw new Error('Invalid command.'); + } + if (command !== 'check') { + if ( + values.staged || + values.base || + values.head || + values.hook || + values.advisory || + (values.all && values.rule) || + (values.examples && values['examples-dir']) || + (command === 'test' && values.thresholds !== undefined) || + (command === 'calibrate' && !values.rule && !values.all) || + values.thresholds?.split(',').some((value) => !value.trim()) || + (values.format && !['text', 'json'].includes(values.format)) + ) { + throw new Error('Invalid example command arguments.'); + } + const controller = new AbortController(); + const interrupt = () => controller.abort(); + process.once('SIGINT', interrupt); + try { + const options = { + cwd: values.cwd, + ruleId: values.rule, + examplesPath: values.examples, + examplesDirectory: values['examples-dir'], + repeats: + values.repeats === undefined ? undefined : Number(values.repeats), + concurrency: + values.concurrency === undefined + ? undefined + : Number(values.concurrency), + deadlineMs: + values['timeout-ms'] === undefined + ? undefined + : Number(values['timeout-ms']), + dryRun: values['dry-run'], + signal: controller.signal, + evaluate: evaluateLocalRepositoryJudgement, + onDiagnostic: values.verbose + ? (message: string) => console.error(`judgement: ${message}`) + : undefined, + }; + const report = + command === 'test' + ? await testRules(options) + : await calibrateRules({ + ...options, + thresholds: values.thresholds?.split(',').map(Number), + }); + console.log( + values.format === 'json' + ? JSON.stringify(report) + : formatExampleSuite(report), + ); + return exampleSuiteExitCode(report); + } finally { + process.removeListener('SIGINT', interrupt); + } + } + if ( + values.all || + values.rule || + values.examples || + values['examples-dir'] || + values.repeats || + values.thresholds || + values.concurrency + ) { + throw new Error('Example options require test or calibrate.'); + } if ( (values.staged && values.base) || (values.head && !values.base) || @@ -60,7 +155,11 @@ async function main() { // exit after reporting rather than waiting for the server pools to idle out. void main() .then((code) => process.exit(code)) - .catch(() => { + .catch((error: unknown) => { + if (error instanceof Error && error.name === 'AbortError') { + console.error('judgement: interrupted.'); + process.exit(130); + } console.error( 'judgement: failed to start. Check arguments, .env.local, and local Roomote configuration.', ); diff --git a/packages/cloud-agents/src/server/judgement-local.ts b/packages/cloud-agents/src/server/judgement-local.ts index 2e2aff6d05..831160378d 100644 --- a/packages/cloud-agents/src/server/judgement-local.ts +++ b/packages/cloud-agents/src/server/judgement-local.ts @@ -1,6 +1,34 @@ -import { check, type CheckOptions } from '@roo-code/judgement'; +import { + check, + type CheckOptions, + type JudgeRequest, +} from '@roo-code/judgement'; + +/** Developer checks and example suites share Roomote's inference configuration. */ +export async function evaluateLocalRepositoryJudgement( + request: JudgeRequest, + signal: AbortSignal, +) { + let answer; + try { + // Dry runs and repositories with no applicable rules need no database. + const { evaluateRepositoryJudgement } = await import('./judge-file'); + signal.throwIfAborted(); + answer = await evaluateRepositoryJudgement(request); + signal.throwIfAborted(); + } catch { + throw new Error( + 'Roomote inference could not complete the check. Verify the local database, deployment settings, and provider access.', + ); + } + if (!answer) { + throw new Error( + 'No supported Judgement backend is configured. Select Jev in Roomote Settings > Models and configure its provider credentials. R_JUDGMENT_MODEL overrides that selection.', + ); + } + return answer; +} -/** Developer checks share the API's deployment settings and credential resolver. */ export async function checkLocalRepositoryJudgement(options: CheckOptions) { const failures = new Set(); const report = await check({ @@ -8,23 +36,10 @@ export async function checkLocalRepositoryJudgement(options: CheckOptions) { cache: false, evaluate: async (request, signal) => { try { - // Dry runs and repositories with no applicable rules need no database. - const { evaluateRepositoryJudgement } = await import('./judge-file'); - signal.throwIfAborted(); - const answer = await evaluateRepositoryJudgement(request); - signal.throwIfAborted(); - if (!answer) { - failures.add( - 'No supported Judgement backend is configured. Select Jev in Roomote Settings > Models and configure its provider credentials. R_JUDGMENT_MODEL overrides that selection.', - ); - throw new Error('Backend unavailable'); - } - return answer; - } catch { - if (!failures.size) - failures.add( - 'Roomote inference could not complete the check. Verify the local database, deployment settings, and provider access.', - ); + return await evaluateLocalRepositoryJudgement(request, signal); + } catch (error) { + // The shared adapter only throws the safe configuration messages above. + failures.add((error as Error).message); throw new Error('Roomote inference unavailable'); } }, diff --git a/packages/types/src/judge-policy.ts b/packages/types/src/judge-policy.ts index efc2064719..18212fa676 100644 --- a/packages/types/src/judge-policy.ts +++ b/packages/types/src/judge-policy.ts @@ -1,6 +1,6 @@ import { z } from 'zod'; -export const JUDGE_POLICY_FILE_NAME = 'JUDGE.json'; +export const JUDGE_POLICY_FILE_NAME = '.judgement/rules.json'; export const JUDGE_DEFAULT_THRESHOLD = 0.85; export const JUDGE_MAX_FILE_CONTEXT_BYTES = 48_000; export const JUDGE_MAX_PATCH_CONTEXT_BYTES = 32_000; diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 3331613fa7..7dc571bfbc 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -942,8 +942,8 @@ importers: specifier: ^1.29.0 version: 1.29.0(@cfworker/json-schema@4.1.1)(zod@3.25.76) '@roo-code/judgement': - specifier: 0.1.5 - version: 0.1.5 + specifier: 0.2.0 + version: 0.2.0 '@roomote/auth': specifier: workspace:^ version: link:../../packages/auth @@ -1147,8 +1147,8 @@ importers: specifier: 1.18.30 version: 1.18.30 '@roo-code/judgement': - specifier: 0.1.5 - version: 0.1.5 + specifier: 0.2.0 + version: 0.2.0 '@roomote/ado': specifier: workspace:^ version: link:../ado @@ -5101,8 +5101,8 @@ packages: cpu: [x64] os: [win32] - '@roo-code/judgement@0.1.5': - resolution: {integrity: sha512-1WB83Vw6MIn5v7vS1feuX5MefVzz/zKxLgt91gejhxuLWUv+spDB95a/s6BGETl0kfQg5dZOAUs9utQsdpkljA==} + '@roo-code/judgement@0.2.0': + resolution: {integrity: sha512-DXPbL+pEcY0DnSBdUIcB+8+5wDdiuZzQa1C+eH1I7YfliKtGL39S5KxMkrFO3u6gpmzDaqVXAD+18DEhA0UTAQ==} engines: {node: '>=22.20.0'} hasBin: true @@ -15587,7 +15587,7 @@ snapshots: '@rollup/rollup-win32-x64-msvc@4.61.1': optional: true - '@roo-code/judgement@0.1.5': + '@roo-code/judgement@0.2.0': dependencies: picomatch: 4.0.7 diff --git a/pnpm-workspace.yaml b/pnpm-workspace.yaml index 10b03a453a..3902f00d08 100644 --- a/pnpm-workspace.yaml +++ b/pnpm-workspace.yaml @@ -6,7 +6,7 @@ minimumReleaseAge: 10080 # lockfile-affecting changes while it finishes maturing. minimumReleaseAgeExclude: # First-party release reviewed with the Judgement integration. - - "@roo-code/judgement@0.1.5" + - "@roo-code/judgement@0.2.0" - jose # Allow the patched releases while they finish maturing. - brace-expansion