From 9b5b39b2c6a2da843391b61566abb02e44719067 Mon Sep 17 00:00:00 2001 From: Lily Shen <115414357+lilyshen0722@users.noreply.github.com> Date: Sun, 1 Nov 2026 21:18:18 -0800 Subject: [PATCH 1/2] fix(scout): a first reply answers what was asked in two or three sentences, then one next step Staging walk 2026-11-02: asked what it could do, Scout named five things in one sentence and the reply folded behind "Show more" on a phone. The model writes short answers to every other ask; the prompt's "answer the literal question" made the literal answer to "what can you do?" the whole catalogue. The prompt now budgets a reply at two or three short sentences, about 200 characters, the last one the next step, with commands or steps they asked for exempt; never lists capabilities; and repeats the budget in the hard rules, the last thing the model reads. The "Show me what you can do" start now asks for the one most useful thing rather than what Scout can do. Measured offline on staging LiteLLM with the runtime's own prompt and cue builders (deepseek-v4-flash, 9 first posts, 3 samples each): median 265 -> 181 characters, desktop folds 6/27 -> 0/27, phone folds 19/27 -> 4/27. Command answers keep the connect card and the OS question. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_015AKgpQ2pr6xRMhN5mcfoHF --- .../nativeAgents.scoutReplyLength.test.js | 28 +++++++++++++++++++ backend/config/native-agents/scout.ts | 15 ++++++++-- 2 files changed, 40 insertions(+), 3 deletions(-) create mode 100644 backend/__tests__/unit/config/nativeAgents.scoutReplyLength.test.js diff --git a/backend/__tests__/unit/config/nativeAgents.scoutReplyLength.test.js b/backend/__tests__/unit/config/nativeAgents.scoutReplyLength.test.js new file mode 100644 index 0000000000..50a0de6ed6 --- /dev/null +++ b/backend/__tests__/unit/config/nativeAgents.scoutReplyLength.test.js @@ -0,0 +1,28 @@ +/** + * Scout's first replies stay short (F60, staging walk 2026-11-02). + * + * Asked what it could do, Scout named five things in one sentence, and the reply + * folded behind "Show more" on a phone. The prompt told it to answer the literal + * question, and the literal answer to "what can you do?" is the whole catalogue. + * Measured offline on the same model before this rule (9 first posts, 3 samples + * each): median 265 characters, 6 of 27 replies past the desktop fold, 19 of 27 + * past it at phone width. With it: 181, 0 of 27, 4 of 27. + */ +const { scoutApp } = require('../../../config/native-agents/scout'); + +describe('Scout reply length', () => { + const prompt = scoutApp.systemPrompt; + + it('budgets every reply and names the case that ran long', () => { + expect(prompt).toContain('Keep a reply to two or three short sentences, about 200 characters'); + expect(prompt).toContain('Only commands or steps they asked for may run longer'); + expect(prompt).toContain('Never list what you or Commonly can do'); + expect(prompt).toContain('name the one thing most useful to them now and offer to do it'); + }); + + it('repeats the budget in the hard rules, the last thing the model reads', () => { + const hardRules = prompt.slice(prompt.lastIndexOf('ROLE-SPECIFIC HARD RULES:')); + expect(hardRules).toContain('A first impression is also short'); + expect(hardRules).toContain('never a list of what you can do'); + }); +}); diff --git a/backend/config/native-agents/scout.ts b/backend/config/native-agents/scout.ts index 2da10e8b97..7ee6cf0b16 100644 --- a/backend/config/native-agents/scout.ts +++ b/backend/config/native-agents/scout.ts @@ -106,8 +106,8 @@ export const scoutApp = { + 'the person\'s own message: ' + `"${startLabel('ask-scout')}", "${startLabel('connect-agent')}" and "${startLabel('start-pod')}". ` + `For "${startLabel('ask-scout')}", answer in two or three short sentences: ` - + 'name what you can do for them here, then offer one thing you can do right ' - + 'now in this pod, such as putting their first task on the board. ' + + 'name the one thing most useful to them here, not a list, then offer to do ' + + 'it right now in this pod, such as putting their first task on the board. ' + `For "${startLabel('connect-agent')}", find out which tool they run ` + '(Claude Code or Codex) and their operating system, one question per turn, ' + 'then propose the connect card when the card rule below allows it; in that first reply, also say in one short ' @@ -316,6 +316,13 @@ export const scoutApp = { + 'HOW YOU ANSWER:\n' + '- Answer the literal question first, in your first sentence, with the ' + 'concrete step or command. Then stop.\n' + // F60 (staging walk 2026-11-02): asked what it can do, Scout named five + // things in one sentence, and the reply folded behind "Show more" on a phone. + + '- Keep a reply to two or three short sentences, about 200 characters, and ' + + 'make the last one the next step. Only commands or steps they asked for may ' + + 'run longer. Never list what you or Commonly can do: asked what you can do, ' + + 'what this is, or just greeted, name the one thing most useful to them now ' + + 'and offer to do it.\n' + '- Ask at most one question per turn, and only when the answer truly ' + 'depends on it (their OS, or which tool they run).\n' + '- Before ANY terminal command: if you do not know their operating ' @@ -365,7 +372,9 @@ export const scoutApp = { 'Never claim the user\'s own agent is connected or working, or that it ' + 'is not, unless commonly_agent_status said so in this turn. And ' + 'remember you are the product\'s first impression: an invented product ' - + 'fact from you is worse than one from anyone else.', + + 'fact from you is worse than one from anyone else. A first impression is ' + + 'also short: two or three short sentences, the last one the next step, ' + + 'and never a list of what you can do.', ), model: 'deepseek-v4-flash', triggers: ['mention', 'chat.message'], From 9fa3a5b5b8356456feb0175b001943d265cc3cd8 Mon Sep 17 00:00:00 2001 From: Lily Shen <115414357+lilyshen0722@users.noreply.github.com> Date: Sun, 1 Nov 2026 21:44:38 -0800 Subject: [PATCH 2/2] fix(scout): the reply budget exempts required caveats and named options Review of the first commit: the hard-rules copy of the budget carried no exemption, so the line the model reads last could cut the Linux invite-only caveat, the Windows hosted path or the second half of an MCP answer. Both copies now exempt a command or step the person asked for and any caveat the rules require. The ban is narrowed to volunteering a list: asked which tools, models or paths work, Scout names them. Offline A/B with four caveat cases added (Linux, Windows, "what can I connect?", DeepSeek; 3 samples each): the Linux caveat appears 3/3, Windows offers the hosted path 3/3, both list questions name the options. The 9 original posts: median 204 characters, desktop folds 1/27, phone 5/27 (main: 265, 6/27, 19/27). Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_015AKgpQ2pr6xRMhN5mcfoHF --- .../nativeAgents.scoutReplyLength.test.js | 17 +++++++++++------ backend/config/native-agents/scout.ts | 12 +++++++----- 2 files changed, 18 insertions(+), 11 deletions(-) diff --git a/backend/__tests__/unit/config/nativeAgents.scoutReplyLength.test.js b/backend/__tests__/unit/config/nativeAgents.scoutReplyLength.test.js index 50a0de6ed6..390cdeb1ff 100644 --- a/backend/__tests__/unit/config/nativeAgents.scoutReplyLength.test.js +++ b/backend/__tests__/unit/config/nativeAgents.scoutReplyLength.test.js @@ -14,15 +14,20 @@ describe('Scout reply length', () => { const prompt = scoutApp.systemPrompt; it('budgets every reply and names the case that ran long', () => { - expect(prompt).toContain('Keep a reply to two or three short sentences, about 200 characters'); - expect(prompt).toContain('Only commands or steps they asked for may run longer'); - expect(prompt).toContain('Never list what you or Commonly can do'); - expect(prompt).toContain('name the one thing most useful to them now and offer to do it'); + expect(prompt).toContain('two or three short sentences, about 200 characters'); + expect(prompt).toContain('Never volunteer a list of what you or Commonly can do'); + expect(prompt).toContain('name the one thing most useful to them now'); }); - it('repeats the budget in the hard rules, the last thing the model reads', () => { + it('exempts what the person asked for and every caveat the rules require', () => { + expect(prompt).toContain('a caveat these rules tell you to state, may run longer'); + expect(prompt).toContain('Asked which tools, models or paths work, name them'); + }); + + it('repeats the budget in the hard rules, the last thing the model reads, with its exemption', () => { const hardRules = prompt.slice(prompt.lastIndexOf('ROLE-SPECIFIC HARD RULES:')); expect(hardRules).toContain('A first impression is also short'); - expect(hardRules).toContain('never a list of what you can do'); + expect(hardRules).toContain('unless a command or a caveat these rules require needs more'); + expect(hardRules).toContain('never a tour of what you can do'); }); }); diff --git a/backend/config/native-agents/scout.ts b/backend/config/native-agents/scout.ts index 7ee6cf0b16..91794a2a53 100644 --- a/backend/config/native-agents/scout.ts +++ b/backend/config/native-agents/scout.ts @@ -319,10 +319,11 @@ export const scoutApp = { // F60 (staging walk 2026-11-02): asked what it can do, Scout named five // things in one sentence, and the reply folded behind "Show more" on a phone. + '- Keep a reply to two or three short sentences, about 200 characters, and ' - + 'make the last one the next step. Only commands or steps they asked for may ' - + 'run longer. Never list what you or Commonly can do: asked what you can do, ' - + 'what this is, or just greeted, name the one thing most useful to them now ' - + 'and offer to do it.\n' + + 'make the last one the next step. Only a command or step they asked for, or ' + + 'a caveat these rules tell you to state, may run longer. Never volunteer a ' + + 'list of what you or Commonly can do: asked what you can do, what this is, ' + + 'or just greeted, name the one thing most useful to them now and offer to ' + + 'do it. Asked which tools, models or paths work, name them.\n' + '- Ask at most one question per turn, and only when the answer truly ' + 'depends on it (their OS, or which tool they run).\n' + '- Before ANY terminal command: if you do not know their operating ' @@ -374,7 +375,8 @@ export const scoutApp = { + 'remember you are the product\'s first impression: an invented product ' + 'fact from you is worse than one from anyone else. A first impression is ' + 'also short: two or three short sentences, the last one the next step, ' - + 'and never a list of what you can do.', + + 'unless a command or a caveat these rules require needs more, and never ' + + 'a tour of what you can do.', ), model: 'deepseek-v4-flash', triggers: ['mention', 'chat.message'],