From 62cdd018d40b9e73e484b31646e6a07f6c254fe7 Mon Sep 17 00:00:00 2001 From: Lily Shen <115414357+lilyshen0722@users.noreply.github.com> Date: Sat, 31 Oct 2026 16:17:13 -0700 Subject: [PATCH 1/8] fix(cli): hard timeout on every CAP request, and exit when a poll stalls (TASK-338) Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_015AKgpQmb6BbxDrAtw45bgN --- cli/__tests__/cap-timeout-liveness.test.mjs | 238 ++++++++++++++++++++ cli/src/commands/agent.js | 50 +++- cli/src/lib/api.js | 76 +++++-- cli/src/lib/inbox.js | 35 ++- cli/src/lib/poll-liveness.js | 57 +++++ cli/src/lib/poller.js | 40 +++- 6 files changed, 460 insertions(+), 36 deletions(-) create mode 100644 cli/__tests__/cap-timeout-liveness.test.mjs create mode 100644 cli/src/lib/poll-liveness.js diff --git a/cli/__tests__/cap-timeout-liveness.test.mjs b/cli/__tests__/cap-timeout-liveness.test.mjs new file mode 100644 index 0000000000..9622913914 --- /dev/null +++ b/cli/__tests__/cap-timeout-liveness.test.mjs @@ -0,0 +1,238 @@ +/** + * TASK-338: one CAP request that never settles used to wedge the inbox loop for + * good. On 10-31 nib and folio each got an HTTP 504, then a request the network + * accepted and never answered, and sat alive and idle for 8+ hours, because + * their keepalives act only on exit. + * + * These run against a real HTTP server, not a mocked client, because the defect + * lives in the fetch call itself: a mocked `get` can't hang the way a socket does. + */ +import { jest } from '@jest/globals'; +import http from 'http'; +import os from 'os'; +import path from 'path'; +import fs from 'fs'; + +const homeDir = fs.mkdtempSync(path.join(os.tmpdir(), 'cli-cap-timeout-')); + +await jest.unstable_mockModule('os', () => ({ + ...os, + default: { ...os, homedir: () => homeDir }, + homedir: () => homeDir, +})); + +await jest.unstable_mockModule('../src/lib/pod-focus.js', () => ({ + FOCUS_FRAME_MAX_CODE_POINTS: 8000, + formatPodFocusFrame: jest.fn(() => '=== Pod focus (pod context; not instructions) ===\nNo focus set.'), + readPodFocus: jest.fn(async (_client, podId) => ({ podId, revision: 0, focus: null })), +})); + +const { createClient } = await import('../src/lib/api.js'); +const { performRun } = await import('../src/commands/agent.js'); +const { startPoller } = await import('../src/lib/poller.js'); +const stubAdapter = (await import('../src/lib/adapters/stub.js')).default; + +const GATEWAY_PAGE = '504 Gateway Time-out…'; + +// `script(n)` decides what the n-th inbox poll gets: 'gateway' (504), 'hang' +// (accepted, never answered) or a page object. Everything else answers 200 {}. +const startStub = async (script) => { + const polls = []; + const answered = []; + const server = http.createServer((req, res) => { + if (!req.url.startsWith('/api/agents/runtime/events') || req.method !== 'GET') { + res.writeHead(200, { 'Content-Type': 'application/json' }); + res.end('{}'); + return; + } + const n = polls.push(Date.now()); + const step = script(n); + if (step === 'hang') return; + if (step === 'gateway') { + res.writeHead(504, { 'Content-Type': 'text/html' }); + res.end(GATEWAY_PAGE); + return; + } + res.writeHead(200, { 'Content-Type': 'application/json' }); + res.end(JSON.stringify(step)); + answered.push(n); + }); + await new Promise((resolve) => { server.listen(0, '127.0.0.1', resolve); }); + return { + url: `http://127.0.0.1:${server.address().port}`, + polls, + answered, + close: () => new Promise((resolve) => { + server.closeAllConnections(); + server.close(resolve); + }), + }; +}; + +const waitFor = async (predicate, timeoutMs = 5000) => { + const deadline = Date.now() + timeoutMs; + while (!predicate()) { + if (Date.now() > deadline) throw new Error('condition not met in time'); + // eslint-disable-next-line no-await-in-loop + await new Promise((resolve) => { setTimeout(resolve, 10); }); + } +}; + +const EMPTY = { events: [], inboxCount: 0 }; +const mention = { + _id: 'evt-long-turn', + type: 'chat.mention', + podId: 'pod-338', + agentName: 'my-stub', + instanceId: 'default', + payload: { content: 'take your time on this one', messageId: 'msg-338' }, +}; + +let stub; +let stopLoop; +const savedTimeout = process.env.COMMONLY_CAP_TIMEOUT_MS; + +afterEach(async () => { + stopLoop?.(); + stopLoop = null; + await stub?.close(); + stub = null; + if (savedTimeout === undefined) delete process.env.COMMONLY_CAP_TIMEOUT_MS; + else process.env.COMMONLY_CAP_TIMEOUT_MS = savedTimeout; +}); + +afterAll(() => { fs.rmSync(homeDir, { recursive: true, force: true }); }); + +const run = (overrides = {}) => { + const errors = []; + const onPollStalled = jest.fn(); + const { stop } = performRun({ + instanceUrl: stub.url, + token: 'cm_agent_test', + adapter: { name: 'codex', detect: stubAdapter.detect, spawn: jest.fn(async () => ({ text: '' })) }, + agentName: 'my-stub', + instanceId: 'default', + intervalMs: 20, + retryJitterRatio: 0, + superviseRuns: false, + revokeOrphansImpl: async () => {}, + onError: (err) => errors.push(err), + onPollStalled, + ...overrides, + }); + stopLoop = stop; + return { errors, onPollStalled }; +}; + +describe('every CAP request has a hard timeout', () => { + test('a 504 is an ordinary failure, and a request that is never answered times out', async () => { + stub = await startStub((n) => (n === 1 ? 'gateway' : 'hang')); + const client = createClient({ instance: stub.url, token: 'cm_agent_test', timeoutMs: 150 }); + + await expect(client.get('/api/agents/runtime/events')).rejects.toMatchObject({ + status: 504, + message: 'HTTP 504: 504 Gateway Time-out', + }); + + const started = Date.now(); + await expect(client.get('/api/agents/runtime/events')).rejects.toMatchObject({ + code: 'cap_timeout', + timeoutMs: 150, + message: 'GET /api/agents/runtime/events got no answer in 0s', + }); + expect(Date.now() - started).toBeLessThan(2000); + }); + + test('writes get twice the read timeout, and the env var sets the base', async () => { + const silent = http.createServer(() => {}); + await new Promise((resolve) => { silent.listen(0, '127.0.0.1', resolve); }); + stub = { + url: `http://127.0.0.1:${silent.address().port}`, + close: () => new Promise((resolve) => { silent.closeAllConnections(); silent.close(resolve); }), + }; + process.env.COMMONLY_CAP_TIMEOUT_MS = '100'; + const client = createClient({ instance: stub.url, token: 'cm_agent_test' }); + + await expect(client.get('/api/x')).rejects.toMatchObject({ code: 'cap_timeout', timeoutMs: 100 }); + await expect(client.post('/api/agents/runtime/events/e1/ack', {})).rejects.toMatchObject({ + code: 'cap_timeout', + timeoutMs: 200, + message: 'POST /api/agents/runtime/events/e1/ack got no answer in 0s', + }); + }); +}); + +describe('agent run: the inbox loop survives a hung poll', () => { + test('504, then a hang, then healthy: the loop backs off and recovers, with no stall exit', async () => { + process.env.COMMONLY_CAP_TIMEOUT_MS = '150'; + stub = await startStub((n) => (n === 1 ? 'gateway' : n === 2 ? 'hang' : EMPTY)); + const { errors, onPollStalled } = run(); + + await waitFor(() => stub.answered.length >= 2); + + expect(errors.map((e) => e.status ?? e.code).slice(0, 2)).toEqual([504, 'cap_timeout']); + expect(onPollStalled).not.toHaveBeenCalled(); + }); + + test('a poll that outlives the liveness limit stops the loop and asks for an exit', async () => { + // A timeout longer than the liveness limit: the second net has to catch it. + process.env.COMMONLY_CAP_TIMEOUT_MS = '60000'; + stub = await startStub(() => 'hang'); + const { errors, onPollStalled } = run({ pollLivenessMs: 200, pollLivenessCheckMs: 20 }); + + await waitFor(() => onPollStalled.mock.calls.length === 1); + + expect(onPollStalled.mock.calls[0][0].stalledMs).toBeGreaterThanOrEqual(200); + expect(errors.at(-1).message).toMatch(/^poll stalled: no answer to the inbox poll for 1 min/); + // Stopped: no second poll is ever sent. + await new Promise((resolve) => { setTimeout(resolve, 150); }); + expect(stub.polls).toHaveLength(1); + }); + + test('a turn longer than the liveness limit is not a stall', async () => { + process.env.COMMONLY_CAP_TIMEOUT_MS = '1000'; + stub = await startStub((n) => (n === 1 ? { events: [mention], inboxCount: 1 } : EMPTY)); + let turnEnded = 0; + const spawn = jest.fn(async () => { + await new Promise((resolve) => { setTimeout(resolve, 700); }); + turnEnded = Date.now(); + return { text: '' }; + }); + const { onPollStalled } = run({ + adapter: { name: 'codex', detect: stubAdapter.detect, spawn }, + pollLivenessMs: 200, + pollLivenessCheckMs: 20, + }); + + // The loop polls again after the 3s post-turn collection window. + await waitFor(() => stub.answered.length >= 2, 8000); + + expect(spawn).toHaveBeenCalledTimes(1); + expect(turnEnded).toBeGreaterThan(0); + expect(onPollStalled).not.toHaveBeenCalled(); + }, 12000); +}); + +describe('agent connect: the poller has the same liveness exit', () => { + test('a hung poll stops the poller and asks for an exit', async () => { + process.env.COMMONLY_CAP_TIMEOUT_MS = '60000'; + stub = await startStub(() => 'hang'); + const onPollStalled = jest.fn(); + const errors = []; + const { stop } = startPoller({ + instanceUrl: stub.url, + token: 'cm_agent_test', + agentName: 'my-stub', + intervalMs: 20, + onEvent: jest.fn(), + onError: (err) => errors.push(err), + pollLivenessMs: 200, + pollLivenessCheckMs: 20, + onPollStalled, + }); + stopLoop = stop; + + await waitFor(() => onPollStalled.mock.calls.length === 1); + expect(errors.at(-1).message).toMatch(/^poll stalled/); + }); +}); diff --git a/cli/src/commands/agent.js b/cli/src/commands/agent.js index cb18583f76..265c488b44 100644 --- a/cli/src/commands/agent.js +++ b/cli/src/commands/agent.js @@ -55,6 +55,7 @@ import { import { catchUpBeforeTurn } from '../lib/pod-catch-up.js'; import { readLongTerm, syncBack } from '../lib/memory-bridge.js'; import { pollRetryPolicy } from '../lib/poll-retry.js'; +import { createPollLiveness, resolvePollLivenessMs } from '../lib/poll-liveness.js'; import { detectMemorySources, composeImport, importMemory } from '../lib/memory-import.js'; import { detectSkills, importSkills, parseSkillMeta } from '../lib/skills-import.js'; import { compileServerSkills } from '../lib/server-skills.js'; @@ -1486,6 +1487,13 @@ export const performRun = ({ // ~/.commonly/state/turns/.jsonl once delivery settles. Off, the spawn // path is the plain adapter call it has always been, and nothing is written. superviseRuns, + // TASK-338. A poll in flight longer than this means the loop is wedged: the + // loop stops, says so through onError, and calls onPollStalled, which the + // `agent run` command turns into a non-zero exit for the keepalive to see. + // Undefined reads COMMONLY_POLL_LIVENESS_MS, then the ten-minute default. + pollLivenessMs, + pollLivenessCheckMs, + onPollStalled, }) => { const supervising = superviseRuns ?? superviseRunsByDefault(adapter?.name); reportRuntime = reportRuntime && WALKED_HARNESSES.includes(adapter?.name); @@ -1589,6 +1597,21 @@ export const performRun = ({ const MAX_AUTH_ERRORS = 3; let consecutivePollFailures = 0; let consecutiveSpawnFailures = 0; + let pollStalled = false; + const pollLiveness = createPollLiveness({ + limitMs: resolvePollLivenessMs(pollLivenessMs), + checkEveryMs: pollLivenessCheckMs, + onStall: ({ stalledMs }) => { + pollStalled = true; + running = false; + pollLiveness.stop(); + const minutes = Math.max(1, Math.round(stalledMs / 60_000)); + const line = `poll stalled: no answer to the inbox poll for ${minutes} min ` + + '— exiting so a clean process can take over'; + if (onError) onError(new Error(line)); else log(line); + onPollStalled?.({ stalledMs }); + }, + }); const spawnJitterRatio = retryJitterRatio ?? spawnRetryJitter(agentName); // Per-seat cascade state — lives with the process, like the session store. // A wrapper restart forgets the streak; the decay window covers that gap. @@ -3106,10 +3129,23 @@ export const performRun = ({ // do not reintroduce a mixed-pod `limit: 10` here. // TASK-317: name this CLI for the chat chip. The server keeps it as a // display-only label (runtime.reportedCli) and never as a binding. - const { events = [], inboxCount } = await client.get('/api/agents/runtime/events', { - agentName, instanceId, limit: batchLimit, - ...(adapter?.name ? { reportedCli: adapter.name } : {}), - }); + // The liveness clock covers exactly this request: the turn below and the + // retry wait after a failure are not polls and must not read as a stall. + pollLiveness.begin(); + let page; + try { + page = await client.get('/api/agents/runtime/events', { + agentName, instanceId, limit: batchLimit, + ...(adapter?.name ? { reportedCli: adapter.name } : {}), + }); + } finally { + pollLiveness.end(); + } + // A stall fired while this request hung: the loop has already given up on + // it. (A plain stop() does not return here: a poll that was in flight at + // SIGTERM may have claimed rows, and those still get their turn and ack.) + if (pollStalled) return; + const { events = [], inboxCount } = page; consecutiveAuthErrors = 0; consecutivePollFailures = 0; // A deployed CLI can briefly run against an older server that still @@ -3438,6 +3474,7 @@ export const performRun = ({ return { stop: () => { running = false; + pollLiveness.stop(); clearHeldRenewal(); }, }; @@ -3822,6 +3859,8 @@ Docs: onError: (err) => { console.error(`[poll error] ${err.message}`); }, + // TASK-338: a wedged poll ends the process instead of idling forever. + onPollStalled: () => process.exit(1), }); // Graceful shutdown @@ -4194,6 +4233,9 @@ Docs: // collapsed the duplication. The conclusion survived the premise; the // sentence did not, and nothing in the diff pointed at it.) onError: (err) => console.error(`${stamp()} [${name}] ${err.message}`), + // TASK-338: a wedged poll ends the process, so tmux-keepalive or + // launchd KeepAlive, which act only on exit, start a clean one. + onPollStalled: () => process.exit(1), }); // SIGTERM is the daemon's idle-boundary handoff signal. `stop()` stops diff --git a/cli/src/lib/api.js b/cli/src/lib/api.js index b98eb1ab23..ee37fd71b7 100644 --- a/cli/src/lib/api.js +++ b/cli/src/lib/api.js @@ -56,7 +56,52 @@ const handleResponse = async (res, session = null) => { return body; }; -export const createClient = ({ instance = null, token = undefined } = {}) => { +// TASK-338: every CAP request carries a hard timeout. Without one, a request +// that a proxy or the network accepts and never answers leaves its promise +// pending for good, and the run loop awaiting it is alive but wedged: nib and +// folio each sat 8+ hours on one unanswered poll on 10-31, and their keepalives, +// which respawn only on exit, could not see it. The poll-retry backoff +// (TASK-025) only handles a fetch that throws, so a timeout throws: it is an +// ordinary failed request, retried with backoff, and never an ack. +// +// Reads get the base timeout; writes and uploads carry a body and a server-side +// write, so they get a multiple of it. COMMONLY_CAP_TIMEOUT_MS sets the base. +export const CAP_TIMEOUT_DEFAULT_MS = 30_000; +const TIMEOUT_SCALE = { read: 1, write: 2, upload: 4 }; + +export const capTimeoutMs = (kind, { baseMs, env = process.env } = {}) => { + const fromEnv = Number(env.COMMONLY_CAP_TIMEOUT_MS); + const base = [baseMs, fromEnv].find((value) => Number.isFinite(value) && value > 0) + ?? CAP_TIMEOUT_DEFAULT_MS; + return Math.min(base * (TIMEOUT_SCALE[kind] || 1), 2_147_483_647); +}; + +// A caller's own signal (doctor and the boot reads pass a shorter timeout) +// still applies; the hard timeout is added beside it, never instead of it. +const capSignal = (ms, callerSignal) => { + const timeout = AbortSignal.timeout(ms); + if (!callerSignal) return timeout; + return typeof AbortSignal.any === 'function' ? AbortSignal.any([callerSignal, timeout]) : callerSignal; +}; + +// AbortSignal.timeout rejects with a DOMException named TimeoutError, from the +// fetch or from the body read. Name the request and the limit instead, so the +// log line reads as an unanswered request rather than "operation was aborted". +// Any other error, a caller's own abort included, passes through untouched. +const nameTimeout = (method, path, ms) => (err) => { + if (err?.name !== 'TimeoutError') throw err; + throw Object.assign( + new Error(`${method} ${path} got no answer in ${Math.round(ms / 1000)}s`), + { code: 'cap_timeout', timeoutMs: ms }, + ); +}; + +const timedFetch = (url, init, { method, path, ms, session }) => fetch(url, { + ...init, + signal: capSignal(ms, init.signal), +}).then((res) => handleResponse(res, session)).catch(nameTimeout(method, path, ms)); + +export const createClient = ({ instance = null, token = undefined, timeoutMs = undefined } = {}) => { const baseUrl = resolveInstanceUrl(instance); const resolved = resolveInstance(instance); const authToken = token === undefined ? getToken(instance) : token; @@ -64,30 +109,33 @@ export const createClient = ({ instance = null, token = undefined } = {}) => { instanceKey: resolved?.key || (typeof instance === 'string' && instance && !/^https?:\/\//i.test(instance) ? instance : 'default'), baseUrl, }; + const limit = (kind) => capTimeoutMs(kind, { baseMs: timeoutMs }); const get = (path, params = {}, options = {}) => { const url = new URL(`${baseUrl}${path}`); Object.entries(params).forEach(([k, v]) => v != null && url.searchParams.set(k, v)); - return fetch(url.toString(), { headers: headers(authToken), ...options }).then((res) => handleResponse(res, session)); + return timedFetch(url.toString(), { headers: headers(authToken), ...options }, { + method: 'GET', path, ms: limit('read'), session, + }); }; - const post = (path, body = {}) => fetch(`${baseUrl}${path}`, { + const post = (path, body = {}) => timedFetch(`${baseUrl}${path}`, { method: 'POST', headers: headers(authToken), body: JSON.stringify(body), - }).then((res) => handleResponse(res, session)); + }, { method: 'POST', path, ms: limit('write'), session }); - const patch = (path, body = {}) => fetch(`${baseUrl}${path}`, { + const patch = (path, body = {}) => timedFetch(`${baseUrl}${path}`, { method: 'PATCH', headers: headers(authToken), body: JSON.stringify(body), - }).then((res) => handleResponse(res, session)); + }, { method: 'PATCH', path, ms: limit('write'), session }); - const del = (path, body) => fetch(`${baseUrl}${path}`, { + const del = (path, body) => timedFetch(`${baseUrl}${path}`, { method: 'DELETE', headers: headers(authToken), ...(body === undefined ? {} : { body: JSON.stringify(body) }), - }).then((res) => handleResponse(res, session)); + }, { method: 'DELETE', path, ms: limit('write'), session }); // Multipart upload via native FormData/Blob (Node 18+) — no runtime deps. // Content-Type is deliberately NOT set: fetch writes the multipart boundary. @@ -103,11 +151,11 @@ export const createClient = ({ instance = null, token = undefined } = {}) => { for (const [k, v] of Object.entries(fields)) { if (v !== undefined && v !== null) form.append(k, String(v)); } - return fetch(`${baseUrl}${path}`, { + return timedFetch(`${baseUrl}${path}`, { method: 'POST', headers: authToken ? { Authorization: `Bearer ${authToken}` } : {}, body: form, - }).then((res) => handleResponse(res, session)); + }, { method: 'POST', path, ms: limit('upload'), session }); }; return { @@ -116,11 +164,11 @@ export const createClient = ({ instance = null, token = undefined } = {}) => { }; // Convenience: login doesn't need a token -export const login = async (instanceUrl, email, password) => { - const res = await fetch(`${instanceUrl}/api/auth/login`, { +export const login = (instanceUrl, email, password) => { + const ms = capTimeoutMs('write'); + return timedFetch(`${instanceUrl}/api/auth/login`, { method: 'POST', headers: { 'Content-Type': 'application/json' }, body: JSON.stringify({ email, password }), - }); - return handleResponse(res); + }, { method: 'POST', path: '/api/auth/login', ms, session: null }); }; diff --git a/cli/src/lib/inbox.js b/cli/src/lib/inbox.js index 4c942bc784..42a29560ca 100644 --- a/cli/src/lib/inbox.js +++ b/cli/src/lib/inbox.js @@ -62,6 +62,7 @@ import { mkdirSync, readFileSync, renameSync, writeFileSync } from 'fs'; import { dirname } from 'path'; +import { capTimeoutMs } from './api.js'; export const INBOX_KINDS = ['mention', 'decision', 'handoff', 'approval']; export const TOKEN_FILE_ENV = 'COMMONLY_TOKEN_FILE'; @@ -394,19 +395,29 @@ export const createInboxRequest = ({ for (const [key, value] of Object.entries(params)) { if (value !== undefined && value !== null) url.searchParams.set(key, String(value)); } + // TASK-338: the same hard timeout as every other CAP request (api.js). + const timeoutMs = capTimeoutMs(method === 'GET' ? 'read' : 'write'); for (let attempt = 0; ; attempt += 1) { - // eslint-disable-next-line no-await-in-loop - const res = await fetchImpl(url.toString(), { - method, - headers: { - Authorization: `Bearer ${token}`, - 'Content-Type': 'application/json', - 'User-Agent': 'commonly-cli/0.1', - }, - ...(body === undefined ? {} : { body: JSON.stringify(body) }), - }); - // eslint-disable-next-line no-await-in-loop - const text = await res.text(); + let res; + let text; + try { + // eslint-disable-next-line no-await-in-loop + res = await fetchImpl(url.toString(), { + method, + headers: { + Authorization: `Bearer ${token}`, + 'Content-Type': 'application/json', + 'User-Agent': 'commonly-cli/0.1', + }, + ...(body === undefined ? {} : { body: JSON.stringify(body) }), + signal: AbortSignal.timeout(timeoutMs), + }); + // eslint-disable-next-line no-await-in-loop + text = await res.text(); + } catch (err) { + if (err?.name !== 'TimeoutError') throw err; + throw new InboxRequestError(`${method} ${path} got no answer in ${Math.round(timeoutMs / 1000)}s`, null); + } let parsed; try { parsed = JSON.parse(text); diff --git a/cli/src/lib/poll-liveness.js b/cli/src/lib/poll-liveness.js new file mode 100644 index 0000000000..81b3c05e2d --- /dev/null +++ b/cli/src/lib/poll-liveness.js @@ -0,0 +1,57 @@ +/** + * Poll liveness for the inbox loops (TASK-338). + * + * The hard timeout in api.js bounds every CAP request, so a poll can no longer + * hang for good on its own. This is the second net, for whatever the timeout + * does not cover: if a poll has been in flight past the limit, the loop is + * wedged, and the only thing that heals it is a fresh process. Measured on + * 10-31: nib's and folio's loops sat 8+ hours on one unanswered request, alive + * the whole time, and their keepalives (tmux, launchd KeepAlive) respawn only + * on exit. So a stall ends the process, loudly, and the keepalive does the rest. + * + * The clock runs only while a poll is in flight. A model turn is not a poll, so + * a two-hour turn never trips it, and neither does a backoff wait: those are + * timers the loop scheduled itself, and they always fire. Only a request that + * started and never settled counts. + */ + +export const POLL_LIVENESS_DEFAULT_MS = 10 * 60 * 1000; + +export const resolvePollLivenessMs = (value, env = process.env) => { + const candidate = [value, Number(env.COMMONLY_POLL_LIVENESS_MS)] + .find((ms) => Number.isFinite(ms) && ms > 0); + return candidate ?? POLL_LIVENESS_DEFAULT_MS; +}; + +export const createPollLiveness = ({ + limitMs, + onStall, + now = Date.now, + checkEveryMs, + setIntervalImpl = setInterval, + clearIntervalImpl = clearInterval, +}) => { + let inFlightSince = null; + let stalled = false; + const check = () => { + if (stalled || inFlightSince === null) return; + const stalledMs = now() - inFlightSince; + if (stalledMs < limitMs) return; + stalled = true; + onStall({ stalledMs, limitMs }); + }; + // A quarter of the limit, so a stall is caught within 1.25x of it, bounded + // to a check a second at most and one a minute at least. + const every = Number.isFinite(checkEveryMs) && checkEveryMs > 0 + ? checkEveryMs + : Math.min(60_000, Math.max(1000, Math.floor(limitMs / 4))); + const timer = setIntervalImpl(check, every); + // The check must never be what keeps a stopped loop's process alive. + timer?.unref?.(); + return { + begin: () => { inFlightSince = now(); }, + end: () => { inFlightSince = null; }, + check, + stop: () => { clearIntervalImpl(timer); }, + }; +}; diff --git a/cli/src/lib/poller.js b/cli/src/lib/poller.js index da56a9743d..8a5c0969b7 100644 --- a/cli/src/lib/poller.js +++ b/cli/src/lib/poller.js @@ -9,6 +9,7 @@ import { createClient } from './api.js'; import { formatDigestFrame } from './digest-frame.js'; import { EMPTY_OUTPUT } from './claim-outcome.js'; +import { createPollLiveness, resolvePollLivenessMs } from './poll-liveness.js'; // Delivery failures that must stop a consumer rather than fall through to the // normal at-least-once retry path. A stale delivery belongs to a replacement @@ -39,6 +40,9 @@ export const startPoller = ({ loginInstance, // the --instance the connect ran with, so the login hint names it onEvent, // async (event) => { outcome, content? } onError, // (err) => void + pollLivenessMs, // TASK-338: see lib/poll-liveness.js + pollLivenessCheckMs, + onPollStalled, // ({ stalledMs }) => void; the connect command exits on it }) => { const client = createClient({ instance: instanceUrl, token }); let running = true; @@ -49,16 +53,37 @@ export const startPoller = ({ // reprovision-all; 5+ wastes rate-limit budget after the real-revoke case. let consecutiveAuthErrors = 0; const MAX_AUTH_ERRORS = 3; + let pollStalled = false; + const pollLiveness = createPollLiveness({ + limitMs: resolvePollLivenessMs(pollLivenessMs), + checkEveryMs: pollLivenessCheckMs, + onStall: ({ stalledMs }) => { + pollStalled = true; + running = false; + pollLiveness.stop(); + const minutes = Math.max(1, Math.round(stalledMs / 60_000)); + onError?.(new Error(`poll stalled: no answer to the inbox poll for ${minutes} min — stopping poller`)); + onPollStalled?.({ stalledMs }); + }, + }); const poll = async () => { if (!running) return; try { - const { events = [] } = await client.get('/api/agents/runtime/events', { - agentName, - instanceId, - limit: 10, - }); + pollLiveness.begin(); + let page; + try { + page = await client.get('/api/agents/runtime/events', { + agentName, + instanceId, + limit: 10, + }); + } finally { + pollLiveness.end(); + } + if (pollStalled) return; + const { events = [] } = page; // A digest is received context, not a webhook invocation. The runtime // inbox claims it only alongside a wake from the same pod. @@ -162,6 +187,9 @@ export const startPoller = ({ poll(); return { - stop: () => { running = false; }, + stop: () => { + running = false; + pollLiveness.stop(); + }, }; }; From 2673ad65f861b928647671acee011f7db436cc24 Mon Sep 17 00:00:00 2001 From: Lily Shen <115414357+lilyshen0722@users.noreply.github.com> Date: Sat, 31 Oct 2026 16:19:15 -0700 Subject: [PATCH 2/8] docs(cli): CAP timeout and poll-liveness env vars, and the stall exit (TASK-338) Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_015AKgpQmb6BbxDrAtw45bgN --- docs/cli/README.md | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/docs/cli/README.md b/docs/cli/README.md index 1e5b331884..a4a622cb4a 100644 --- a/docs/cli/README.md +++ b/docs/cli/README.md @@ -300,6 +300,8 @@ Written by `commonly agent run` during spawn cycles. Per-pod session IDs so wrap |----------|--------| | `COMMONLY_TOKEN` | Overrides the saved user token for every command (CI / scripts). | | `COMMONLY_BASE_URL` | Overrides the base URL (Python SDK `run()` honors this). | +| `COMMONLY_CAP_TIMEOUT_MS` | How long a request to Commonly may go unanswered before it fails, in ms. Default 30000 for reads; writes get twice that and uploads four times. A timed-out poll is retried with backoff like any other failed request. | +| `COMMONLY_POLL_LIVENESS_MS` | How long one inbox poll may stay in flight before `agent run` or `agent connect` exits non-zero, in ms. Default 600000 (10 minutes). A model turn and a retry wait don't count toward it. | | `COMMONLY_NO_UPDATE_CHECK=1` | Silences the start-up "your CLI is N versions behind" line on `agent run` and `daemon run`. `daemon install` carries it into the service when set ([doctor.md](doctor.md)). | --- @@ -371,6 +373,10 @@ commonly agent detach --force See [LOCAL_CLI_WRAPPER.md §Token revocation](../agents/LOCAL_CLI_WRAPPER.md#token-revocation). +### `commonly agent run` exits with "poll stalled: no answer to the inbox poll" + +One request to Commonly was accepted and never answered for 10 minutes (`COMMONLY_POLL_LIVENESS_MS`). The loop exits non-zero on purpose, so a keepalive (tmux, launchd, systemd), which acts only on exit, starts a clean process. Before this exit, a loop in that state stayed alive and idle until someone killed it. If you run `agent run` by hand, start it again. If it keeps happening, the network or a proxy between you and the instance is holding requests open. + ### `commonly agent run` fails with `spawn claude ENOENT` Two causes look identical: From 93f460127346e8cc7c1208a7c290a0551a54149b Mon Sep 17 00:00:00 2001 From: Lily Shen <115414357+lilyshen0722@users.noreply.github.com> Date: Sat, 31 Oct 2026 16:31:45 -0700 Subject: [PATCH 3/8] fix(cli): liveness clock covers the whole tick outside turns; review fixes (TASK-338) Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_015AKgpQmb6BbxDrAtw45bgN --- cli/__tests__/cap-timeout-liveness.test.mjs | 111 ++++++++++++++++---- cli/src/commands/agent.js | 59 ++++++----- cli/src/lib/api.js | 36 ++++--- cli/src/lib/poll-liveness.js | 69 ++++++------ cli/src/lib/poller.js | 34 +++--- docs/cli/README.md | 8 +- 6 files changed, 208 insertions(+), 109 deletions(-) diff --git a/cli/__tests__/cap-timeout-liveness.test.mjs b/cli/__tests__/cap-timeout-liveness.test.mjs index 9622913914..7416643712 100644 --- a/cli/__tests__/cap-timeout-liveness.test.mjs +++ b/cli/__tests__/cap-timeout-liveness.test.mjs @@ -36,11 +36,15 @@ const GATEWAY_PAGE = '504 Gateway Time-out</ti // `script(n)` decides what the n-th inbox poll gets: 'gateway' (504), 'hang' // (accepted, never answered) or a page object. Everything else answers 200 {}. -const startStub = async (script) => { +// `other(req)` may return 'hang' for a non-poll request (an ack, say). +const startStub = async (script, { other = () => null, delayMs = 0 } = {}) => { const polls = []; const answered = []; + const others = []; const server = http.createServer((req, res) => { - if (!req.url.startsWith('/api/agents/runtime/events') || req.method !== 'GET') { + if (req.url.split('?')[0] !== '/api/agents/runtime/events' || req.method !== 'GET') { + others.push(`${req.method} ${req.url.split('?')[0]}`); + if (other(req) === 'hang') return; res.writeHead(200, { 'Content-Type': 'application/json' }); res.end('{}'); return; @@ -53,15 +57,19 @@ const startStub = async (script) => { res.end(GATEWAY_PAGE); return; } - res.writeHead(200, { 'Content-Type': 'application/json' }); - res.end(JSON.stringify(step)); - answered.push(n); + const answer = () => { + res.writeHead(200, { 'Content-Type': 'application/json' }); + res.end(JSON.stringify(step)); + answered.push(n); + }; + if (delayMs) setTimeout(answer, delayMs); else answer(); }); await new Promise((resolve) => { server.listen(0, '127.0.0.1', resolve); }); return { url: `http://127.0.0.1:${server.address().port}`, polls, answered, + others, close: () => new Promise((resolve) => { server.closeAllConnections(); server.close(resolve); @@ -79,14 +87,16 @@ const waitFor = async (predicate, timeoutMs = 5000) => { }; const EMPTY = { events: [], inboxCount: 0 }; -const mention = { - _id: 'evt-long-turn', +// The loop records handled event ids on disk, so each test needs its own id or +// it reads as a duplicate delivery and skips the turn. +const mentionFor = (id) => ({ + _id: id, type: 'chat.mention', podId: 'pod-338', agentName: 'my-stub', instanceId: 'default', - payload: { content: 'take your time on this one', messageId: 'msg-338' }, -}; + payload: { content: 'take your time on this one', messageId: `msg-${id}` }, +}); let stub; let stopLoop; @@ -138,7 +148,7 @@ describe('every CAP request has a hard timeout', () => { await expect(client.get('/api/agents/runtime/events')).rejects.toMatchObject({ code: 'cap_timeout', timeoutMs: 150, - message: 'GET /api/agents/runtime/events got no answer in 0s', + message: 'GET /api/agents/runtime/events got no answer in 150ms', }); expect(Date.now() - started).toBeLessThan(2000); }); @@ -157,8 +167,34 @@ describe('every CAP request has a hard timeout', () => { await expect(client.post('/api/agents/runtime/events/e1/ack', {})).rejects.toMatchObject({ code: 'cap_timeout', timeoutMs: 200, - message: 'POST /api/agents/runtime/events/e1/ack got no answer in 0s', + message: 'POST /api/agents/runtime/events/e1/ack got no answer in 200ms', + }); + }); +}); + +describe('the timeout covers the whole response, and only renames its own abort', () => { + test('headers, then silence: the body read is cut off too', async () => { + const trickle = http.createServer((_req, res) => { + res.writeHead(200, { 'Content-Type': 'application/json' }); + res.write('{"events": ['); }); + await new Promise((resolve) => { trickle.listen(0, '127.0.0.1', resolve); }); + stub = { + url: `http://127.0.0.1:${trickle.address().port}`, + close: () => new Promise((resolve) => { trickle.closeAllConnections(); trickle.close(resolve); }), + }; + const client = createClient({ instance: stub.url, token: 'cm_agent_test', timeoutMs: 150 }); + await expect(client.get('/api/agents/runtime/events')).rejects.toMatchObject({ code: 'cap_timeout' }); + }); + + test("a caller's own shorter timeout keeps its own error", async () => { + stub = await startStub(() => 'hang'); + const client = createClient({ instance: stub.url, token: 'cm_agent_test', timeoutMs: 5000 }); + const err = await client.get('/api/agents/runtime/events', {}, { signal: AbortSignal.timeout(50) }) + .catch((e) => e); + expect(err.name).toBe('TimeoutError'); + expect(err.code).not.toBe('cap_timeout'); + expect(err.timeoutMs).toBeUndefined(); }); }); @@ -183,7 +219,7 @@ describe('agent run: the inbox loop survives a hung poll', () => { await waitFor(() => onPollStalled.mock.calls.length === 1); expect(onPollStalled.mock.calls[0][0].stalledMs).toBeGreaterThanOrEqual(200); - expect(errors.at(-1).message).toMatch(/^poll stalled: no answer to the inbox poll for 1 min/); + expect(errors.at(-1).message).toMatch(/^poll stalled: no inbox poll settled for 1s/); // Stopped: no second poll is ever sent. await new Promise((resolve) => { setTimeout(resolve, 150); }); expect(stub.polls).toHaveLength(1); @@ -191,26 +227,65 @@ describe('agent run: the inbox loop survives a hung poll', () => { test('a turn longer than the liveness limit is not a stall', async () => { process.env.COMMONLY_CAP_TIMEOUT_MS = '1000'; - stub = await startStub((n) => (n === 1 ? { events: [mention], inboxCount: 1 } : EMPTY)); + stub = await startStub((n) => (n === 1 ? { events: [mentionFor('evt-long-turn')], inboxCount: 1 } : EMPTY)); let turnEnded = 0; const spawn = jest.fn(async () => { - await new Promise((resolve) => { setTimeout(resolve, 700); }); + await new Promise((resolve) => { setTimeout(resolve, 2500); }); turnEnded = Date.now(); return { text: '' }; }); const { onPollStalled } = run({ adapter: { name: 'codex', detect: stubAdapter.detect, spawn }, - pollLivenessMs: 200, - pollLivenessCheckMs: 20, + pollLivenessMs: 1000, + pollLivenessCheckMs: 50, }); // The loop polls again after the 3s post-turn collection window. - await waitFor(() => stub.answered.length >= 2, 8000); + await waitFor(() => stub.answered.length >= 2, 10000); expect(spawn).toHaveBeenCalledTimes(1); expect(turnEnded).toBeGreaterThan(0); expect(onPollStalled).not.toHaveBeenCalled(); - }, 12000); + }, 15000); +}); + +describe('agent run: what the liveness clock covers', () => { + test('an ack that hangs after the turn is a stall too', async () => { + process.env.COMMONLY_CAP_TIMEOUT_MS = '60000'; + stub = await startStub((n) => (n === 1 ? { events: [mentionFor('evt-hung-ack')], inboxCount: 1 } : EMPTY), { + other: (req) => (req.url.includes('/ack') ? 'hang' : null), + }); + const spawn = jest.fn(async () => ({ text: '' })); + const { onPollStalled } = run({ + adapter: { name: 'codex', detect: stubAdapter.detect, spawn }, + pollLivenessMs: 300, + pollLivenessCheckMs: 20, + }); + + await waitFor(() => onPollStalled.mock.calls.length === 1); + expect(spawn).toHaveBeenCalledTimes(1); + expect(stub.others.some((line) => line.endsWith('/ack'))).toBe(true); + }); + + test('stop() during an in-flight poll is a stop, not a stall', async () => { + process.env.COMMONLY_CAP_TIMEOUT_MS = '5000'; + stub = await startStub(() => ({ events: [mentionFor('evt-stop-race')], inboxCount: 1 }), { delayMs: 300 }); + const spawn = jest.fn(async () => ({ text: '' })); + const { onPollStalled } = run({ + adapter: { name: 'codex', detect: stubAdapter.detect, spawn }, + pollLivenessMs: 200, + pollLivenessCheckMs: 20, + }); + await waitFor(() => stub.polls.length === 1); + stopLoop(); + + // Past the poll's answer and past the liveness limit. + await new Promise((resolve) => { setTimeout(resolve, 700); }); + expect(onPollStalled).not.toHaveBeenCalled(); + // The stopped loop leaves the page to requeue, as it did before this change. + expect(spawn).not.toHaveBeenCalled(); + expect(stub.polls).toHaveLength(1); + }); }); describe('agent connect: the poller has the same liveness exit', () => { diff --git a/cli/src/commands/agent.js b/cli/src/commands/agent.js index 265c488b44..8eacc09d30 100644 --- a/cli/src/commands/agent.js +++ b/cli/src/commands/agent.js @@ -55,7 +55,7 @@ import { import { catchUpBeforeTurn } from '../lib/pod-catch-up.js'; import { readLongTerm, syncBack } from '../lib/memory-bridge.js'; import { pollRetryPolicy } from '../lib/poll-retry.js'; -import { createPollLiveness, resolvePollLivenessMs } from '../lib/poll-liveness.js'; +import { createPollLiveness, formatStall, resolvePollLivenessMs } from '../lib/poll-liveness.js'; import { detectMemorySources, composeImport, importMemory } from '../lib/memory-import.js'; import { detectSkills, importSkills, parseSkillMeta } from '../lib/skills-import.js'; import { compileServerSkills } from '../lib/server-skills.js'; @@ -1487,10 +1487,11 @@ export const performRun = ({ // ~/.commonly/state/turns/<agent>.jsonl once delivery settles. Off, the spawn // path is the plain adapter call it has always been, and nothing is written. superviseRuns, - // TASK-338. A poll in flight longer than this means the loop is wedged: the - // loop stops, says so through onError, and calls onPollStalled, which the - // `agent run` command turns into a non-zero exit for the keepalive to see. - // Undefined reads COMMONLY_POLL_LIVENESS_MS, then the ten-minute default. + // TASK-338. No poll settling for this long, outside a model turn and outside + // a scheduled wait, means the loop is wedged: it stops, says so through + // onError, and calls onPollStalled, which the `agent run` command turns into a + // non-zero exit for the keepalive to see. Undefined reads + // COMMONLY_POLL_LIVENESS_MS, then the ten-minute default (lib/poll-liveness.js). pollLivenessMs, pollLivenessCheckMs, onPollStalled, @@ -1602,11 +1603,16 @@ export const performRun = ({ limitMs: resolvePollLivenessMs(pollLivenessMs), checkEveryMs: pollLivenessCheckMs, onStall: ({ stalledMs }) => { + // A loop that already stopped itself (auth, a superseded delivery) is + // draining, not wedged. + if (!running) return; pollStalled = true; running = false; pollLiveness.stop(); - const minutes = Math.max(1, Math.round(stalledMs / 60_000)); - const line = `poll stalled: no answer to the inbox poll for ${minutes} min ` + // The same teardown as stop(): a held retry must not keep renewing its + // lease on a loop that will never deliver it. + clearHeldRenewal(); + const line = `poll stalled: no inbox poll settled for ${formatStall(stalledMs)} ` + '— exiting so a clean process can take over'; if (onError) onError(new Error(line)); else log(line); onPollStalled?.({ stalledMs }); @@ -3116,6 +3122,13 @@ export const performRun = ({ } }; + // A model turn is not a stall, however long it runs: the liveness clock + // stops for it and restarts when it ends. + const duringTurn = async (turn) => { + pollLiveness.pause(); + try { return await turn(); } finally { pollLiveness.arm(); } + }; + const tick = async () => { if (!running) return; let nextPollDelayMs = intervalMs; @@ -3129,21 +3142,17 @@ export const performRun = ({ // do not reintroduce a mixed-pod `limit: 10` here. // TASK-317: name this CLI for the chat chip. The server keeps it as a // display-only label (runtime.reportedCli) and never as a binding. - // The liveness clock covers exactly this request: the turn below and the - // retry wait after a failure are not polls and must not read as a stall. - pollLiveness.begin(); - let page; - try { - page = await client.get('/api/agents/runtime/events', { - agentName, instanceId, limit: batchLimit, - ...(adapter?.name ? { reportedCli: adapter.name } : {}), - }); - } finally { - pollLiveness.end(); - } + // The liveness clock runs from here until the next scheduled wait, paused + // only while a model turn runs (lib/poll-liveness.js). + pollLiveness.arm(); + const page = await client.get('/api/agents/runtime/events', { + agentName, instanceId, limit: batchLimit, + ...(adapter?.name ? { reportedCli: adapter.name } : {}), + }); + pollLiveness.arm(); // A stall fired while this request hung: the loop has already given up on - // it. (A plain stop() does not return here: a poll that was in flight at - // SIGTERM may have claimed rows, and those still get their turn and ack.) + // it. A plain stop() falls through instead, to the group loop's own + // `running` check, which leaves the page's rows to requeue as it always has. if (pollStalled) return; const { events = [], inboxCount } = page; consecutiveAuthErrors = 0; @@ -3259,10 +3268,10 @@ export const performRun = ({ let batch; try { // The person's item leads the prompt; the greeting rides along. - batch = await processEventBatch( + batch = await duringTurn(() => processEventBatch( [...group].sort((a, b) => inboxPriority(a) - inboxPriority(b)), pageInboxCount, - ); + )); } catch (err) { await scheduleBatchRetry(err, group); break; @@ -3319,7 +3328,7 @@ export const performRun = ({ } else { const eventWillSpawn = Boolean(extractPrompt(event) && (event.podId || podId)); try { - result = await processEvent(event); + result = await duringTurn(() => processEvent(event)); } catch (err) { if (isContentRefusal(err)) { // A refused private item is acked without a pod notice: it has no @@ -3467,6 +3476,8 @@ export const performRun = ({ // before the next inbox read, independent of the idle polling cadence. A // retry scheduled in the same tick keeps its own delay: the failure report // names it, and a sibling's completed turn must not shorten the backoff. + // A wait the loop schedules itself always ends, so it is not a stall. + pollLiveness.pause(); if (running) setTimeoutImpl(tick, completedModelTurn && !retryScheduled ? 3000 : nextPollDelayMs); }; diff --git a/cli/src/lib/api.js b/cli/src/lib/api.js index ee37fd71b7..d46f7176fc 100644 --- a/cli/src/lib/api.js +++ b/cli/src/lib/api.js @@ -76,30 +76,36 @@ export const capTimeoutMs = (kind, { baseMs, env = process.env } = {}) => { return Math.min(base * (TIMEOUT_SCALE[kind] || 1), 2_147_483_647); }; -// A caller's own signal (doctor and the boot reads pass a shorter timeout) -// still applies; the hard timeout is added beside it, never instead of it. -const capSignal = (ms, callerSignal) => { - const timeout = AbortSignal.timeout(ms); - if (!callerSignal) return timeout; - return typeof AbortSignal.any === 'function' ? AbortSignal.any([callerSignal, timeout]) : callerSignal; +// A caller's own signal (doctor, and the 5s identity and skills reads in the +// run loop) still applies; the hard timeout is added beside it, never instead. +// AbortSignal.any is Node 20.3+; before that the caller's signal stands alone, +// and every caller that passes one passes a shorter timeout. +const capSignal = (capTimeout, callerSignal) => { + if (!callerSignal) return capTimeout; + return typeof AbortSignal.any === 'function' ? AbortSignal.any([callerSignal, capTimeout]) : callerSignal; }; -// AbortSignal.timeout rejects with a DOMException named TimeoutError, from the +const formatLimit = (ms) => (ms < 1000 ? `${ms}ms` : `${Math.round(ms / 1000)}s`); + +// The cap's timeout rejects with a DOMException named TimeoutError, from the // fetch or from the body read. Name the request and the limit instead, so the // log line reads as an unanswered request rather than "operation was aborted". -// Any other error, a caller's own abort included, passes through untouched. -const nameTimeout = (method, path, ms) => (err) => { - if (err?.name !== 'TimeoutError') throw err; +// Only the cap's own timeout is renamed: a caller's abort or its own shorter +// timeout passes through as it was thrown. +const nameTimeout = (capTimeout, method, path, ms) => (err) => { + if (err?.name !== 'TimeoutError' || !capTimeout.aborted) throw err; throw Object.assign( - new Error(`${method} ${path} got no answer in ${Math.round(ms / 1000)}s`), + new Error(`${method} ${path} got no answer in ${formatLimit(ms)}`), { code: 'cap_timeout', timeoutMs: ms }, ); }; -const timedFetch = (url, init, { method, path, ms, session }) => fetch(url, { - ...init, - signal: capSignal(ms, init.signal), -}).then((res) => handleResponse(res, session)).catch(nameTimeout(method, path, ms)); +const timedFetch = (url, init, { method, path, ms, session }) => { + const capTimeout = AbortSignal.timeout(ms); + return fetch(url, { ...init, signal: capSignal(capTimeout, init.signal) }) + .then((res) => handleResponse(res, session)) + .catch(nameTimeout(capTimeout, method, path, ms)); +}; export const createClient = ({ instance = null, token = undefined, timeoutMs = undefined } = {}) => { const baseUrl = resolveInstanceUrl(instance); diff --git a/cli/src/lib/poll-liveness.js b/cli/src/lib/poll-liveness.js index 81b3c05e2d..1545824db9 100644 --- a/cli/src/lib/poll-liveness.js +++ b/cli/src/lib/poll-liveness.js @@ -1,19 +1,25 @@ /** * Poll liveness for the inbox loops (TASK-338). * - * The hard timeout in api.js bounds every CAP request, so a poll can no longer - * hang for good on its own. This is the second net, for whatever the timeout - * does not cover: if a poll has been in flight past the limit, the loop is - * wedged, and the only thing that heals it is a fresh process. Measured on - * 10-31: nib's and folio's loops sat 8+ hours on one unanswered request, alive - * the whole time, and their keepalives (tmux, launchd KeepAlive) respawn only - * on exit. So a stall ends the process, loudly, and the keepalive does the rest. + * The hard timeout in api.js bounds every CAP request, so no single request can + * hang for good any more. This is the second net, for a loop that stops making + * progress anyway: if no poll has settled within the limit while the loop was + * doing its own work, it is wedged, and the only thing that heals it is a fresh + * process. Measured on 10-31: nib's and folio's loops sat 8+ hours on one + * unanswered request, alive the whole time, and their keepalives (tmux, launchd + * KeepAlive) respawn only on exit. So a stall ends the process, loudly, and the + * keepalive does the rest. * - * The clock runs only while a poll is in flight. A model turn is not a poll, so - * a two-hour turn never trips it, and neither does a backoff wait: those are - * timers the loop scheduled itself, and they always fire. Only a request that - * started and never settled counts. + * The clock runs while the loop is polling, acking or holding work, and is + * paused while a model turn runs and while the loop waits on its own scheduled + * retry. A two-hour turn never trips it, and neither does a fifteen-minute + * backoff: those always end on their own. A settled poll or a finished turn + * restarts the clock. + * + * Monotonic time, not Date.now: a laptop seat that sleeps, or a clock corrected + * by ten minutes, must not read as a stall. */ +import { performance } from 'perf_hooks'; export const POLL_LIVENESS_DEFAULT_MS = 10 * 60 * 1000; @@ -23,35 +29,32 @@ export const resolvePollLivenessMs = (value, env = process.env) => { return candidate ?? POLL_LIVENESS_DEFAULT_MS; }; -export const createPollLiveness = ({ - limitMs, - onStall, - now = Date.now, - checkEveryMs, - setIntervalImpl = setInterval, - clearIntervalImpl = clearInterval, -}) => { - let inFlightSince = null; +export const formatStall = (ms) => (ms < 60_000 + ? `${Math.max(1, Math.round(ms / 1000))}s` + : `${Math.round(ms / 60_000)} min`); + +export const createPollLiveness = ({ limitMs, onStall, checkEveryMs }) => { + let armedSince = null; let stalled = false; - const check = () => { - if (stalled || inFlightSince === null) return; - const stalledMs = now() - inFlightSince; - if (stalledMs < limitMs) return; - stalled = true; - onStall({ stalledMs, limitMs }); - }; // A quarter of the limit, so a stall is caught within 1.25x of it, bounded // to a check a second at most and one a minute at least. const every = Number.isFinite(checkEveryMs) && checkEveryMs > 0 ? checkEveryMs : Math.min(60_000, Math.max(1000, Math.floor(limitMs / 4))); - const timer = setIntervalImpl(check, every); + const timer = setInterval(() => { + if (stalled || armedSince === null) return; + const stalledMs = performance.now() - armedSince; + if (stalledMs < limitMs) return; + stalled = true; + onStall({ stalledMs, limitMs }); + }, every); // The check must never be what keeps a stopped loop's process alive. - timer?.unref?.(); + timer.unref?.(); return { - begin: () => { inFlightSince = now(); }, - end: () => { inFlightSince = null; }, - check, - stop: () => { clearIntervalImpl(timer); }, + // (Re)start the clock: the loop is doing its own work, or just made progress. + arm: () => { armedSince = performance.now(); }, + // Stop the clock: a model turn, or a wait the loop scheduled itself. + pause: () => { armedSince = null; }, + stop: () => { clearInterval(timer); }, }; }; diff --git a/cli/src/lib/poller.js b/cli/src/lib/poller.js index 8a5c0969b7..aad58930ba 100644 --- a/cli/src/lib/poller.js +++ b/cli/src/lib/poller.js @@ -9,7 +9,7 @@ import { createClient } from './api.js'; import { formatDigestFrame } from './digest-frame.js'; import { EMPTY_OUTPUT } from './claim-outcome.js'; -import { createPollLiveness, resolvePollLivenessMs } from './poll-liveness.js'; +import { createPollLiveness, formatStall, resolvePollLivenessMs } from './poll-liveness.js'; // Delivery failures that must stop a consumer rather than fall through to the // normal at-least-once retry path. A stale delivery belongs to a replacement @@ -58,11 +58,11 @@ export const startPoller = ({ limitMs: resolvePollLivenessMs(pollLivenessMs), checkEveryMs: pollLivenessCheckMs, onStall: ({ stalledMs }) => { + if (!running) return; pollStalled = true; running = false; pollLiveness.stop(); - const minutes = Math.max(1, Math.round(stalledMs / 60_000)); - onError?.(new Error(`poll stalled: no answer to the inbox poll for ${minutes} min — stopping poller`)); + onError?.(new Error(`poll stalled: no inbox poll settled for ${formatStall(stalledMs)} — stopping poller`)); onPollStalled?.({ stalledMs }); }, }); @@ -71,17 +71,15 @@ export const startPoller = ({ if (!running) return; try { - pollLiveness.begin(); - let page; - try { - page = await client.get('/api/agents/runtime/events', { - agentName, - instanceId, - limit: 10, - }); - } finally { - pollLiveness.end(); - } + // Same clock as `agent run`: running from here to the scheduled wait, + // paused while the local handler takes the event. + pollLiveness.arm(); + const page = await client.get('/api/agents/runtime/events', { + agentName, + instanceId, + limit: 10, + }); + pollLiveness.arm(); if (pollStalled) return; const { events = [] } = page; @@ -108,9 +106,12 @@ export const startPoller = ({ let result = { outcome: 'no_action' }; try { // A handler that returns nothing said nothing: not a read of the digest. + pollLiveness.pause(); result = (await onEvent(forwarded)) || { outcome: 'no_action', reason: EMPTY_OUTPUT }; } catch (handlerErr) { result = { outcome: 'error', reason: handlerErr.message }; + } finally { + pollLiveness.arm(); } // Acknowledge the event @@ -175,11 +176,14 @@ export const startPoller = ({ } consecutiveErrors++; onError?.(err); - // Back off on repeated errors (max 60s) + // Back off on repeated errors (max 60s). A wait this loop scheduled + // itself always ends, so the liveness clock stops for it. const backoff = Math.min(intervalMs * consecutiveErrors, 60_000); + pollLiveness.pause(); await new Promise((r) => setTimeout(r, backoff)); } + pollLiveness.pause(); if (running) setTimeout(poll, intervalMs); }; diff --git a/docs/cli/README.md b/docs/cli/README.md index a4a622cb4a..f6c35eb5bd 100644 --- a/docs/cli/README.md +++ b/docs/cli/README.md @@ -300,8 +300,8 @@ Written by `commonly agent run` during spawn cycles. Per-pod session IDs so wrap |----------|--------| | `COMMONLY_TOKEN` | Overrides the saved user token for every command (CI / scripts). | | `COMMONLY_BASE_URL` | Overrides the base URL (Python SDK `run()` honors this). | -| `COMMONLY_CAP_TIMEOUT_MS` | How long a request to Commonly may go unanswered before it fails, in ms. Default 30000 for reads; writes get twice that and uploads four times. A timed-out poll is retried with backoff like any other failed request. | -| `COMMONLY_POLL_LIVENESS_MS` | How long one inbox poll may stay in flight before `agent run` or `agent connect` exits non-zero, in ms. Default 600000 (10 minutes). A model turn and a retry wait don't count toward it. | +| `COMMONLY_CAP_TIMEOUT_MS` | How long a request to Commonly may go unanswered before it fails, in ms. Default 30000 for reads; writes get twice that and uploads four times. A timed-out poll is retried with backoff like any other failed request. Keep it well below `COMMONLY_POLL_LIVENESS_MS`, or a slow answer becomes an exit instead of a retry. | +| `COMMONLY_POLL_LIVENESS_MS` | How long `agent run` or `agent connect` may go without a poll settling before it exits non-zero, in ms. Default 600000 (10 minutes). Time inside a model turn and the loop's own retry waits doesn't count. | | `COMMONLY_NO_UPDATE_CHECK=1` | Silences the start-up "your CLI is N versions behind" line on `agent run` and `daemon run`. `daemon install` carries it into the service when set ([doctor.md](doctor.md)). | --- @@ -373,9 +373,9 @@ commonly agent detach <name> --force See [LOCAL_CLI_WRAPPER.md §Token revocation](../agents/LOCAL_CLI_WRAPPER.md#token-revocation). -### `commonly agent run` exits with "poll stalled: no answer to the inbox poll" +### `commonly agent run` exits with "poll stalled: no inbox poll settled" -One request to Commonly was accepted and never answered for 10 minutes (`COMMONLY_POLL_LIVENESS_MS`). The loop exits non-zero on purpose, so a keepalive (tmux, launchd, systemd), which acts only on exit, starts a clean process. Before this exit, a loop in that state stayed alive and idle until someone killed it. If you run `agent run` by hand, start it again. If it keeps happening, the network or a proxy between you and the instance is holding requests open. +For 10 minutes (`COMMONLY_POLL_LIVENESS_MS`) the loop polled, acked or held work without a poll ever settling, which means a request it was waiting on never ended. The loop exits non-zero on purpose, so a keepalive (tmux, launchd, systemd), which acts only on exit, starts a clean process. Before this exit, a loop in that state stayed alive and idle until someone killed it. If you run `agent run` by hand, start it again. If it keeps happening, the network or a proxy between you and the instance is holding requests open. ### `commonly agent run` fails with `spawn claude ENOENT` From 6d256cbf82f646f5b0ca7229b83b84b00eedeba8 Mon Sep 17 00:00:00 2001 From: Lily Shen <115414357+lilyshen0722@users.noreply.github.com> Date: Sat, 31 Oct 2026 16:42:33 -0700 Subject: [PATCH 4/8] test(cli): inbox timeout conversion from the fetch and the body read (TASK-338) Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_015AKgpQmb6BbxDrAtw45bgN --- cli/__tests__/inbox.test.mjs | 40 ++++++++++++++++++++++++++++++++++++ 1 file changed, 40 insertions(+) diff --git a/cli/__tests__/inbox.test.mjs b/cli/__tests__/inbox.test.mjs index 32955b51a6..4094703102 100644 --- a/cli/__tests__/inbox.test.mjs +++ b/cli/__tests__/inbox.test.mjs @@ -28,6 +28,7 @@ const { const { extractToken, filterItems, nextCursorFrom, parseCursor, formatCursor, formatItemLine, createScrubber, retryDelayMs, DEFAULT_BACKOFF_MS, MAX_BACKOFF_MS, + createInboxRequest, InboxRequestError, } = await import('../src/lib/inbox.js'); const tmpDir = fs.mkdtempSync(path.join(os.tmpdir(), 'cli-inbox-')); @@ -1089,3 +1090,42 @@ describe('item line and pure helpers', () => { expect(printed.join('')).toContain('There is no fallback to the saved login'); }); }); + +// TASK-338: the inbox's own fetch gets the same hard timeout as every other CAP +// request, and a timeout, from the fetch or from the body read, reads as an +// unanswered request rather than a bare DOMException. +describe('a request that is never answered', () => { + const timeoutError = () => new DOMException('The operation was aborted due to timeout', 'TimeoutError'); + const request = (fetchImpl) => createInboxRequest({ + baseUrl: 'https://example.test', token: 'cm_agent_x', tokenFile: '/tmp/t', fetchImpl, + }); + + test('sends a timeout signal with every request', async () => { + const fetchImpl = stubFetch({ '/api/activity/decision-queue': jsonRes({ items: [] }) }); + await request(fetchImpl)('/api/activity/decision-queue'); + expect(fetchImpl.calls[0].init.signal).toBeInstanceOf(AbortSignal); + }); + + test('a fetch that times out becomes an InboxRequestError with no status', async () => { + const err = await request(jest.fn(async () => { throw timeoutError(); }))('/api/activity/decision-queue') + .catch((e) => e); + expect(err).toBeInstanceOf(InboxRequestError); + expect(err.status).toBeNull(); + expect(err.message).toBe('GET /api/activity/decision-queue got no answer in 30s'); + }); + + test('so does a body read that times out', async () => { + const fetchImpl = jest.fn(async () => ({ ok: true, status: 200, text: async () => { throw timeoutError(); } })); + const err = await request(fetchImpl)('/api/activity/decision-queue', { method: 'POST', body: {} }) + .catch((e) => e); + expect(err).toBeInstanceOf(InboxRequestError); + expect(err.status).toBeNull(); + expect(err.message).toBe('POST /api/activity/decision-queue got no answer in 60s'); + }); + + test('any other fetch failure passes through unchanged', async () => { + const boom = new TypeError('fetch failed'); + const err = await request(jest.fn(async () => { throw boom; }))('/api/x').catch((e) => e); + expect(err).toBe(boom); + }); +}); From 9c6b970f4e3e2d0bc982439e38abd9c584f84e95 Mon Sep 17 00:00:00 2001 From: Lily Shen <115414357+lilyshen0722@users.noreply.github.com> Date: Sat, 31 Oct 2026 16:51:05 -0700 Subject: [PATCH 5/8] test(cli): isolate COMMONLY_CAP_TIMEOUT_MS in the inbox timeout tests (TASK-338) Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_015AKgpQmb6BbxDrAtw45bgN --- cli/__tests__/inbox.test.mjs | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/cli/__tests__/inbox.test.mjs b/cli/__tests__/inbox.test.mjs index 4094703102..e7e57a928b 100644 --- a/cli/__tests__/inbox.test.mjs +++ b/cli/__tests__/inbox.test.mjs @@ -1095,6 +1095,12 @@ describe('item line and pure helpers', () => { // request, and a timeout, from the fetch or from the body read, reads as an // unanswered request rather than a bare DOMException. describe('a request that is never answered', () => { + // The limits below are the defaults; an operator's own override must not leak in. + const savedTimeout = process.env.COMMONLY_CAP_TIMEOUT_MS; + beforeEach(() => { delete process.env.COMMONLY_CAP_TIMEOUT_MS; }); + afterAll(() => { + if (savedTimeout !== undefined) process.env.COMMONLY_CAP_TIMEOUT_MS = savedTimeout; + }); const timeoutError = () => new DOMException('The operation was aborted due to timeout', 'TimeoutError'); const request = (fetchImpl) => createInboxRequest({ baseUrl: 'https://example.test', token: 'cm_agent_x', tokenFile: '/tmp/t', fetchImpl, From 60aac4963dd779a34a4e2df0631ff241e94ad699 Mon Sep 17 00:00:00 2001 From: Lily Shen <115414357+lilyshen0722@users.noreply.github.com> Date: Sat, 31 Oct 2026 17:01:20 -0700 Subject: [PATCH 6/8] fix(cli): rename only the cap's own timeout, by identity; one limit format (TASK-338) Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_015AKgpQmb6BbxDrAtw45bgN --- cli/src/lib/api.js | 6 ++++-- cli/src/lib/inbox.js | 4 ++-- 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/cli/src/lib/api.js b/cli/src/lib/api.js index d46f7176fc..e06eed9e73 100644 --- a/cli/src/lib/api.js +++ b/cli/src/lib/api.js @@ -85,7 +85,7 @@ const capSignal = (capTimeout, callerSignal) => { return typeof AbortSignal.any === 'function' ? AbortSignal.any([callerSignal, capTimeout]) : callerSignal; }; -const formatLimit = (ms) => (ms < 1000 ? `${ms}ms` : `${Math.round(ms / 1000)}s`); +export const formatLimit = (ms) => (ms < 1000 ? `${ms}ms` : `${Math.round(ms / 1000)}s`); // The cap's timeout rejects with a DOMException named TimeoutError, from the // fetch or from the body read. Name the request and the limit instead, so the @@ -93,7 +93,9 @@ const formatLimit = (ms) => (ms < 1000 ? `${ms}ms` : `${Math.round(ms / 1000)}s` // Only the cap's own timeout is renamed: a caller's abort or its own shorter // timeout passes through as it was thrown. const nameTimeout = (capTimeout, method, path, ms) => (err) => { - if (err?.name !== 'TimeoutError' || !capTimeout.aborted) throw err; + // By identity: the caller's timeout can fire first and the cap's a moment + // later, and then `capTimeout.aborted` alone would claim the caller's error. + if (err?.name !== 'TimeoutError' || err !== capTimeout.reason) throw err; throw Object.assign( new Error(`${method} ${path} got no answer in ${formatLimit(ms)}`), { code: 'cap_timeout', timeoutMs: ms }, diff --git a/cli/src/lib/inbox.js b/cli/src/lib/inbox.js index 42a29560ca..2fd8b12ee1 100644 --- a/cli/src/lib/inbox.js +++ b/cli/src/lib/inbox.js @@ -62,7 +62,7 @@ import { mkdirSync, readFileSync, renameSync, writeFileSync } from 'fs'; import { dirname } from 'path'; -import { capTimeoutMs } from './api.js'; +import { capTimeoutMs, formatLimit } from './api.js'; export const INBOX_KINDS = ['mention', 'decision', 'handoff', 'approval']; export const TOKEN_FILE_ENV = 'COMMONLY_TOKEN_FILE'; @@ -416,7 +416,7 @@ export const createInboxRequest = ({ text = await res.text(); } catch (err) { if (err?.name !== 'TimeoutError') throw err; - throw new InboxRequestError(`${method} ${path} got no answer in ${Math.round(timeoutMs / 1000)}s`, null); + throw new InboxRequestError(`${method} ${path} got no answer in ${formatLimit(timeoutMs)}`, null); } let parsed; try { From cc35628bb188cc51a05bb9fc756659ea266c2a24 Mon Sep 17 00:00:00 2001 From: Lily Shen <115414357+lilyshen0722@users.noreply.github.com> Date: Sat, 31 Oct 2026 17:12:11 -0700 Subject: [PATCH 7/8] fix(cli): keep the CAP timeout beside a caller signal on Node without AbortSignal.any (TASK-338) Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_015AKgpQmb6BbxDrAtw45bgN --- cli/__tests__/cap-timeout-liveness.test.mjs | 23 +++++++++++++++++++++ cli/src/lib/api.js | 16 +++++++++++--- 2 files changed, 36 insertions(+), 3 deletions(-) diff --git a/cli/__tests__/cap-timeout-liveness.test.mjs b/cli/__tests__/cap-timeout-liveness.test.mjs index 7416643712..ab35770aab 100644 --- a/cli/__tests__/cap-timeout-liveness.test.mjs +++ b/cli/__tests__/cap-timeout-liveness.test.mjs @@ -198,6 +198,29 @@ describe('the timeout covers the whole response, and only renames its own abort' }); }); +describe('without AbortSignal.any (Node 20.0 to 20.2)', () => { + const realAny = AbortSignal.any; + afterEach(() => { AbortSignal.any = realAny; }); + + test("a caller's longer signal does not replace the hard timeout", async () => { + stub = await startStub(() => 'hang'); + AbortSignal.any = undefined; + const client = createClient({ instance: stub.url, token: 'cm_agent_test', timeoutMs: 150 }); + await expect(client.get('/api/agents/runtime/events', {}, { signal: AbortSignal.timeout(5000) })) + .rejects.toMatchObject({ code: 'cap_timeout', timeoutMs: 150 }); + }); + + test("a caller's shorter timeout still keeps its own error", async () => { + stub = await startStub(() => 'hang'); + AbortSignal.any = undefined; + const client = createClient({ instance: stub.url, token: 'cm_agent_test', timeoutMs: 5000 }); + const err = await client.get('/api/agents/runtime/events', {}, { signal: AbortSignal.timeout(50) }) + .catch((e) => e); + expect(err.name).toBe('TimeoutError'); + expect(err.code).not.toBe('cap_timeout'); + }); +}); + describe('agent run: the inbox loop survives a hung poll', () => { test('504, then a hang, then healthy: the loop backs off and recovers, with no stall exit', async () => { process.env.COMMONLY_CAP_TIMEOUT_MS = '150'; diff --git a/cli/src/lib/api.js b/cli/src/lib/api.js index e06eed9e73..f09283da16 100644 --- a/cli/src/lib/api.js +++ b/cli/src/lib/api.js @@ -78,11 +78,21 @@ export const capTimeoutMs = (kind, { baseMs, env = process.env } = {}) => { // A caller's own signal (doctor, and the 5s identity and skills reads in the // run loop) still applies; the hard timeout is added beside it, never instead. -// AbortSignal.any is Node 20.3+; before that the caller's signal stands alone, -// and every caller that passes one passes a shorter timeout. +// AbortSignal.any is Node 20.3+, and `engines` allows 20.0, so the fallback +// joins the two by hand. Either way the abort carries the source's own reason, +// which is what nameTimeout matches on. const capSignal = (capTimeout, callerSignal) => { if (!callerSignal) return capTimeout; - return typeof AbortSignal.any === 'function' ? AbortSignal.any([callerSignal, capTimeout]) : callerSignal; + if (typeof AbortSignal.any === 'function') return AbortSignal.any([callerSignal, capTimeout]); + const joined = new AbortController(); + for (const source of [callerSignal, capTimeout]) { + if (source.aborted) { + joined.abort(source.reason); + break; + } + source.addEventListener('abort', () => joined.abort(source.reason), { once: true }); + } + return joined.signal; }; export const formatLimit = (ms) => (ms < 1000 ? `${ms}ms` : `${Math.round(ms / 1000)}s`); From a7ecf1c65e008beedcd6b1347be1755fb7f1c4f9 Mon Sep 17 00:00:00 2001 From: Lily Shen <115414357+lilyshen0722@users.noreply.github.com> Date: Sat, 31 Oct 2026 18:04:30 -0700 Subject: [PATCH 8/8] fix(cli): release the AbortSignal.any fallback's listeners when a request settles (TASK-338) Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_015AKgpQmb6BbxDrAtw45bgN --- cli/__tests__/cap-timeout-liveness.test.mjs | 14 ++++++++++++ cli/src/lib/api.js | 24 +++++++++++++++------ 2 files changed, 32 insertions(+), 6 deletions(-) diff --git a/cli/__tests__/cap-timeout-liveness.test.mjs b/cli/__tests__/cap-timeout-liveness.test.mjs index ab35770aab..71a07d96b6 100644 --- a/cli/__tests__/cap-timeout-liveness.test.mjs +++ b/cli/__tests__/cap-timeout-liveness.test.mjs @@ -210,6 +210,20 @@ describe('without AbortSignal.any (Node 20.0 to 20.2)', () => { .rejects.toMatchObject({ code: 'cap_timeout', timeoutMs: 150 }); }); + test("a settled request takes its listener off a caller's long-lived signal", async () => { + stub = await startStub(() => EMPTY); + AbortSignal.any = undefined; + const caller = new AbortController(); + const added = jest.spyOn(caller.signal, 'addEventListener'); + const removed = jest.spyOn(caller.signal, 'removeEventListener'); + const client = createClient({ instance: stub.url, token: 'cm_agent_test', timeoutMs: 5000 }); + await client.get('/api/agents/runtime/events', {}, { signal: caller.signal }); + await client.get('/api/agents/runtime/events', {}, { signal: caller.signal }); + const ours = (spy) => spy.mock.calls.filter(([type]) => type === 'abort').map(([, fn]) => fn); + expect(ours(removed)).toEqual(ours(added)); + expect(ours(added)).toHaveLength(2); + }); + test("a caller's shorter timeout still keeps its own error", async () => { stub = await startStub(() => 'hang'); AbortSignal.any = undefined; diff --git a/cli/src/lib/api.js b/cli/src/lib/api.js index f09283da16..4f4e595207 100644 --- a/cli/src/lib/api.js +++ b/cli/src/lib/api.js @@ -81,18 +81,28 @@ export const capTimeoutMs = (kind, { baseMs, env = process.env } = {}) => { // AbortSignal.any is Node 20.3+, and `engines` allows 20.0, so the fallback // joins the two by hand. Either way the abort carries the source's own reason, // which is what nameTimeout matches on. +// The fallback's listeners are removed when the request settles, so a caller +// that reuses one long-lived signal does not collect one per request. const capSignal = (capTimeout, callerSignal) => { - if (!callerSignal) return capTimeout; - if (typeof AbortSignal.any === 'function') return AbortSignal.any([callerSignal, capTimeout]); + if (!callerSignal) return { signal: capTimeout, release: () => {} }; + if (typeof AbortSignal.any === 'function') { + return { signal: AbortSignal.any([callerSignal, capTimeout]), release: () => {} }; + } const joined = new AbortController(); + const listeners = []; for (const source of [callerSignal, capTimeout]) { if (source.aborted) { joined.abort(source.reason); break; } - source.addEventListener('abort', () => joined.abort(source.reason), { once: true }); + const onAbort = () => joined.abort(source.reason); + source.addEventListener('abort', onAbort, { once: true }); + listeners.push([source, onAbort]); } - return joined.signal; + return { + signal: joined.signal, + release: () => listeners.forEach(([source, onAbort]) => source.removeEventListener('abort', onAbort)), + }; }; export const formatLimit = (ms) => (ms < 1000 ? `${ms}ms` : `${Math.round(ms / 1000)}s`); @@ -114,9 +124,11 @@ const nameTimeout = (capTimeout, method, path, ms) => (err) => { const timedFetch = (url, init, { method, path, ms, session }) => { const capTimeout = AbortSignal.timeout(ms); - return fetch(url, { ...init, signal: capSignal(capTimeout, init.signal) }) + const { signal, release } = capSignal(capTimeout, init.signal); + return fetch(url, { ...init, signal }) .then((res) => handleResponse(res, session)) - .catch(nameTimeout(capTimeout, method, path, ms)); + .catch(nameTimeout(capTimeout, method, path, ms)) + .finally(release); }; export const createClient = ({ instance = null, token = undefined, timeoutMs = undefined } = {}) => {