From abe2bed49f3821b756638aaa81b8d867b9153e76 Mon Sep 17 00:00:00 2001 From: Jeremy Liang Date: Sat, 17 Oct 2026 00:27:46 +0800 Subject: [PATCH 1/5] fix: gate watchdog recovery on bounded runtime observation --- src/lib/transport/kernel-watchdog.mjs | 65 ++++- .../unit/kernel-watchdog-observation.test.mjs | 230 ++++++++++++++++++ test/unit/kernel-watchdog.test.mjs | 10 + 3 files changed, 304 insertions(+), 1 deletion(-) create mode 100644 test/unit/kernel-watchdog-observation.test.mjs diff --git a/src/lib/transport/kernel-watchdog.mjs b/src/lib/transport/kernel-watchdog.mjs index b171907ec..5658c7da2 100644 --- a/src/lib/transport/kernel-watchdog.mjs +++ b/src/lib/transport/kernel-watchdog.mjs @@ -19,8 +19,48 @@ import { readExistingKernelConfig, dataplaneUsesCcNode, WRAP_SLOT_MAX, + slotContainerName, } from './rust-kernel-supervisor.mjs' import { resolveInferenceEngine } from '../vm/slot-engine.mjs' +import { slotHost } from '../vm/slot-host.mjs' +import { dockerRequest } from '../cluster/docker-remote.mjs' + +/** A failed observation is not evidence of a dead container. Use the slot's own host, including KVM runners. */ +export async function observeWatchdogContainer( + vm, + { hostFor = slotHost, request = dockerRequest, timeoutMs = 1500 } = {}, +) { + let timer + try { + const name = slotContainerName({ vm }) + if (!name) return { state: 'unknown' } + const observation = Promise.resolve().then(async () => { + const res = await request(hostFor(vm).dockerApi(), { + path: `/containers/${encodeURIComponent(name)}/json`, + timeoutMs, + }) + if (res.status === 404) return { state: 'missing' } + if (res.status !== 200) return { state: 'unknown' } + const state = JSON.parse(res.body.toString('utf8'))?.State + if (state?.Paused === true) return { state: 'paused' } + if (state?.Restarting === true) return { state: 'restarting' } + if (state?.Running === true) return { state: 'running' } + if (state?.Running === false && ['created', 'exited', 'dead'].includes(state.Status)) return { state: 'stopped' } + return { state: 'unknown' } + }) + // Covers connector/SSH setup too, before dockerRequest's own HTTP timeout starts. + return await Promise.race([ + observation, + new Promise((resolve) => { + timer = setTimeout(() => resolve({ state: 'unknown' }), timeoutMs) + }), + ]) + } catch { + return { state: 'unknown' } + } finally { + clearTimeout(timer) + } +} export const DEFAULT_KERNEL_WATCHDOG = Object.freeze({ enabled: true, @@ -82,6 +122,7 @@ export function createKernelWatchdog({ homeDirFor, ensure = ensureRustKernel, health = rustKernelHealth, + observeRuntime = observeWatchdogContainer, onFault = null, ensureTelemetry = null, telemetryEveryMs = TELEMETRY_HEAL_MS, @@ -171,16 +212,33 @@ export function createKernelWatchdog({ // Plain recovery keeps past attempts so a crash loop stays bounded by the window. if (kernelFaults.delete(vm.id)) restarts.delete(vm.id) // Config convergence, not a failure: outside the restart budget. - if (kernelSlotMismatch(exec)) await ensure(exec, { timeoutMs: config.timeout_ms }) + if (kernelSlotMismatch(exec)) { + const observed = await observeRuntime(vm) + if (!held.has(vm.id) && observed?.state === 'running') await ensure(exec, { timeoutMs: config.timeout_ms }) + } return } const reason = current?.unhealthy_reason || current?.error || 'kernel process down' + const observed = await observeRuntime(vm) + if (held.has(vm.id)) return + // Permission/transport failures and deliberately paused containers never authorize recovery. + if (!['running', 'missing', 'stopped', 'restarting'].includes(observed?.state)) return + if (observed.state !== 'running') { + if (vm.status !== 'running' || typeof revive !== 'function') return + if (!admitRestart(vm, `container ${observed.state}`, now())) return + const revived = await revive(vm) + if (!revived?.ok) console.warn(`[kernel-watchdog] ${vm.id} boot failed: ${revived?.error || revived?.code}`) + return + } if (!admitRestart(vm, reason, now())) return console.warn(`[kernel-watchdog] ${vm.id} restarting container: ${reason}`) const result = await ensure(exec, { timeoutMs: config.timeout_ms }) if (held.has(vm.id)) return // The container itself is gone or exited: a kernel restart cannot help, a full boot can. if (result?.reason === 'container_unavailable' && typeof revive === 'function' && vm.status === 'running') { + // The container may disappear during ensure; do not mistake a daemon/permission error for absence. + const latest = await observeRuntime(vm) + if (held.has(vm.id) || !['missing', 'stopped', 'restarting'].includes(latest?.state)) return console.warn(`[kernel-watchdog] ${vm.id} container unavailable; booting slot`) const revived = await revive(vm).catch((error) => ({ ok: false, error: error?.message || String(error) })) if (!revived?.ok) console.warn(`[kernel-watchdog] ${vm.id} boot failed: ${revived?.error || revived?.code}`) @@ -208,10 +266,15 @@ export function createKernelWatchdog({ inflight.set(vm.id, work) try { await work + } catch { + // One failed probe/recovery must not starve later slots or reject an interval's promise. + console.warn(`[kernel-watchdog] ${vm.id} check failed; skipping this tick`) } finally { if (inflight.get(vm.id) === work) inflight.delete(vm.id) } } + } catch { + console.warn('[kernel-watchdog] tick failed; retrying at the next interval') } finally { inTick = false } diff --git a/test/unit/kernel-watchdog-observation.test.mjs b/test/unit/kernel-watchdog-observation.test.mjs new file mode 100644 index 000000000..e0f61f41b --- /dev/null +++ b/test/unit/kernel-watchdog-observation.test.mjs @@ -0,0 +1,230 @@ +import assert from 'node:assert/strict' +import test from 'node:test' +import * as watchdog from '../../src/lib/transport/kernel-watchdog.mjs' +import { kernelFaults } from '../../src/lib/transport/rust-kernel-client.mjs' + +test.afterEach(() => { + for (const id of ['vm-observation', 'vm-bad', 'vm-good']) kernelFaults.delete(id) +}) + +function fixture({ state = 'running', runtime, node_id, status = 'running', observe, health, ensure, revive } = {}) { + let at = 0 + const actions = [] + const vm = { id: 'vm-observation', status, runtime, node_id } + const wd = watchdog.createKernelWatchdog({ + listTargets: () => [vm], + now: () => at, + config: { restart_backoff_sec: [10] }, + health: health || (async () => null), + observeRuntime: observe || (async () => ({ state })), + ensure: + ensure || + (async () => { + actions.push('ensure') + return { ok: true } + }), + revive: + revive || + (async () => { + actions.push('revive') + return { ok: true } + }), + }) + return { + wd, + vm, + actions, + advance: (ms) => { + at = ms + }, + } +} + +for (const state of ['unknown', 'paused']) { + test(`${state} runtime does not spend recovery budget or cause a restart`, async () => { + const f = fixture({ state }) + for (const ms of [0, 10_000, 20_000]) { + f.advance(ms) + await f.wd.tick() + } + assert.deepEqual(f.actions, []) + assert.equal(f.wd.restartsInWindow(f.vm.id), 0) + assert.equal(kernelFaults.has(f.vm.id), false) + }) +} +for (const state of ['missing', 'stopped', 'restarting']) { + for (const placement of [ + { runtime: { type: 'docker' } }, + { runtime: { type: 'kvm' } }, + { node_id: 'node-fixture' }, + ]) { + test(`${state} ${JSON.stringify(placement)} revives only after backoff, without a blind kernel restart`, async () => { + const f = fixture({ state, ...placement }) + try { + await f.wd.tick() + assert.deepEqual(f.actions, []) + f.advance(10_000) + await f.wd.tick() + assert.deepEqual(f.actions, ['revive']) + assert.equal(f.wd.restartsInWindow(f.vm.id), 1) + f.advance(30_000) + await f.wd.tick() + assert.deepEqual(f.actions, ['revive']) + assert.ok(kernelFaults.has(f.vm.id)) + } finally { + kernelFaults.delete(f.vm.id) + } + }) + } +} + +test('unknown observations recover without consuming retries; running-but-dead kernel is still restarted', async () => { + let state = 'unknown' + const f = fixture({ observe: async () => ({ state }) }) + await f.wd.tick() + f.advance(30_000) + await f.wd.tick() + state = 'running' + await f.wd.tick() + assert.deepEqual(f.actions, []) + f.advance(40_000) + await f.wd.tick() + assert.deepEqual(f.actions, ['ensure']) +}) + +test('stopped intent is not probed or revived and planned intent is not booted', async () => { + for (const status of ['stopped', 'disabled', 'planned']) { + const f = fixture({ state: 'missing', status }) + await f.wd.tick() + f.advance(10_000) + await f.wd.tick() + assert.deepEqual(f.actions, []) + assert.equal(f.wd.restartsInWindow(f.vm.id), 0) + } +}) + +test('a permission-like container_unavailable result cannot trigger boot without confirmed absence', async () => { + const f = fixture({ + ensure: async () => ({ ok: false, reason: 'container_unavailable', error: 'daemon not running' }), + }) + await f.wd.tick() + f.advance(10_000) + await f.wd.tick() + assert.deepEqual(f.actions, []) +}) + +test('hold during an observation prevents restart and releases the in-flight tick', async () => { + let finish + const f = fixture({ + observe: () => + new Promise((resolve) => { + finish = resolve + }), + }) + const tick = f.wd.tick() + await new Promise((r) => setImmediate(r)) + const held = f.wd.hold(f.vm.id) + finish({ state: 'missing' }) + await Promise.all([tick, held]) + f.advance(10_000) + await f.wd.tick() + assert.deepEqual(f.actions, []) +}) + +test('one rejected health probe or ensure call does not starve the next slot or reject tick', async () => { + for (const phase of ['health', 'ensure', 'observe']) { + let at = 0 + const ensured = [] + const wd = watchdog.createKernelWatchdog({ + config: { restart_backoff_sec: [10] }, + listTargets: () => [{ id: 'vm-bad' }, { id: 'vm-good' }], + now: () => at, + health: async (exec) => { + if (phase === 'health' && exec.vmId === 'vm-bad') throw new Error('fixture health') + return null + }, + observeRuntime: async (vm) => { + if (phase === 'observe' && vm.id === 'vm-bad') throw new Error('fixture permission') + return { state: 'running' } + }, + ensure: async (exec) => { + if (phase === 'ensure' && exec.vmId === 'vm-bad') throw new Error('fixture ensure') + ensured.push(exec.vmId) + return { ok: true } + }, + }) + await wd.tick() + at = 10_000 + await wd.tick() + assert.ok(ensured.includes('vm-good')) + } +}) + +test('overlapping ticks do not duplicate observations or recovery', async () => { + let finish, + calls = 0 + const f = fixture({ + observe: () => { + calls++ + return new Promise((r) => { + finish = r + }) + }, + }) + const first = f.wd.tick() + await new Promise((r) => setImmediate(r)) + await f.wd.tick() + assert.equal(calls, 1) + finish({ state: 'running' }) + await first +}) + +test('Docker observation distinguishes 404, HTTP failures, malformed payloads and state flags on the selected host', async () => { + const cases = [ + [404, {}, 'missing'], + [403, {}, 'unknown'], + [500, {}, 'unknown'], + [200, {}, 'unknown'], + [200, { State: { Running: true } }, 'running'], + [200, { State: { Running: true, Paused: true } }, 'paused'], + [200, { State: { Running: true, Restarting: true } }, 'restarting'], + [200, { State: { Running: false, Status: 'exited' } }, 'stopped'], + [200, { State: { Running: false } }, 'unknown'], + ] + for (const [status, payload, state] of cases) { + const vm = { id: 'vm-fixture', node_id: 'node-fixture', runtime: { container: 'custom/name' } } + const connector = () => { + throw new Error('fixture must not connect') + } + const result = await watchdog.observeWatchdogContainer(vm, { + hostFor: (target) => { + assert.equal(target, vm) + return { dockerApi: () => connector } + }, + request: async (connect, options) => { + assert.equal(connect, connector) + assert.equal(options.path, '/containers/custom%2Fname/json') + return { status, body: Buffer.from(JSON.stringify(payload)) } + }, + }) + assert.equal(result.state, state) + } + const options = { hostFor: () => ({ dockerApi: () => () => {} }) } + for (const request of [ + async () => { + throw new Error('EACCES') + }, + async () => ({ status: 200, body: Buffer.from('broken json') }), + ]) { + assert.equal((await watchdog.observeWatchdogContainer({ id: 'vm-01' }, { ...options, request })).state, 'unknown') + } + assert.equal( + ( + await watchdog.observeWatchdogContainer( + { id: 'vm-01' }, + { ...options, request: () => new Promise(() => {}), timeoutMs: 10 }, + ) + ).state, + 'unknown', + ) +}) diff --git a/test/unit/kernel-watchdog.test.mjs b/test/unit/kernel-watchdog.test.mjs index e67897592..66739f500 100644 --- a/test/unit/kernel-watchdog.test.mjs +++ b/test/unit/kernel-watchdog.test.mjs @@ -24,6 +24,7 @@ test('watchdog restarts only a dead kernel or one that gave up on its CLI', asyn const ensured = [] let now = 0 const wd = createKernelWatchdog({ + observeRuntime: async () => ({ state: 'running' }), now: () => now, listTargets: () => [ { id: 'vm-up', inference_engine: 'rust' }, @@ -63,6 +64,7 @@ test('container restarts are bounded, spaced out, then the VM is faulted until h const ensured = [] const faults = [] const wd = createKernelWatchdog({ + observeRuntime: async () => ({ state: 'running' }), config: { restart_backoff_sec: [60, 300], restart_window_sec: 3600 }, listTargets: () => [{ id: 'vm-loop', inference_engine: 'rust' }], homeDirFor: () => '/tmp/vm-loop', @@ -120,6 +122,7 @@ test('watchdog does not restart a live crag slot', async () => { fs.writeFileSync(path.join(run, 'kernel.json'), JSON.stringify({ dataplane: 'crag' })) const ensured = [] const wd = createKernelWatchdog({ + observeRuntime: async () => ({ state: 'running' }), listTargets: () => [{ id: 'vm-crag', inference_engine: 'rust' }], homeDirFor: () => home, health: async () => ({ ok: true, engine: 'rust', ready_slots: 0, cli_pid: 0 }), @@ -137,6 +140,7 @@ test('telemetry sidecar is checked once per window on every running slot', async let now = 0 const checked = [] const wd = createKernelWatchdog({ + observeRuntime: async () => ({ state: 'running' }), now: () => now, telemetryEveryMs: 600_000, listTargets: () => [ @@ -168,6 +172,7 @@ test('inherited rust slot reaches the restart ladder after CLI budget is spent', const faults = [] let now = 0 const wd = createKernelWatchdog({ + observeRuntime: async () => ({ state: 'running' }), now: () => now, getRouting: () => ({ inference: { engine: 'rust' } }), listTargets: () => [{ id: 'vm-inherit' }], @@ -211,6 +216,7 @@ test('watchdog rereads routing each tick; Codex stays out of the restart path', let routingCalls = 0 let routing = { inference: { engine: 'rust' } } const wd = createKernelWatchdog({ + observeRuntime: async () => ({ state: 'running' }), getRouting: () => { routingCalls += 1 return routing @@ -237,6 +243,7 @@ test('a missing container is booted, not just kernel-restarted, inside the same let now = 0 const revived = [] const wd = createKernelWatchdog({ + observeRuntime: async () => ({ state: 'missing' }), listTargets: () => [ { id: 'vm-gone', status: 'running', inference_engine: 'rust' }, { id: 'vm-planned', status: 'planned', inference_engine: 'rust' }, @@ -262,6 +269,7 @@ test('a held slot is left alone; reset gives a faulted slot a fresh budget', asy const probed = [] const bridged = [] const wd = createKernelWatchdog({ + observeRuntime: async () => ({ state: 'running' }), config: { restart_backoff_sec: [60] }, listTargets: () => [ { id: 'vm-held', inference_engine: 'rust' }, @@ -309,6 +317,7 @@ test('a hold taken while the health probe is pending stops the restart and the b const probe = deferred() const actions = [] const wd = createKernelWatchdog({ + observeRuntime: async () => ({ state: 'running' }), config: { restart_backoff_sec: [10] }, listTargets: () => [{ id: 'vm-race', status: 'running', inference_engine: 'rust' }], health: () => probe.promise, @@ -336,6 +345,7 @@ test('hold waits for a restart already in flight, and the boot after it is skipp const actions = [] let at = 0 const wd = createKernelWatchdog({ + observeRuntime: async () => ({ state: 'running' }), config: { restart_backoff_sec: [10] }, listTargets: () => [{ id: 'vm-busy', status: 'running', inference_engine: 'rust' }], health: async () => null, From f42e0aea08660e618c35c5673558107a1b1ed869 Mon Sep 17 00:00:00 2001 From: Jeremy Liang Date: Sat, 17 Oct 2026 06:45:18 +0800 Subject: [PATCH 2/5] fix: recreate remote restart loops during watchdog revival --- src/lib/cluster/remote-slot.mjs | 3 + test/unit/remote-slot-restart-loop.test.mjs | 147 ++++++++++++++++++++ 2 files changed, 150 insertions(+) create mode 100644 test/unit/remote-slot-restart-loop.test.mjs diff --git a/src/lib/cluster/remote-slot.mjs b/src/lib/cluster/remote-slot.mjs index da6972f7f..057ce42ae 100644 --- a/src/lib/cluster/remote-slot.mjs +++ b/src/lib/cluster/remote-slot.mjs @@ -667,6 +667,9 @@ export async function startRemoteSlot(vm, projectRoot, { recreate = false, routi existing.Config?.Labels?.['kin.vm.layout'] === SLOT_LAYOUT && runtimeLabelOf(existing) === runtimeKind(vm) && codexRoleMatches(existing, vm) && + // Running stays true during Docker restart-policy backoff. Recreate the + // loop just like the local runtime; otherwise watchdog revival is a no-op. + !existing.State?.Restarting && !recreate const common = { nodeId, diff --git a/test/unit/remote-slot-restart-loop.test.mjs b/test/unit/remote-slot-restart-loop.test.mjs new file mode 100644 index 000000000..1a4a65da7 --- /dev/null +++ b/test/unit/remote-slot-restart-loop.test.mjs @@ -0,0 +1,147 @@ +import assert from 'node:assert/strict' +import fs from 'node:fs' +import vmContext from 'node:vm' +import test from 'node:test' +import { createKernelWatchdog } from '../../src/lib/transport/kernel-watchdog.mjs' +import { kernelFaults } from '../../src/lib/transport/rust-kernel-client.mjs' + +// Execute the exact production lifecycle body with synthetic I/O boundaries. +// No SSH, Docker daemon, credentials or production filesystem is accessed. +const source = fs.readFileSync(new URL('../../src/lib/cluster/remote-slot.mjs', import.meta.url), 'utf8') +const start = source.slice( + source.indexOf('export async function startRemoteSlot('), + source.indexOf('/** Remote twin of reloadSlotWorker'), +) + +function fixture({ + running = true, + restarting = false, + missing = false, + mismatch = false, + kind = 'docker', + removeFails = false, +} = {}) { + const calls = [] + const target = { + id: 'vm-restart-fixture', + node_id: 'node-fixture', + status: 'running', + runtime: { type: kind, accel: 'kvm' }, + } + let info = missing + ? null + : { + Config: { Image: mismatch ? 'old-image' : 'image', Labels: { 'kin.vm.layout': '2' } }, + HostConfig: { NetworkMode: 'network' }, + State: { Running: running, Restarting: restarting }, + } + const io = { + prepare: async () => ({ + nodeId: target.node_id, + session: { docker: {}, client: {} }, + image: 'image', + user: 'fixture', + remoteDir: '/fixture', + slotDir: '/fixture', + }), + isKvmRuntime: () => kind === 'kvm', + ensureRemoteEgress: async () => ({ ok: true, network: 'network' }), + containerName: () => 'kin-fixture', + clusterManager: () => ({ ensureSlotRelays: async () => ({}) }), + inspectContainerOrNull: async () => info, + runtimeLabelOf: () => kind, + runtimeKind: () => kind, + codexRoleMatches: () => true, + SLOT_LAYOUT: '2', + applyRuntime: () => calls.push('runtime'), + removeContainer: async () => { + calls.push('remove') + if (removeFails) throw new Error('remove denied') + info = null + }, + writeWorkerFiles: () => calls.push('files'), + ensureGuestMachineIdFile: () => {}, + pushSlotFiles: async () => {}, + migrateClaudeDir: async () => {}, + runRemote: async () => {}, + shellQuote: (s) => s, + reconcileSlotCredentials: async () => {}, + releaseRemoteEgress: async () => {}, + createContainerRaw: async () => { + calls.push('create') + info = { State: { Running: false, Restarting: false } } + }, + slotContainerBody: () => ({}), + containerAction: async (_docker, _name, action) => { + calls.push(action) + if (info.State.Restarting) throw new Error('restart loop was not replaced') + info.State.Running = true + }, + chownRemoteKvmBinds: async () => {}, + startRemoteTelemetry: async () => {}, + failure: (err) => ({ ok: false, error: err.message }), + } + const boot = vmContext.runInNewContext(`${start.replace('export ', '')}; startRemoteSlot`, io) + return { calls, target, boot: () => boot(target, '/fixture'), state: () => info?.State } +} + +for (const kind of ['docker', 'kvm']) { + for (const running of [true, false]) { + test(`${kind} restart-loop Running=${running} is removed, recreated and started`, async () => { + const f = fixture({ kind, running, restarting: true }) + assert.equal((await f.boot()).action, 'created') + assert.deepEqual(f.calls, ['remove', 'files', 'create', 'start', 'runtime']) + assert.equal(f.state().Restarting, false) + assert.equal(f.state().Running, true) + }) + } +} + +for (const [options, action, calls] of [ + [{}, 'already-running', ['runtime']], + [{ running: false }, 'started', ['files', 'start', 'runtime']], + [{ missing: true }, 'created', ['files', 'create', 'start', 'runtime']], + [{ mismatch: true }, 'created', ['remove', 'files', 'create', 'start', 'runtime']], +]) { + test(`remote lifecycle preserves ${action} behavior: ${JSON.stringify(options)}`, async () => { + const f = fixture(options) + assert.equal((await f.boot()).action, action) + assert.deepEqual(f.calls, calls) + }) +} + +test('failed restart-loop removal does not create or claim successful recovery', async () => { + const f = fixture({ restarting: true, removeFails: true }) + assert.equal((await f.boot()).ok, false) + assert.deepEqual(f.calls, ['remove']) +}) + +test('watchdog revival reaches the remote lifecycle and clears a restart loop without spending another attempt', async () => { + const f = fixture({ restarting: true }) + let now = 0 + const wd = createKernelWatchdog({ + listTargets: () => [f.target], + now: () => now, + config: { restart_backoff_sec: [10] }, + health: async () => (f.state()?.Restarting ? null : { ok: true, healthy: true }), + observeRuntime: async () => ({ state: f.state()?.Restarting ? 'restarting' : 'running' }), + revive: f.boot, + ensure: async () => { + throw new Error('must revive the remote container, not blindly restart the kernel') + }, + }) + try { + await wd.tick() + assert.deepEqual(f.calls, []) + now = 10_000 + await wd.tick() + assert.equal(f.state().Restarting, false) + assert.equal(wd.restartsInWindow(f.target.id), 1) + now = 30_000 + await wd.tick() + assert.equal(kernelFaults.has(f.target.id), false) + assert.equal(f.calls.filter((x) => x === 'create').length, 1) + } finally { + kernelFaults.delete(f.target.id) + } +}) From 5b3e39cca7f04f8332745b668037f99e75c49e77 Mon Sep 17 00:00:00 2001 From: Cursor Agent Date: Sat, 17 Oct 2026 00:51:40 +0000 Subject: [PATCH 3/5] docs: add CHANGELOG entry for watchdog observation (#491) Co-authored-by: ted --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 20254c9d8..58f53f7dd 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,7 @@ ## 未发布 +- 内核看门狗在容器级恢复/重建前先对槽位所在 Docker 主机做有界观测（运行/缺失/已停/Restarting/暂停/unknown）；HTTP/权限/传输失败或观测超时视为 unknown，不消耗重启预算、不盲重启；确认 missing/stopped/restarting 且槽位仍为运行意图时才 revival/recreate，运行中容器仍走既有 kernel 重启；远程 Restarting 崩溃循环纳入看门狗重建；单槽异常与 tick 级 promise 拒绝互不影响。（#491） - Docker 槽创建可选 `machine.docker_cpus`（`0` 或省略不限；`0.01`–`16` 为 CPU 时间上限）：写入槽位记录并在容器创建/重建时生效（本地 `--cpus`、集群 `NanoCpus`），KVM 与非法值拒绝；不提供在线 PATCH 或把 `vcpus` 隐式映射为 Docker 上限；损坏落盘值在 reload/恢复/重建前拒绝。（#490） - 修复通知监控在冷却/无事件轮询时误清 `last_error`：仅在实际投递成功时清除，保留上次投递失败；解析 `dispatchNotify` 的 `ok: false` 与部分通道失败，同批次任一失败优先于成功事件。（#483） - 代理出口地理查询经 SOCKS5 时由代理端解析目标域名（查询路径内将 `socks5://` 规范为 `socks5h:`），避免依赖控制面本地 DNS；IPv4 与 IPv6 两阶段探测一致，不修改已保存代理 URI，HTTP(S) 代理与本地直连行为不变。（#482） From 0cd2a498c69dc9a95cc78cc60707a3f9dc5f1813 Mon Sep 17 00:00:00 2001 From: Cursor Agent Date: Sat, 17 Oct 2026 01:07:59 +0000 Subject: [PATCH 4/5] ci: retrigger after flaky panel-users test Co-authored-by: ted From 9fe10e52aa4db2a74404de98ca06ea56757d560b Mon Sep 17 00:00:00 2001 From: Cursor Agent Date: Sat, 17 Oct 2026 01:29:17 +0000 Subject: [PATCH 5/5] test: stub slotDockerCpus in remote restart-loop vm fixture (#491) Co-authored-by: ted --- test/unit/remote-slot-restart-loop.test.mjs | 1 + 1 file changed, 1 insertion(+) diff --git a/test/unit/remote-slot-restart-loop.test.mjs b/test/unit/remote-slot-restart-loop.test.mjs index 1a4a65da7..0f14e0530 100644 --- a/test/unit/remote-slot-restart-loop.test.mjs +++ b/test/unit/remote-slot-restart-loop.test.mjs @@ -49,6 +49,7 @@ function fixture({ containerName: () => 'kin-fixture', clusterManager: () => ({ ensureSlotRelays: async () => ({}) }), inspectContainerOrNull: async () => info, + slotDockerCpus: () => 0, runtimeLabelOf: () => kind, runtimeKind: () => kind, codexRoleMatches: () => true,