From 628984ca625a4d307b859d9f32d49e52e2d21ee6 Mon Sep 17 00:00:00 2001 From: Bill Date: Sat, 17 Oct 2026 23:33:19 +0800 Subject: [PATCH 1/2] fix: harden Windows process ownership and cleanup Bind Windows tree termination to verified native handles and preserve incomplete cleanup evidence across recovery, retries, and worker commits. Harden filesystem lock transitions, late identity writes, and repository root guards. Add lifecycle regressions, ADR-009, and synchronized bundles. Validation: Windows Node 22/24 and isolated Linux Node 24 matrices, identity and recovery counterexamples, and owned process-tree tests. The staged sources match the independently accepted source manifest. --- docs/ARCHITECTURE.md | 8 +- docs/PROJECT_STATE.md | 24 +- docs/README.md | 6 +- ...windows-identity-bound-tree-termination.md | 82 + docs/decisions/README.md | 1 + .../2026-10-17-filesystem-root-guards.md | 121 ++ plugins/codex-zcode-bridge/dist/bridge.mjs | 1399 ++++++++++-- .../codex-zcode-bridge/worker/worker-main.mjs | 1931 ++++++++++++----- scripts/lib/repo-root-pattern.mjs | 73 +- src/adapters/process-spawn.ts | 212 +- src/adapters/windows-process-cleanup.ts | 1201 +++++++++- src/adapters/zcode-adapter.ts | 56 +- src/adapters/zcode-app-server-adapter.ts | 66 +- src/manager/task-manager.ts | 276 ++- src/runtime/model-settings.ts | 36 +- src/store/process-lock.ts | 230 +- src/store/task-store.ts | 16 +- src/worker/run-task.ts | 872 ++++---- test/audit-remediation.test.ts | 116 +- test/helpers.ts | 112 +- test/late-identity-lifecycle.test.ts | 175 ++ test/legacy-retry-generation.test.ts | 141 ++ test/manager-helpers.ts | 23 +- test/manager-recovery-latch.test.ts | 201 ++ test/owned-fixture-cleanup.test.ts | 152 ++ test/process-lock.test.ts | 651 +++++- test/process-spawn.test.ts | 342 ++- test/runtime-fixture.ts | 4 +- test/runtime-remediation.test.ts | 16 +- test/spawn-generation-identity.test.ts | 267 +++ test/teardown-retry.test.ts | 139 ++ test/validate-plugin-root-pattern.test.ts | 99 +- ...windows-cleanup-review-regressions.test.ts | 264 +++ test/windows-process-cleanup.test.ts | 441 +++- test/windows-transport-bounds.test.ts | 202 ++ test/worker-env-portability.test.ts | 112 +- 36 files changed, 8504 insertions(+), 1563 deletions(-) create mode 100644 docs/decisions/ADR-009-windows-identity-bound-tree-termination.md create mode 100644 docs/reports/2026-10-17-filesystem-root-guards.md create mode 100644 test/late-identity-lifecycle.test.ts create mode 100644 test/legacy-retry-generation.test.ts create mode 100644 test/manager-recovery-latch.test.ts create mode 100644 test/owned-fixture-cleanup.test.ts create mode 100644 test/spawn-generation-identity.test.ts create mode 100644 test/teardown-retry.test.ts create mode 100644 test/windows-cleanup-review-regressions.test.ts create mode 100644 test/windows-transport-bounds.test.ts diff --git a/docs/ARCHITECTURE.md b/docs/ARCHITECTURE.md index e19f8fe..5d0fbd7 100644 --- a/docs/ARCHITECTURE.md +++ b/docs/ARCHITECTURE.md @@ -1,9 +1,9 @@ # 当前架构 > Status: AUTHORITATIVE -> Last updated: 2026-10-15 -> Last verified: 2026-10-15，最新上游整合分支的验证结果见[审查说明](reports/2026-10-15-browser-and-transport-fixes.md)；旧版本实测与当前分支回归分别记录。 -> Verified against: 上游fae74e45033a50fb560c0014d1a6d60a7dc771d1 + 本分支修复；受测源码指纹见审查说明。 +> Last updated: 2026-10-17 +> Last verified: 2026-10-17，Windows 进程树身份绑定终止（ADR-009）经本机 Node 22.18.0/24.19.0 全量矩阵与 Codex 独立反例验证，证据在私有归档；其余沿承原核对范围。 +> Verified against: 0902236c488adf7a3280dfdc38401dfa9f41cb49 + 本分支修复；受测源码指纹见各修复记录。 本文件描述当前代码，不是历史阶段的冻结设计。公共类型见 `src/interfaces.ts`，MCP schema 见 `src/mcp/schemas.ts`；共享边界见 [SHARED_CORE.md](SHARED_CORE.md)。 @@ -41,7 +41,7 @@ Manager 用进程内 promise 队列以及同一 data root 下的 `.tasks/.manage 同一 data root 内，FIFO 队列按 worker 上限和执行路径重叠规则启动 detached worker。Spawner 显式传 attempt。worker 在进入 adapter 前通过 attempt 目录的永久 `execution.claim` 抢占执行权，拒绝重复、旧 attempt 和终态入场。`state.lock` 保护状态更新和结果提交；worker 提交还校验当前 attempt/非终态。Manager 的延迟 PID 写入只作用于仍 running 的同一 attempt。 -未启动 worker 可重拉一次；抢占过的 attempt 不会重复执行。已开始的 worker 不自动重跑。worker 丢失但记录的 ZCode PID 仍存活时保留占用，要求 `zcode_cancel` 验证清理。清理失败的终态任务也保留目录与 slot；续跑被拒绝，再次 cancel 或只读恢复探测确认退出后才释放，原 failed 结果不改写。进程恢复使用 PID 和启动 fingerprint；unknown 不当作已退出。Windows 强制清理先采集当前树及身份，再执行 taskkill，并核验采集到的根与后代均已退出；非零退出也可以由新退出证据解消，存活或 unknown 均不释放。公共错误使用稳定诊断代码，不转发本地化 taskkill 文本。采样后新建、根退出前已脱离树的后代，以及查询与发信号间的 PID 复用窗口仍属未充分验证边界。见 ADR-005。 +未启动 worker 可重拉一次；抢占过的 attempt 不会重复执行。已开始的 worker 不自动重跑。worker 丢失但记录的 ZCode PID 仍存活时保留占用，要求 `zcode_cancel` 验证清理。清理失败的终态任务也保留目录与 slot；续跑被拒绝，再次 cancel 或只读恢复探测确认退出后才释放，原 failed 结果不改写。进程恢复使用 PID 和启动 fingerprint；unknown 不当作已退出。Windows 强制清理按 ADR-009 执行：一个持久 PowerShell 事务 helper 为每个已验证成员持有原生句柄（持有期间 PID 不可复用），归属按创建时间表示区间与句柄精确时间零容差证明（早于父创建的旧记录在触碰前排除；同 100ns 不可区分即拒绝；自然退出父、未确认退出保守拒绝）；Windows 终止必须携带调用者期望根指纹，信号仅经"同一句柄内验证、同一句柄内 TerminateProcess"发出，不使用 taskkill /T；未解析 hold、候选消失、传输异常、helper 关闭失败均携带完整已知成员集合显式失败并保留占用。已记录的 tree-cleanup 失败以 `cleanup_unverified` 锁存：恢复与 worker 提交不得抹除证据；部分或无法恢复完整成员的记录保守阻断，完整 marker 回退及 legacy 无失败记录的 probe 恢复按 ADR-009 具体分支执行；已记录树失败的正常释放由后续 cancel 的新鲜 probe 与 claimant 扫描门控。公共错误使用稳定诊断代码，不转发本地化 taskkill 文本。已记录 PID 的 claimant 闭包外（未观察中间祖先）不可证明，不以完整清理名义宣称。见 ADR-005、ADR-009。 worker 每 3 秒原子写入一次绑定 attempt 与 PID 的私有 heartbeat，字段含 session、turn、Bridge event 序号和已观测到的 ZCode event 序号。管理器遇到一次负向 PID 探测时，若 heartbeat 不超过 15 秒则暂缓失联判定。ZCode turn 完成后，worker 在清理进程前写入 outcome checkpoint，并在清理成功后更新验证标志；worker 在提交最终 result 之前退出时，管理器可据 checkpoint 恢复报告，清理未验证则保留 `cleanup_failed` 与 workspace 占用。 diff --git a/docs/PROJECT_STATE.md b/docs/PROJECT_STATE.md index fb91b6f..4824466 100644 --- a/docs/PROJECT_STATE.md +++ b/docs/PROJECT_STATE.md @@ -1,9 +1,9 @@ # Project State > Status: AUTHORITATIVE -> Last updated: 2026-10-15 -> Last verified: 2026-10-15，最新上游整合分支的验证结果见[审查说明](reports/2026-10-15-browser-and-transport-fixes.md)；旧版本实测与当前分支回归分别记录。 -> Verified against: 上游fae74e45033a50fb560c0014d1a6d60a7dc771d1 + 本分支修复；受测源码指纹见审查说明。 +> Last updated: 2026-10-17 +> Last verified: 2026-10-17，本轮 Windows 进程树身份绑定终止（ADR-009）经本机 Node 22.18.0（461 项 460 pass/1 skip）与 24.19.0（459 pass/2 skip）全量矩阵、94 项专项单测、有界真实 fixture 4/4 与 Codex 九脚本十用例独立反例验证，证据在本轮私有归档；前轮文件系统根守卫的独立矩阵与 Linux 回归见[修复记录](reports/2026-10-17-filesystem-root-guards.md)，Linux 侧对本轮 ADR-009 改动的回归由 Codex 收尾执行。 +> Verified against: 0902236c488adf7a3280dfdc38401dfa9f41cb49 + 本分支修复；受测源码指纹见修复记录与本轮私有归档。 当前状态快照，回答"现在什么能用、什么不能用"。路线图和优先级不在这里。 @@ -11,16 +11,21 @@ ## 版本与发布 -- 当前审查基线：`1.2.4`（`package.json`、`plugins/codex-zcode-bridge/plugin.json`）。 +- 当前审查基线：`1.2.6`（`package.json`、`plugins/codex-zcode-bridge/plugin.json`）。 - 发布方式：release-please 监听 `master`，合并后自动开版本 PR；合并版本 PR 才产生 tag 与 GitHub Release。 - CI：`.github/workflows/ci.yml`，ubuntu 与 windows 两个作业，跑 typecheck、build、test、validate:plugin，并校验生成的 bundle 已随源码提交。 ## 本分支改动 -本分支基于上游1.2.4，保留其provider额度耗尽处理和控制面约束，加入可选浏览器宿主及协议帧/大审批输入修复。版本号未提升，不代表已发布；详情见[审查说明](reports/2026-10-15-browser-and-transport-fixes.md)。 +本分支基于上游 1.2.6（0902236），加入两项修复。 + +Windows 文件系统时序与根守卫（前轮）：`state.lock` 发布前存在性预检(已存在的空/未知 owner 锁不被 POSIX rename 覆盖)、发布 rename 的 EEXIST/ENOTEMPTY 按确定目的在场竞争移交 existing-owner 策略(不逃逸、不烧重试预算;EPERM/EACCES/EBUSY 目的缺失保持有界重试后可见抛出)、撤回 rename 对白名单瞬时码有界重试(撤回耗尽后锁保持完整、存活 owner 不驱逐;非瞬时码与共享 reclaim guard 清理失败保持可见,guard 清理在全部出口单点执行);worker 的迟到 self/runtime identity 写盘在终态前有界收束、超时丢弃,不再复活已清理的 store;测试 fixture 树清理带 Node 文档化有界重试;`buildRepoRootLeakPattern` 拒绝文件系统根输入（POSIX 根、盘符根及其词法等价写法、UNC 共享根）并修复 URL 主机名等于短根的误报。版本号未提升，不代表已发布；证据与边界见[修复记录](reports/2026-10-17-filesystem-root-guards.md)。 + +Windows 进程树身份绑定归属与终止（本轮，ADR-009）：生产 Windows 清理重写为单持久 PowerShell 事务——每个已验证成员持有原生句柄（持有期间 PID 不可复用），归属按 CIM 表示区间与句柄精确创建/退出时间零容差证明，早于父创建的旧记录在触碰前排除；信号仅在"同一句柄内 GetProcessTimes 验证、同一句柄内 TerminateProcess"发出，`taskkill /T` 移除；Windows 终止必须携带调用者期望根指纹（spawn 代捕获/持久化 identity），POSIX 语义不变。未解析 hold、候选消失、传输异常、helper 关闭失败均携带完整已知成员集合显式失败；manager 端 `cleanup_unverified` 单调锁存（恢复路径、worker 晚到提交、连续失败、记录损坏/截断均不得抹除），释放由后续 cancel 的新鲜 probe 与 recorded-tree claimant 扫描门控。传输有界（1 MiB/行、8 MiB/65536 行缓冲、4096 枚举行、120 s 会话）。 ## 稳定 +- Windows 进程树终止（ADR-009）：本机 Node 22/24 全量、专项单测 94 项、有界真实 fixture（正向身份绑定终止、错误指纹拒绝且零信号、缺身份拒绝、独立收束）与 Codex 九脚本十用例反例全部通过；claimant 闭包外（未观察中间祖先）不可证明，不以完整清理名义宣称。 - TaskManager 生命周期：`queued` / `running` / `completed` / `failed` / `cancelled` / `waiting_for_master`，含 attempt 独占、续跑归档、清理验证。 - MCP 默认工具 13 个，清单见 `INTERFACES.md`。 - `zcode_events` 的增量事件、cursor 与 raw/summary 视图。 @@ -35,7 +40,7 @@ - `zcode_progress_probe`：默认关闭，只有直接调用 `createBridgeServer({ enableExperiments: true })` 才注册。 - Desktop 索引同步：best effort，事务内校验 schema 与 Bridge owner；尚未写入真实 Desktop 数据库。 -- 可选 headless 浏览器宿主（ADR-007）：默认关闭；显式启用后通过独立managed CDP后端处理browserList/browserExecute。1.2.3修复版本曾在Windows Edge和ZCode Agent 0.16.9验证原生工具链；本1.2.4整合分支的验证单列于[审查说明](reports/2026-10-15-browser-and-transport-fixes.md)。桌面IAB、Computer Use、录制及子会话派生未实现。 +- 可选 headless 浏览器宿主（ADR-007）：默认关闭；显式启用后通过独立managed CDP后端处理browserList/browserExecute。1.2.3修复版本曾在Windows Edge和ZCode Agent 0.16.9验证原生工具链；历史本机1.2.4整合验证见[审查说明](reports/2026-10-15-browser-and-transport-fixes.md)。本轮1.2.6修复包已完成Windows Edge/Linux Chromium的打包MCP初始化与真实browser backend操作，范围见[本轮记录](reports/2026-10-17-filesystem-root-guards.md)，不等同其它模型或app-server版本的原生工具认证。桌面IAB、Computer Use、录制及子会话派生未实现。 ## 不支持 @@ -43,11 +48,11 @@ ## 调查中 / NOT RUN -以下边界没有实跑验证，不能当成已知可用： +以下为仍在调查或未完成认证的边界；已观察证据与未验证范围分别说明： - 浏览器原生链路仅在独立验收记录所列版本验证，其他模型、版本、平台未扩展验证。 - 浏览器宿主的跨安装版本兼容：协议 schema 依观测 commit `aac4755666d09fdcd70272fcf063c077a639015f` 适配；咨询字段做了容错，身份字段缺失或不可解析仍拒绝，但未在其它 app-server 版本上实测。 -- Linux/macOS 上的浏览器可执行文件发现：路径表来自官方实现，本机只验证了 Windows Edge。 +- Linux x64浏览器发现与Chromium操作已在本轮rootless断网容器验证；macOS/ARM及其它浏览器安装布局仍未验证。 - `session/read` 原生当前状态查询：未接入。 - 真实 ZCode app-server RPC 探针与跨版本兼容：只在记录过的 0.16.9 路径上观测过。 @@ -55,8 +60,7 @@ - 跨 Host 并发 attach/control：研究阶段结论为 NO-GO，除非上游提供 ownership/control 协议。 - 真实 GUI 关闭时序、UI 响应与取消时延。 - 真实 Desktop 数据库写入与刷新行为。 -- PID 重用，以及自行脱离进程组的后代进程。 -- Windows 树采样后新增的后代，以及采样与发信号之间的身份变化；已运行的真实 Windows 树清理回归不能覆盖这些窗口。 +- 其他 PID 复用与自行脱离进程组的后代场景：ADR-009 覆盖已记录 PID 的 claimant 闭包与句柄可证明窗口；闭包外（未观察中间祖先、对抗性伪造创建时间）不在证明范围。 - 旧不可读取 owner 与崩溃遗留 reclaim guard 的安全恢复；没有按年龄清除旧锁。 - 长会话真实 provider/报告一次合格率。调度已减少同次重复扫描，仍有随历史库增长的全量校验成本。 diff --git a/docs/README.md b/docs/README.md index 6f3c8da..8457aab 100644 --- a/docs/README.md +++ b/docs/README.md @@ -1,7 +1,7 @@ # 文档索引 / Documentation > Status: AUTHORITATIVE（仅指本索引） -> Last updated: 2026-10-14 +> Last updated: 2026-10-17 英文使用者从 [README.md](../README.md) 开始。技术文档为中文。本文是文档路由：先看状态等级，再按读者路径选文档。 @@ -48,7 +48,7 @@ English readers start at [README.md](../README.md). This index is bilingual; the |---|---|---|---| | [README.md](../README.md) | en | 2026-10-05 | 未记录 | | [README.zh-CN.md](../README.zh-CN.md) | zh | 2026-10-03 | 未记录 | -| [PROJECT_STATE.md](PROJECT_STATE.md) | zh | 2026-10-14 | 部分核对；详见文档顶部 | +| [PROJECT_STATE.md](PROJECT_STATE.md) | zh | 2026-10-17 | 部分核对；详见文档顶部 | | [ARCHITECTURE.md](ARCHITECTURE.md) | zh | 2026-10-14 | 部分核对；详见文档顶部 | | [INTERFACES.md](INTERFACES.md) | zh | 2026-10-14 | 部分核对；详见文档顶部 | | [SHARED_CORE.md](SHARED_CORE.md) | zh | 2026-10-06 | 未记录 | @@ -70,6 +70,7 @@ English readers start at [README.md](../README.md). This index is bilingual; the | [ADR-004](decisions/ADR-004-separate-control-state-and-observation.md) | 2026-10-07 | Accepted：分离运行时控制、权威状态与补充观察 | | [ADR-005](decisions/ADR-005-long-session-reliability.md) | 2026-10-08 | Accepted：长会话清理、锁释放、报告修复与 provider 兼容 | | [ADR-006](decisions/ADR-006-provider-quota-failure-diagnostics.md) | 2026-10-14 | Accepted：安全识别并明确提示 provider 用量额度耗尽 | +| [ADR-009](decisions/ADR-009-windows-identity-bound-tree-termination.md) | 2026-10-17 | DECISION：Windows 进程树身份绑定归属与终止 | | [decisions/roadmap-decisions-2026-09-27.md](decisions/roadmap-decisions-2026-09-27.md) | 2026-09-27 | 路线图与决策讨论 | | [decisions/reliability-repair-plan-v2-2026-10-03.md](decisions/reliability-repair-plan-v2-2026-10-03.md) | 2026-10-03 | 可靠性修复计划，含未完成项 | @@ -100,6 +101,7 @@ English readers start at [README.md](../README.md). This index is bilingual; the |---|---| | [reports/task-feedback-v0-1-implementation-2026-10-06.md](reports/task-feedback-v0-1-implementation-2026-10-06.md) | Task Feedback v0.1 的交付报告，一次性材料 | | [reports/2026-10-08-long-session-reliability.md](reports/2026-10-08-long-session-reliability.md) | 长会话故障证据、本地修复、验证与未运行边界 | +| [reports/2026-10-17-filesystem-root-guards.md](reports/2026-10-17-filesystem-root-guards.md) | Windows 锁时序重试、teardown 容错与文件系统根守卫的证据、修复与边界 | ### 自动生成 Generated diff --git a/docs/decisions/ADR-009-windows-identity-bound-tree-termination.md b/docs/decisions/ADR-009-windows-identity-bound-tree-termination.md new file mode 100644 index 0000000..71c3ea8 --- /dev/null +++ b/docs/decisions/ADR-009-windows-identity-bound-tree-termination.md @@ -0,0 +1,82 @@ +# ADR-009 Windows 进程树身份绑定归属与终止 + +> Status: DECISION +> Last updated: 2026-10-17 +> Last verified: 2026-10-17（实现后本机验证：Windows Node 22.18.0 全量 461 项 460 pass/1 skip、Node 24.19.0 全量 461 项 459 pass/2 skip，均 0 fail；Codex 反例脚本 9 个（10 个运行用例，recovery-latch 含两种模式）全部 ok；有界真实 fixture 4/4。证据与命令输出存于本轮私有证据归档 `local/windows-process-ownership-20261017/worker/attempt-2/`，不在源码仓库内） +> Verified against: 0902236c488adf7a3280dfdc38401dfa9f41cb49 + 本分支修复（含 review-round-1 R1–R4、review-round-1-addendum（absent 候选、传输预算、missing/truncated/repeated/repeated-truncated 成员证据）、review-round-2-callers C1 的全部返工） + +## 背景 + +旧策略（ADR-005 时代实现）：PowerShell 按 `ParentProcessId` 闭包展开树并记录各成员 StartTime ticks，probe 核验后执行 `taskkill /PID /T /F`。2026-10-17 只读回放与两轮集中评估确认的缺陷： + +1. 旧无关记录入树：`Win32_Process` 文档明确 `ParentProcessId` 在 PID 复用后不可信，需结合 `CreationDate`；旧闭包只按父 PID 匹配。 +2. 验证集与操作集解耦：`taskkill /T` 发信号时自行重展开；manager"先 probe 后仅传 PID"存在交接窗口；终止失败可被根退出证据解消。 +3. 失败证据可被抹除：恢复路径、worker 晚到提交、连续失败、记录损坏/缺失/截断均可能清掉已知后代失败锁存（R1/R3 及补充）。 +4. 未解析候选被当作可执行集合、传输无帧/总预算、helper 关闭回执不验证（R2/R4 及补充）。 + +## 决策 + +**身份绑定的归属与终止**：信号只经操作系统进程句柄发出，`taskkill /T` 完全移除；未知即保守失败，占用与失败证据单调保持。 + +### D1 归属：表示区间精确推理，零容差 + +- 单个持久 PowerShell helper 一次性枚举全表 `pid|ppid|CIM CreationDate.ToFileTimeUtc()|Get-Process StartTime.Ticks 或 unknown`；闭包在 TS 内确定性计算，行到达顺序不影响结果（父代未决的行保持 pending，杀序为真后序：每个成员先于自己的父被信号）。 +- CIM `CreationDate` 在本机观测为微秒截断（真实创建时间位于闭区间 `[cim, cim+1µs]`，N=215，偏差 ∈(0,0.9µs)）。归属推理把 CIM 表示区间用于"子在父后/明显早于父"的判定与行-句柄同代一致性核对；**信号对象的身份绑定由同一持有句柄上的精确创建时间保证**，这是与 CIM 表示无关的最终依据。 +- **活父边**：区间不重叠即证明"子在父后"；重叠时对双方当前持有句柄读精确创建时间（100ns），先核对精确值落在各自 CIM 区间（行-句柄同代一致），再比较：子严格晚于父成立；相等（同一 100ns）不可区分即拒绝。**早于父创建的行是旧无关记录，在打开句柄前按时间排除，不触碰无关进程**。 +- **被我们终止的父**：同一句柄 `TerminateProcess` → `WaitForSingleObject` 确认 → 读内核退出 FILETIME。子精确创建时间落在 `[父精确创建, 父精确退出]` 闭区间内为可证明的子（该区间内 PID 归属独占）；早于父创建或晚于父确认退出为可证明排除。等待超时的父（无确认退出时间）→ `process_tree_parent_exit_unconfirmed` 保守失败。 +- **自然退出的父**（无句柄、无退出时间）：其存活候选子不可证明归属 → 保守拒绝。 +- **根**：Windows 终止必须携带调用者期望根指纹（持久化 executor identity 或 spawn 代捕获），逐位相符才展开；**根缺席且未被本事务杀死时一律 `process_tree_root_exited` 拒绝**（新事务无法证明死根之子的归属，不作"无候选即成功"的推断）；根换代即 `process_tree_root_identity_changed` 拒绝。无可信期望身份即 `process_tree_root_identity_required`，不建裸 PID 信任。 + +### D2 终止：句柄绑定与失败携带 + +- 每轮对每个已验证存活成员：`OpenProcess(QUERY_LIMITED|TERMINATE|SYNCHRONIZE)` → 句柄上 `GetProcessTimes` 与该成员 ticks 逐位比对 → `TerminateProcess` → 同句柄等待退出并读退出时间。句柄跨轮持有至最终扫描，持有期间该 PID 不可复用（官方句柄契约）。**验证与信号作用于同一句柄、同一进程对象：不存在"检查后 PID 换代被杀错"的窗口**——这是本设计的核心保障，不依赖任何时间差；后续轮次的枚举只用于收敛证明（发现动态后代/确认闭包为空），不是对错误信号的补救。 +- **R2 语义**：hold 永久 denied、响应缺失/重复/意外、六轮预算耗尽、或候选在枚举与 hold 之间消失（`process_tree_candidate_absent`：其存活子代不可归属）都立即失败，错误携带全部已知成员 + 观察未验证候选 PID；不得把未证明完整的轮次当已验证清理，也不得漏掉存活孙代返回成功。 +- **R3 语义（按实现的实际分支）**：所有异常出口（含传输/查询错误）都包装为携带完整已知成员集合的 `WindowsTreeCleanupError`。manager 端失败记录为结构化 `cleanup-failure.json`（member_pids + complete 标志），状态错误文本内嵌持久 `[cleanup-members: …]` 标记。读取分级： + - meta 返回 null 或读取抛错，且 status 存在**完整** marker → marker 恢复的集合并入 claimant 扫描，允许正常释放； + - meta 对象**存在**但 member_pids 字段缺失/无效/null → `#validMemberPids` 直接拒绝，**即使存在完整 marker 也不绕过**（"存在却缺字段"是损坏，不是丢失）； + - marker 不完整（截断）或 `meta.complete === false`（部分集合标志）→ **持续拒绝，不承诺自动解除**：正常 cancel 不会把 partial 洗白，解除需要另外取得完整证据并修复记录（人工/上游介入后重新落盘完整记录）； + - 集合与 incomplete 状态单调：任何后续失败把 prior marker 无条件并入，不得以新鲜子集洗白； + - 无任何记录与 marker 的 legacy 终态锁存保留原根 probe 释放路径（既有行为）。 +- **R4 语义（实际预算）**：单行 ≤ 1 MiB；缓冲流 ≤ 8 MiB 且 ≤ 65 536 行；单次枚举 ≤ 4 096 行；整个事务会话一个 120 s 总等待预算（每行等待取剩余额，过期后已缓冲的行也不再供出）。超过任一预算即明确失败（`process_tree_transport_overflow` / `process_tree_transport_session_timeout`），失败后停止接收并清空缓冲。关闭必须收到 `CLOSED `（failed==0，逐个 CloseHandle 成功被计数）且 helper 以退出码 0 自然退出；kill 回退可收敛进程但作为可见失败（`process_tree_close_forced`）报告，绝不当成功。helper 的进度记录在源头抑制（`$ProgressPreference='SilentlyContinue'`）；仍有 stderr 数据即按错误记录使传输失败，本地化文本永不外发。 +- **持久摘要的实际边界**：status marker 在成员联串 ≤ 1 600 字符时完整保存，否则截断为前 200 项 + 省略号；结构化记录不受此限。成员超过该界的失败其 marker 必然不完整，按上述规则**保守占用**——这是有意的资源上限，不是静默数据丢失。 + +### D3 收敛与 manager 单调锁存 + +- 多轮：枚举→归属→（无存活成员即成功；否则逐个句柄绑定终止）→下一轮。动态后代经证明窗口同样被绑定终止；不可证明即保守失败。verified 只由"闭包内无存活成员"给出，根退出不构成证据。 +- manager：终止调用携带期望根指纹；任何终止失败 → `CANCEL_FAILED` + `cleanup_unverified` 锁存 + 成员记录；**对已记录 tree-cleanup 失败（记录或完整 marker 存在）的任务，恢复路径（persistedResult/checkpoint/worker_lost/terminal probe）在锁存期间保持占用**；无记录无 marker 的 legacy 终态锁存仍由根 probe 释放。worker 终态提交（`TaskStore.commitWorkerResult`）可保存结果但不得清锁存或覆盖成员证据（无论 patch 清除、重申还是省略该字段）；释放只能由后续 cancel 经新鲜 probe + recorded-tree claimant 扫描门控完成。 +- claimant 扫描的 `clean` 证明："**已记录 PID 名下没有存活的、属于我们或不可归属的 claimant**"——含指纹匹配的存活记录 PID 本身、其存活子代（传递闭包）、已死记录 PID 的存活子代（归属不可判定，保守计入）；已证外国换代的存活 PID 及其子树排除。未观察到的中间祖先在任何证明之外，调用方须记录该边界而非宣称完整。 + +### D4 调用链身份交接（每代状态） + +- adapter（生产 app-server、legacy CLI、model-settings）在 spawn 后经共享 `captureSpawnGenerationIdentity` 采集本代身份。连续性判定依据：**libuv 在同步交付 Node 'exit' 回调之前释放子进程原生句柄；Node 在该回调内设置 exitCode/signalCode；JS continuation 无法插入两者之间**。因此"exitCode/signalCode 均为 null"蕴含"进程仍存活或 libuv 仍持有句柄"，两种状态下 PID 都不可复用——这是可证明的代连续性。close 旗标晚于 exit，不构成句柄持有证明。每次 await 后复核原对象；仅接受 exact 精度；返回时 pid 一致性复核；退出竞态的指纹一律丢弃。Windows 终止前复核仍在原代；已 exit 的子进程永不再信号。 +- 重试复用 RunEntry 时，childClosed/childIdentity 等每 spawn 字段按代重置；上一代迟到的身份捕获不得覆盖当代（review-round-2 C1）。 +- POSIX 保持既有进程组身份/信号语义不变（Windows 专项）。 + +## 依据（均已核对） + +- [Win32_Process](https://learn.microsoft.com/en-us/windows/win32/cimwin32prov/win32-process)：PID 复用与 CreationDate 对父子关系的作用。 +- [taskkill](https://learn.microsoft.com/en-us/windows-server/administration/windows-commands/taskkill)：/T 自行展开子树（本设计弃用）。 +- [TerminateProcess](https://learn.microsoft.com/en-us/windows/win32/api/processthreadsapi/nf-processthreadsapi-terminateprocess)：按句柄定位进程对象。 +- [PROCESS_INFORMATION](https://learn.microsoft.com/en-us/windows/win32/api/processthreadsapi/ns-processthreadsapi-process_information) 与 [Process Handles and Identifiers](https://learn.microsoft.com/en-us/windows/win32/procthread/process-handles-and-identifiers)：PID 在所有句柄关闭且对象释放后才可复用；句柄在进程退出后仍有效——持有句柄阻断复用是本设计窗口归属的依据。 +- libuv `src/win/process.c`（v1.x）：`uv__process_proc_exit` 先 `CloseHandle(process_handle)` 再触发 exit 回调；Node 在 exit 回调内设置 exitCode/signalCode（[child_process 事件顺序](https://nodejs.org/api/child_process.html)）。 + +## 残余边界（披露，不宣称解决） + +- 身份不是同账户恶意进程认证（沿承 process-probe 边界）。 +- claimant 扫描只覆盖已记录 PID 及其当前存活 claimant；未观察中间祖先不可证明。 +- spawn 后、身份采集完成前子进程退出且 PID 被复用的窗口：capture 在 exit 状态复核处丢弃该指纹，Windows 终止随后拒绝（缺身份 fail-closed）；这保证不会杀错对象，但不构成对 spawn 代连续性的正面证明（完全消除需 spawn 时刻持有原生句柄，超出本轮无新依赖边界）。 +- 传输/枚举预算是有界的资源上限：进程表超过 4 096 行、成员联串超过 1 600 字符（截断为部分集）或会话超预算时按明确失败/保守占用处理（保守方向）。 +- 本机 CIM 截断观测与句柄换算核验仅在 Windows 11 10.0.26200 / PowerShell 5.1 上进行；其他 Windows/CIM 实现未经观测，**不能宣称同等的归属保证**——同一句柄的精确时间仍是信号对象身份绑定的不变保障。 + +## 可重跑验证（本轮实际结果；命令在源码仓库执行，原始输出存私有证据归档 worker/attempt-2/logs/） + +- `node --test dist-test/test/{windows-process-cleanup,windows-cleanup-review-regressions,windows-transport-bounds,manager-recovery-latch,spawn-generation-identity,audit-remediation,legacy-retry-generation}.test.js`：94/94。 +- `node --test dist-test/test/process-spawn.test.js`（有界真实 fixture：正向身份绑定终止、错误指纹拒绝且零信号、缺身份拒绝、独立收束）：4/4。 +- Codex 反例脚本（存私有证据归档 evaluator/，新 attempt id 可重跑）：9 个脚本 / 10 个运行用例全部 ok——recovery-latch-probe（recover 与 corrupt-meta 两模式）、adversarial-cleanup、absent-candidate、missing-members、truncated-members、repeated-failure、repeated-truncated、worker-commit-latch、legacy-retry。 +- 全量：Windows Node 22.18.0 `node scripts/test-isolated.mjs` 461 项 460 pass/1 skip；Node 24.19.0 同命令 459 pass/2 skip；`npm run typecheck`、`npm run build`、`npm run validate:plugin`、`git diff --check` 均 exit 0。 + +## 影响 + +- 修改 `src/adapters/windows-process-cleanup.ts`（重写：区间归属、句柄事务、传输预算）、`src/adapters/process-spawn.ts`（期望根指纹、代连续性 helper、claimant 扫描）、`src/adapters/zcode-app-server-adapter.ts`、`src/adapters/zcode-adapter.ts`（每代重置）、`src/runtime/model-settings.ts`、`src/manager/task-manager.ts`（单调锁存、扫描门控）、`src/store/task-store.ts`（commitWorkerResult 锁存保留）及测试；`docs/ARCHITECTURE.md` Windows 清理段随本决策更新。 +- 公共 MCP 工具名/schema、任务状态语义、隐私边界、npm/native 运行时依赖不变；错误为 ASCII 稳定诊断码。 +- 取代旧"采样后 taskkill /T"语义；ADR-005 的 unknown-不当退出、占用保留原则不变。 diff --git a/docs/decisions/README.md b/docs/decisions/README.md index a31cee4..d57b8c2 100644 --- a/docs/decisions/README.md +++ b/docs/decisions/README.md @@ -29,6 +29,7 @@ ADR-001 到 ADR-003 只搬运权威文档里已经写明的结论。ADR-004 原 - [ADR-007](ADR-007-browser-host-rpc.md)：可选headless浏览器宿主；默认关闭，限定版本实测。 - [ADR-008](ADR-008-session-frames-and-large-interactions.md)：会话帧与大审批输入；独立资源边界及可恢复错误。 +- [ADR-009](ADR-009-windows-identity-bound-tree-termination.md)：Windows 进程树身份绑定归属与终止；句柄事务、零容差区间归属、单调失败锁存与有界传输。 ## 命名 diff --git a/docs/reports/2026-10-17-filesystem-root-guards.md b/docs/reports/2026-10-17-filesystem-root-guards.md new file mode 100644 index 0000000..f22949b --- /dev/null +++ b/docs/reports/2026-10-17-filesystem-root-guards.md @@ -0,0 +1,121 @@ +# 文件系统根守卫与 Windows 锁时序修复记录 + +> Status: RESEARCH（一次性交付与验证记录，不替代当前合同） +> Date: 2026-10-17 +> Base: 0902236c488adf7a3280dfdc38401dfa9f41cb49（上游 1.2.6） +> 分支：codex/filesystem-root-guards-20261017 + +PR #45 维护者审查遗留的两条 P3：`state.lock` 竞争 EPERM/ENOTEMPTY 与测试 teardown ENOTEMPTY 同属 Windows 文件系统时序问题；`buildRepoRootLeakPattern` 对病态检出根（`/`、`C:\`）无守卫。本记录汇总证据、修复与验证边界；原始探针输出保存于任务私有目录 `local/filesystem-root-guards-20261017/worker/`。 + +## 锁竞争证据（自然复现 + 确定性注入） + +- 原始失败的测试输出未在旧任务目录中保留（2026-10-16 两轮目录均无 ENOTEMPTY/EPERM 原文），按"区分自然复现与确定性注入"处理，不虚构原始输出。 +- 自然复现（基线代码、真实跨进程竞争，6 进程 × 5000 次 acquire/release）：Node 22.18.0 逃逸 2 次、Node 24.19.0 逃逸 1 次，错误均为 `EPERM: rename '/state.lock' -> '/state.lock..retired'`，即**释放路径**的撤回 rename 失败（这是直接观察）。机制解释（假设，未做句柄层验证）：竞争者 `acquireExistingLock` 读取 owner.json 时可能在目录内短暂持有句柄，Windows 拒绝对含打开句柄子文件的目录 rename；无论具体机制如何，该错误从 `withEventLock` 的 `finally` 抛出并掩盖已成功完成的状态写入是被观察到的行为。获取路径的瞬时失败（目的名 delete-pending）此前已有 #30 的 EEXIST/EPERM/EACCES ≤3 次立即重试，但不含 ENOTEMPTY/EBUSY、无间隔。 +- 确定性注入（补丁 `fs.renameSync` 注入错误码）：瞬时码到达白名单路径时按预算重试；非瞬时码（EINVAL）第一次即抛；预算耗尽抛原始错误码且不留 staging/retired 残留。 + +## 修复（src/store/process-lock.ts） + +- 瞬时码白名单统一为 `EEXIST/EPERM/EACCES/EBUSY/ENOTEMPTY`（依据：本机自然复现 EPERM；Node 22.18 官方 fs 文档将 EBUSY/EMFILE/ENFILE/ENOTEMPTY/EPERM 列为 recursive rm 的可重试码；libuv 将 sharing violation 映射为 EBUSY、access denied 映射为 EPERM/EACCES）。 +- 发布竞争（rename staging→锁名失败且目的名不存在）：保留 ≤3 次重试，白名单扩大，并在重试间加 10ms 停顿等 delete-pending 澄清。 +- 撤回 rename（锁名→`.retired`）与死 owner 回收 rename：新增有界重试（2 秒预算、10ms 间隔）；预算耗尽抛出原错误，共享锁保持完整可读 owner 记录，后继获取者看到存活 owner 得到 null，不驱逐。 +- 私有目录清理:staging/retired 的 rmdir 与 owner 文件 unlink 对瞬时码 1 秒有界重试;尽力而为边界仅适用于这些**私有**残留(ADR-005),不覆盖原获取错误。reclaim guard 不属私有 best-effort——见 Attempt 2/3 的最终语义(共享状态,清理失败必须可见)。crash-safety 测试保持不变并通过。 +- 不变量不变：staged rename 仍是唯一发布点（并发最多一个 owner）、未知 owner 不按年龄驱逐、非瞬时错误与预算耗尽都可见失败。 + +修复后同强度自然竞争（6×5000）在 Node 22.18.0 与 24.19.0 均 0 逃逸。 + +## teardown ENOTEMPTY 修复（test/） + +- `runtime-remediation` 的 fixture 清理是无重试的 `rm(recursive, force)`。150 轮"子进程在树内写日志后退出再删除"模拟未自然复现（如实记录：原始抖动更罕见，与全量并发负载相关）。 +- Node 22.18 文档核对：`fs.rm` 系列在 recursive 模式对 EBUSY/EMFILE/ENFILE/ENOTEMPTY/EPERM 按 `retryDelay` 线性退避重试 `maxRetries` 次，默认 0。确定性注入（Node 22 上补丁 `fs.rmdirSync` 可被 rmSync 观测）证明：连续注入 ENOTEMPTY 时默认选项失败可见，`maxRetries>=2` 存活。 +- 修复：`test/helpers.ts` 导出 `TEMP_TREE_REMOVE_OPTIONS = { maxRetries: 5, retryDelay: 100 }`，`removeTempDir` 与 `runtime-fixture` 的 cleanup 采用。预算耗尽仍抛出，不掩盖真实清理失败；其余测试文件的内联清理未改动（超出本次证据范围）。 + +## 根守卫（scripts/lib/repo-root-pattern.mjs） + +- 基线行为（探针留档）：根 `/` 时任意 `//` 行注释与任意 `file://` URI 均判泄漏；根 `C:\` 时同盘任意绝对路径（`C:\Windows\...`）均判泄漏。检查退化为必然误报。 +- 新增 `assertNotFilesystemRoot(root)` 并在 `buildRepoRootLeakPattern` 入口调用：拒绝 POSIX 根（`/`、`//`）、盘符根及其等价写法（`C:`、`C:\`、`C:/`、大小写、`\\?\C:\`、`\\.\C:\`）、UNC 共享根（`\\server\share`、`\\?\UNC\server\share`）与非字符串/空白输入，错误信息说明输入、根种类与后果（必然误报、POSIX 根连 `//` 注释都命中），并指引用命名目录下的 checkout。抛错使 validate-plugin 以非零退出显式失败，不静默跳过泄漏检查。测试使用输入夹具，未把仓库放到磁盘根目录。 +- 调查中发现并一并修复的相邻误报：URL 主机名恰为短根名时（如根 `/app` 对 `https://app/config`），http 抑制 lookbehind 无法跨 `://` 边界回看，基线会误报；已在 scheme 冒号后首位增加 `(?500ms 协作窗时 manager 终止假 worker 并写终态,迟到 commitWorkerResult 以 stale/terminal 拒绝——与现场错误一致。边界如实说明:600ms 变体证明迟到提交拒绝机制,不证明原 104ms 现场具体超窗原因(矩阵负载下事件循环延迟使 40ms 定时器迟到/轮询饥饿为合理推断,未直接观测)。修复(仅测试 fixture,不动 manager/时钟策略):finish 改为协调于 store 中持久化的 cancel_requested 记录(有界 5s 轮询)后立即提交,确定性落在 500ms 窗内;实质断言不变(worker 结果被验证、无第二次终止、session 传播)。新增边界测试钉住 >500ms 策略:恰好一次已验证终止、迟到提交按契约拒绝、manager 结果优先。 +3. Linux 定向验证脚本 sandbox/worker/linux-targeted-attempt5.mjs(errno 分类二探针+三测试;Windows 侧未运行,NOT RUN,交 Codex 在其既有隔离资源执行)。定向回归 10 文件 147 项 146 通过/0 失败/1 既有跳过;typecheck/build/validate:plugin/diff exit 0;最终双平台矩阵仍由 Codex 执行。 + +## Attempt 6（2026-10-17 收敛修正） + +attempt-5 复验(evaluator/attempt-5/review.md):15 文件指纹匹配,publication 分类 7/7 独立通过,锁分类实现进入最终验证候选不再改动。本轮仅三项收敛: + +1. **600ms 边界测试非确定性**:Codex 反例(cancel-boundary-timing.json:finish(600) 先调度、cancelTask 延 188ms 启动时,worker 605ms 提交仍落 500ms 窗内,0 终止)证明固定定时不能保证提交晚于 manager 终态。重写边界测试为因果序:先 await manager 完成已验证取消(此时其终态已落盘、假 worker 已被恰一次终止),再尝试 worker 侧 commit——确定性 stale 拒绝、恰一次终止、manager 结果优先,无任何定时假设。协作取消测试保留显式意图协调;注释改为事件化协调描述,不再称数学时序保证。 +2. **原始失败原因措辞**:测试注释不再把矩阵负载延迟称为现场观察到的原因;仅保留观察到的事实(104ms TAP 时长的 stale/terminal 拒绝),600ms 诊断为可能的拒绝机制而非已证原始原因。 +3. **POSIX 术语**:process-lock.ts 分类注释改为"目的已作为非空目录存在"(POSIX rename 对既有空目录是静默替换——正是存在性预检针对的问题);报告同步修正。 + +仅运行受影响的取消/锁定向测试+typecheck/build/validate:plugin/diff;不重复 10 文件定向、全量矩阵或 6x5000 压力(记录保留);最终矩阵由 Codex 在修正后的冻结交付上执行。 + +## Attempt 7（2026-10-17 矩阵二相关 fixture 生命周期返工） + +矩阵二复验(evaluator/matrix-attempt-2/review.md):Linux 全通过。Windows 两项失败需分开表述:Windows22 二轮失败是本任务 fixture 生命周期缺口(丢弃 pid、receipt 后立即删树致真实 EBUSY);Windows24 的**主失败**是未改生产 capture 的 malformed-identity 保守拒绝(基线既有问题,Codex 另记债,本轮不扩大修复),其后的 fixture 缺口(finally 不收束 owned children)导致遗留子进程并挂住整轮——两项分别处理,不混同源: + +1. **worker-env-portability 丢弃真实 spawner 返回的 pid,receipt 后立即删树**(受控证明 worker-env-exit-proof.json:receipt 在场而子进程持活时 rmdir 复现 EBUSY,允许退出后清理成功——生命周期缺口成立,未记录原始失败瞬间的句柄持有者):探针 worker 现于 receipt 后监听私有 release 文件;两个环境测试捕获 owned pid,finally 经共享门禁 helper(写 release→有界确认 owned pid 退出→确认后才删树);新增确定性回归"receipt 不等于 exit"(持活 worker 在 receipt 后可证存活,释放并确认退出后才移除树)。 +2. **process-spawn 测试 finally 不收束 owned children**:parent/sleeper 双 fixture 脚本各自独立监听同一私有 release(仅在主断言或其失败之后的 finally 触发,不能掩盖生产行为);父脚本对自己持有的 child 句柄仅在真实 close 事件后退出(spawn error 单独记账并按失败退出);测试侧从 spawn 即登记 error/close、完整行解析 pid 回执,回执失败时已知 owned 父 pid 仍被释放且目录按 unknown-descendant 门禁保留;生产 terminateProcessTree 的原断言完全不变,并新增失败路径覆盖(不调用生产终止时,协作释放独自收束两个已知进程并清理)。按 Safety addendum,在调用未改动的生产树终止器之前执行只读归属前置:CIM 快照闭包内 node 行必须恰为记录的父/孙 pid 且命令行引用本次唯一 fixture 脚本,sleeper 父关系与严格 BigInt 创建时序(无容差)不得早于 root,console host 必须挂于已验证 pid 且不早于其父;任何无关/更旧/未知行在发信号前显式失败(matrix2 事故=共享 ParentProcessId 的十月15 系统行被今日 root 选中,正是该门禁拦截的形状)。新增 fixture fallback 仅通过私有 release 与 owned child handle 收束,不做宽泛树终止;正向真实测试在归属前置通过后仍按原路径调用生产 terminateProcessTree(其 Windows 生产实现内部照旧使用 taskkill 树终止),主测试路径保持不变。 +3. **共享门禁 helper**(test/helpers.ts cleanupOwnedProcessFixture):release 信号写出失败、任一 owned pid 退出确认超时、或显式 preserveTree(未知后代)时一律保留 fixture 树并显式报告;成功路径由调用方一次性标记避免重复调用;primary 失败传播时诊断不被遮蔽也不遮蔽 primary。配套 owned-fixture-cleanup.test.ts 三态回归(合作子进程经 ready 握手/cwd 断言/exit code 0;拒绝子进程保树且显式失败;primary 传播时诊断可见、树保留、错误身份不变)。 + +验证:受影响三文件定向 9 项 9 通过 0 失败(过程失败两轮保留:console.error 恢复顺序、终化器先于断言删树);typecheck/build/validate:plugin/diff 全 exit 0。未修改生产 windows-process-cleanup/process-spawn/manager 策略;未跑全矩阵或多余压力。生产 Windows 身份快照保守拒绝为独立既有问题,由 Codex 记债并限定验收说明。 + +## Codex 独立验收追加（2026-10-17） + +本节由 Evaluator 在 attempt7 代码冻结后追加；只更新文档，未修改受测实现、测试或生产 bundle。此前 worker 自测和各轮失败仍按原时点保留。 + +| 独立完整回归 | 实际结果（每轮） | +| --- | --- | +| Windows x64 / Node 22.18.0，两轮 | 每轮398项：397通过、1跳过、0失败、0取消 | +| Windows x64 / Node 24.19.0 | 398项：396通过、2跳过、0失败、0取消 | +| Linux x64 / Node 22.18.0 | 399项：397通过、2跳过、0失败、0取消 | +| Linux x64 / Node 24.20.0 | 399项：396通过、3跳过、0失败、0取消 | + +双方 typecheck/build/validate:plugin/clean-entrypoints 均通过。Windows 为独立目录与测试数据根，不是 OS 沙箱；Linux 在 /app、rootless、UID1000、断网、零宿主挂载、capabilities清零、no-new-privileges 条件下完成，运行时约束已检查。生成包的跨平台规范化指纹一致。 + +补充独立验证：39个根输入案例、真实6进程/900次临界区互斥与退出回收、锁错误/超时分支、迟到self/runtime identity、4种真实child清理状态、11种无信号归属输入均通过。新测试夹具只有在已知进程退出后才能删树；未确认状态保留现场。Windows Edge与Linux Chromium各完成16步打包链路，页面与服务端均确认单次提交；浏览器进程与本次监听端口关闭。该链路未使用真实模型账号调用。 + +跳过项按平台与Node版本保留：Windows不执行POSIX权限位断言；Linux跳过Windows专属启动/进程树语义；Node24跳过依赖Node22内部可注入路径的rm错误注入分支，保留其接线回归，Node22实际注入验证另存。没有把跳过视为已覆盖。 + +限定接收仍保留两类边界：维护者那一次历史teardown抖动的唯一根因未被证明；生产Windows树采样的旧ParentProcessId记录/身份拒绝是独立既有问题，本轮未放宽或修改生产策略。测试夹具的归属前置通过，不代表该生产问题已解决。macOS/ARM、其它app-server版本与真实模型账号链路不在本轮新增认证范围。 + +完整原始结果、早期失败、源指纹及最终资源清理回执由任务私有 evaluator 目录保留；本节不发布本机日志、会话标识或运行数据。源码交付未提交、推送、发布或安装。 diff --git a/plugins/codex-zcode-bridge/dist/bridge.mjs b/plugins/codex-zcode-bridge/dist/bridge.mjs index 1007770..7bc6a0a 100644 --- a/plugins/codex-zcode-bridge/dist/bridge.mjs +++ b/plugins/codex-zcode-bridge/dist/bridge.mjs @@ -18,6 +18,12 @@ var __export = (target, all) => { import { existsSync as existsSync2, mkdirSync, readFileSync as readFileSync2, renameSync, rmdirSync, statSync as statSync2, unlinkSync, writeFileSync } from "node:fs"; import { randomUUID } from "node:crypto"; import path3 from "node:path"; +function isTransientLockFsError(error2) { + return TRANSIENT_LOCK_FS_CODES.has(error2?.code ?? ""); +} +function pauseBriefly() { + Atomics.wait(new Int32Array(new SharedArrayBuffer(4)), 0, 0, RETRY_PAUSE_MS); +} function publishSelfIdentity(identity) { selfIdentity = identity; } @@ -51,6 +57,7 @@ function tryAcquireProcessLock(directory) { } function tryAcquireProcessLockWithRetry(directory, retries) { const token = randomUUID(); + if (existsSync2(directory)) return acquireExistingLock(directory); const staging = `${directory}.${process.pid}.${token}.pending`; const ownerRecord = { pid: process.pid, @@ -67,14 +74,16 @@ function tryAcquireProcessLockWithRetry(directory, retries) { renameSync(staging, directory); } catch (error2) { removeStagedLock(staging); - if (!existsSync2(directory)) { - const code = error2.code; - if (retries < 3 && (code === "EEXIST" || code === "EPERM" || code === "EACCES")) { - return tryAcquireProcessLockWithRetry(directory, retries + 1); - } - throw error2; + const code = error2.code; + if (!isTransientLockFsError(error2)) throw error2; + if (DESTINATION_PRESENT_CONTENTION_CODES.has(code ?? "") || existsSync2(directory)) { + return acquireExistingLock(directory); + } + if (retries < PUBLICATION_RETRY_ATTEMPTS) { + pauseBriefly(); + return tryAcquireProcessLockWithRetry(directory, retries + 1); } - return acquireExistingLock(directory); + throw error2; } } catch (error2) { removeStagedLock(staging); @@ -84,38 +93,72 @@ function tryAcquireProcessLockWithRetry(directory, retries) { const owner = JSON.parse(readFileSync2(path3.join(directory, "owner.json"), "utf8")); if (owner.token !== token) throw new Error("process lock ownership changed"); const retired = `${directory}.${token}.retired`; - renameSync(directory, retired); + renameWithTransientRetry(directory, retired); removeStagedLock(retired); }; } +function renameWithTransientRetry(source, target) { + const deadline = Date.now() + WITHDRAW_RETRY_MS; + while (true) { + try { + renameSync(source, target); + return; + } catch (error2) { + if (!isTransientLockFsError(error2) || Date.now() >= deadline) throw error2; + pauseBriefly(); + } + } +} function acquireExistingLock(directory) { + let owner; try { - const owner = JSON.parse(readFileSync2(path3.join(directory, "owner.json"), "utf8")); + owner = JSON.parse(readFileSync2(path3.join(directory, "owner.json"), "utf8")); if (!Number.isSafeInteger(owner.pid) || owner.pid <= 0) throw new Error("invalid lock owner"); - try { - process.kill(owner.pid, 0); - } catch (failure2) { - if (failure2.code === "ESRCH") { - if (reclaimDeadOwner(directory)) return tryAcquireProcessLock(directory); - } - } } catch { try { if (Date.now() - statSync2(directory).mtimeMs > 3e4) throw new Error(`unreadable lock owner: ${directory}`); } catch (failure2) { if (failure2.code !== "ENOENT") throw failure2; } + return null; + } + try { + process.kill(owner.pid, 0); + } catch (failure2) { + if (failure2.code === "ESRCH") { + if (reclaimDeadOwner(directory)) return tryAcquireProcessLock(directory); + } } return null; } function removeStagedLock(directory) { - try { - unlinkSync(path3.join(directory, "owner.json")); - } catch { + removePrivateFile(path3.join(directory, "owner.json")); + removePrivateDirectory(directory); +} +function removePrivateFile(file) { + const deadline = Date.now() + PRIVATE_CLEANUP_RETRY_MS; + while (true) { + try { + unlinkSync(file); + return; + } catch (error2) { + const code = error2?.code; + if (code === "ENOENT" || !isTransientLockFsError(error2) || Date.now() >= deadline) return; + pauseBriefly(); + } } - try { - rmdirSync(directory); - } catch { +} +function removePrivateDirectory(directory) { + const deadline = Date.now() + PRIVATE_CLEANUP_RETRY_MS; + while (true) { + try { + rmdirSync(directory); + return; + } catch (error2) { + const code = error2?.code; + if (code === "ENOENT" || !isTransientLockFsError(error2) || Date.now() >= deadline) return; + pauseBriefly(); + } } } function reclaimDeadOwner(directory) { @@ -126,31 +169,70 @@ function reclaimDeadOwner(directory) { if (error2.code === "EEXIST") return false; throw error2; } + let outcome; try { const owner = JSON.parse(readFileSync2(path3.join(directory, "owner.json"), "utf8")); - if (!Number.isSafeInteger(owner.pid) || owner.pid <= 0) return false; + if (!Number.isSafeInteger(owner.pid) || owner.pid <= 0) { + outcome = { kind: "busy" }; + } else { + try { + process.kill(owner.pid, 0); + outcome = { kind: "busy" }; + } catch (failure2) { + if (failure2.code !== "ESRCH") { + outcome = { kind: "busy" }; + } else { + const retired = `${directory}.${randomUUID()}.retired`; + renameWithTransientRetry(directory, retired); + removeStagedLock(retired); + outcome = { kind: "reclaimed" }; + } + } + } + } catch (error2) { + outcome = error2.code === "ENOENT" ? { kind: "busy" } : { kind: "error", error: error2 }; + } + let guardCleanupError = null; + try { + removeSharedGuard(guard); + } catch (error2) { + guardCleanupError = error2; + } + if (outcome.kind === "error") throw outcome.error; + if (guardCleanupError !== null) throw guardCleanupError; + return outcome.kind === "reclaimed"; +} +function removeSharedGuard(guard) { + const deadline = Date.now() + SHARED_GUARD_RETRY_MS; + while (true) { try { - process.kill(owner.pid, 0); - return false; + rmdirSync(guard); + return; } catch (error2) { - if (error2.code !== "ESRCH") return false; + const code = error2?.code; + if (code === "ENOENT") return; + if (!isTransientLockFsError(error2) || Date.now() >= deadline) { + throw new Error( + `could not remove the lock reclaim guard at ${guard} (${code ?? "unknown error"}); a leftover guard blocks all future dead-owner reclaims of this lock and needs manual diagnosis`, + { cause: error2 } + ); + } + pauseBriefly(); } - const retired = `${directory}.${randomUUID()}.retired`; - renameSync(directory, retired); - removeStagedLock(retired); - return true; - } catch (error2) { - if (error2.code !== "ENOENT") throw error2; - return false; - } finally { - rmdirSync(guard); } } -var selfIdentity; +var selfIdentity, TRANSIENT_LOCK_FS_CODES, DESTINATION_PRESENT_CONTENTION_CODES, PUBLICATION_RETRY_ATTEMPTS, WITHDRAW_RETRY_MS, SHARED_GUARD_RETRY_MS, PRIVATE_CLEANUP_RETRY_MS, RETRY_PAUSE_MS; var init_process_lock = __esm({ "src/store/process-lock.ts"() { "use strict"; selfIdentity = null; + TRANSIENT_LOCK_FS_CODES = /* @__PURE__ */ new Set(["EEXIST", "EPERM", "EACCES", "EBUSY", "ENOTEMPTY"]); + DESTINATION_PRESENT_CONTENTION_CODES = /* @__PURE__ */ new Set(["EEXIST", "ENOTEMPTY"]); + PUBLICATION_RETRY_ATTEMPTS = 3; + WITHDRAW_RETRY_MS = 2e3; + SHARED_GUARD_RETRY_MS = 2e3; + PRIVATE_CLEANUP_RETRY_MS = 1e3; + RETRY_PAUSE_MS = 10; } }); @@ -449,7 +531,14 @@ var init_task_store = __esm({ this.assertWorkerAttempt(taskId, attempt); this.#writeJsonAtomic(path4.join(this.taskDir(taskId), "result.json"), result); const current = this.readStatus(taskId); - this.#writeJsonAtomic(path4.join(this.taskDir(taskId), "status.json"), { ...current, ...patch, task_id: taskId, attempt, updated_at: (/* @__PURE__ */ new Date()).toISOString() }); + let effective = patch; + if (current.cleanup_unverified === true) { + effective = { ...patch, cleanup_unverified: true }; + if (typeof current.error === "string" && current.error.includes("[cleanup-members:")) { + effective = { ...effective, error: current.error }; + } + } + this.#writeJsonAtomic(path4.join(this.taskDir(taskId), "status.json"), { ...current, ...effective, task_id: taskId, attempt, updated_at: (/* @__PURE__ */ new Date()).toISOString() }); }); } patchRunningAttempt(taskId, attempt, patch) { @@ -23663,9 +23752,6 @@ function createWorkerSpawner(host) { } var defaultSpawnWorker = createWorkerSpawner(); -// src/adapters/process-spawn.ts -import { spawn as spawn3 } from "node:child_process"; - // src/runtime/process-probe.ts import { readFile } from "node:fs/promises"; import { spawn as spawn2 } from "node:child_process"; @@ -23883,91 +23969,835 @@ function livenessVerdict(pid, isAlive, now) { } // src/adapters/windows-process-cleanup.ts -import { performance } from "node:perf_hooks"; -async function terminateWindowsProcessTree(pid, killWaitMs, dependencies) { - const requests = await dependencies.capture(pid); - if (!requests.some((request) => request.pid === pid)) throw new Error("process_tree_snapshot_invalid: root missing"); - const inspect = async () => { - const verdicts = await dependencies.probe(requests); - if (verdicts.length !== requests.length || verdicts.some((verdict2) => verdict2.state === "unknown")) { - throw new Error("process_tree_probe_unverified: recorded executor exit is unknown"); - } - return verdicts; - }; - const before = await inspect(); - if (before.every((verdict2) => verdict2.state === "exited")) return; - const rootIndex = requests.findIndex((request) => request.pid === pid); - if (before[rootIndex]?.state !== "alive") { - throw new Error("process_tree_descendants_alive: root exited before termination"); - } - let exitCode = null; - let commandFailed = false; +import { spawn as spawn3 } from "node:child_process"; +var CIM_REPRESENTATION_INTERVAL_TICKS = 10n; +function withinCimInterval(exactFileTime, cimFileTime) { + const exact = BigInt(exactFileTime); + const cim = BigInt(cimFileTime); + return exact >= cim && exact <= cim + CIM_REPRESENTATION_INTERVAL_TICKS; +} +function cimIntervalWithin(cimFileTime, lowExact, highExact) { + const cim = BigInt(cimFileTime); + return cim >= BigInt(lowExact) && cim + CIM_REPRESENTATION_INTERVAL_TICKS <= BigInt(highExact); +} +function cimClearlyBefore(cimFileTime, exactFileTime) { + return BigInt(cimFileTime) + CIM_REPRESENTATION_INTERVAL_TICKS < BigInt(exactFileTime); +} +function cimClearlyAfter(cimFileTime, exactFileTime) { + return BigInt(cimFileTime) > BigInt(exactFileTime); +} +function acceptMember(row, input, held, members2, memberPids, decidedIrrelevant, holdRequired, refuse) { + if (row.startTicks === null || row.creationFileTime === null) { + refuse("process_tree_member_identity_unknown", row.pid); + return; + } + const child = held.get(row.pid); + if (!child) { + holdRequired.add(row.pid); + return; + } + if (!withinCimInterval(child.creationFileTime, row.creationFileTime)) { + refuse("process_tree_generation_mismatch", row.pid); + return; + } + if (child.creationTicks !== row.startTicks) { + refuse("process_tree_member_generation_mismatch", row.pid); + return; + } + memberPids.add(row.pid); + members2.push({ pid: row.pid, parentPid: row.parentPid, creationFileTime: row.creationFileTime, startTicks: row.startTicks }); +} +function resolveWindowsTreeMembership(input) { + const rowsByPid = /* @__PURE__ */ new Map(); + for (const row of input.rows) { + if (rowsByPid.has(row.pid)) return { kind: "refuse", code: "process_tree_snapshot_invalid", pid: row.pid }; + rowsByPid.set(row.pid, row); + } + const members2 = []; + const memberPids = /* @__PURE__ */ new Set(); + const decidedIrrelevant = /* @__PURE__ */ new Set(); + const holdRequired = /* @__PURE__ */ new Set(); + const state = { refusal: null }; + const refuse = (code, pid) => { + if (!state.refusal) state.refusal = { code, pid }; + }; + const rootRow = rowsByPid.get(input.rootPid); + if (rootRow) { + if (input.expectedRootTicks === null) { + return { kind: "refuse", code: "process_tree_root_identity_required", pid: input.rootPid }; + } + if (rootRow.startTicks === null || rootRow.creationFileTime === null) { + return { kind: "refuse", code: "process_tree_root_identity_unknown", pid: input.rootPid }; + } + if (rootRow.startTicks !== input.expectedRootTicks) { + return { kind: "refuse", code: "process_tree_root_identity_changed", pid: input.rootPid }; + } + const heldRoot = input.held.get(input.rootPid); + if (!heldRoot) { + holdRequired.add(input.rootPid); + } else if (heldRoot.creationTicks !== rootRow.startTicks || !withinCimInterval(heldRoot.creationFileTime, rootRow.creationFileTime)) { + return { kind: "refuse", code: "process_tree_root_generation_mismatch", pid: input.rootPid }; + } else { + memberPids.add(rootRow.pid); + members2.push({ pid: rootRow.pid, parentPid: rootRow.parentPid, creationFileTime: rootRow.creationFileTime, startTicks: rootRow.startTicks }); + } + } else if (!input.killed.has(input.rootPid)) { + if (input.expectedRootTicks === null) { + return { kind: "refuse", code: "process_tree_root_identity_required", pid: input.rootPid }; + } + return { kind: "refuse", code: "process_tree_root_exited", pid: input.rootPid }; + } + for (let changed = true; changed && !state.refusal; ) { + changed = false; + for (const row of input.rows) { + if (state.refusal) break; + if (memberPids.has(row.pid) || decidedIrrelevant.has(row.pid)) continue; + const parentPid = row.parentPid; + if (parentPid === row.pid) { + decidedIrrelevant.add(row.pid); + changed = true; + continue; + } + const parentRow = memberPids.has(parentPid) ? rowsByPid.get(parentPid) : void 0; + if (parentRow) { + if (row.creationFileTime === null) { + refuse("process_tree_row_creation_unknown", row.pid); + continue; + } + if (parentRow.creationFileTime === null) { + refuse("process_tree_row_creation_unknown", parentPid); + continue; + } + const parent = input.held.get(parentPid); + if (!parent) { + holdRequired.add(parentPid); + continue; + } + if (cimClearlyBefore(row.creationFileTime, parent.creationFileTime)) { + decidedIrrelevant.add(row.pid); + changed = true; + continue; + } + const child = input.held.get(row.pid); + if (!child) { + holdRequired.add(row.pid); + continue; + } + if (!withinCimInterval(child.creationFileTime, row.creationFileTime) || !withinCimInterval(parent.creationFileTime, parentRow.creationFileTime)) { + refuse("process_tree_generation_mismatch", row.pid); + continue; + } + const childCreation = BigInt(child.creationFileTime); + const parentCreation = BigInt(parent.creationFileTime); + if (childCreation === parentCreation) { + refuse("process_tree_edge_indistinguishable", row.pid); + continue; + } + if (childCreation < parentCreation) { + decidedIrrelevant.add(row.pid); + changed = true; + continue; + } + acceptMember(row, input, input.held, members2, memberPids, decidedIrrelevant, holdRequired, refuse); + if (memberPids.has(row.pid)) changed = true; + continue; + } + const killedParent = input.killed.get(parentPid); + if (killedParent) { + if (row.creationFileTime === null) { + refuse("process_tree_row_creation_unknown", row.pid); + continue; + } + if (cimClearlyBefore(row.creationFileTime, killedParent.creationFileTime)) { + decidedIrrelevant.add(row.pid); + changed = true; + continue; + } + if (killedParent.exitFileTime === null) { + refuse("process_tree_parent_exit_unconfirmed", parentPid); + continue; + } + if (cimClearlyAfter(row.creationFileTime, killedParent.exitFileTime)) { + decidedIrrelevant.add(row.pid); + changed = true; + continue; + } + if (cimIntervalWithin(row.creationFileTime, killedParent.creationFileTime, killedParent.exitFileTime)) { + acceptMember(row, input, input.held, members2, memberPids, decidedIrrelevant, holdRequired, refuse); + if (memberPids.has(row.pid)) changed = true; + continue; + } + const child = input.held.get(row.pid); + if (!child) { + holdRequired.add(row.pid); + continue; + } + if (!withinCimInterval(child.creationFileTime, row.creationFileTime)) { + refuse("process_tree_generation_mismatch", row.pid); + continue; + } + const creation = BigInt(child.creationFileTime); + if (creation >= BigInt(killedParent.creationFileTime) && creation <= BigInt(killedParent.exitFileTime)) { + acceptMember(row, input, input.held, members2, memberPids, decidedIrrelevant, holdRequired, refuse); + if (memberPids.has(row.pid)) changed = true; + } else { + decidedIrrelevant.add(row.pid); + changed = true; + } + continue; + } + if (input.naturalGonePids.has(parentPid)) { + if (row.creationFileTime === null) { + refuse("process_tree_row_creation_unknown", row.pid); + continue; + } + const lastCim = input.naturalGoneCreation.get(parentPid) ?? null; + if (lastCim !== null && cimClearlyBefore(row.creationFileTime, lastCim)) { + decidedIrrelevant.add(row.pid); + changed = true; + continue; + } + refuse("process_tree_parent_unprovable", parentPid); + continue; + } + const parentUndecidedLiveRow = rowsByPid.has(parentPid) && !memberPids.has(parentPid) && !decidedIrrelevant.has(parentPid); + if (parentUndecidedLiveRow || parentPid === input.rootPid) continue; + decidedIrrelevant.add(row.pid); + changed = true; + } + } + if (state.refusal) return { kind: "refuse", code: state.refusal.code, pid: state.refusal.pid }; + if (holdRequired.size > 0) return { kind: "members", members: members2, holdRequired: [...holdRequired] }; + if (members2.length === 0) return { kind: "success" }; + return { kind: "members", members: members2, holdRequired: [] }; +} +var WindowsTreeCleanupError = class extends Error { + code; + memberPids; + constructor(code, message, memberPids = []) { + super(message); + this.name = "WindowsTreeCleanupError"; + this.code = code; + this.memberPids = memberPids; + } +}; +async function terminateWindowsProcessTree(pid, expectedRootTicks, killWaitMs, dependencies) { + if (!Number.isInteger(pid) || pid <= 0) { + throw new WindowsTreeCleanupError("process_tree_invalid_pid", `a positive integer PID is required, got: ${String(pid)}`); + } + const now = dependencies.now ?? Date.now; + const sleep4 = dependencies.sleep ?? ((ms) => new Promise((resolve) => setTimeout(resolve, ms))); + const transaction = await dependencies.open(); + const killed = /* @__PURE__ */ new Map(); + const held = /* @__PURE__ */ new Map(); + const naturalGone = /* @__PURE__ */ new Set(); + const naturalGoneCreation = /* @__PURE__ */ new Map(); + const memberReceipts = /* @__PURE__ */ new Map(); + const killReceipts = []; + const unverifiedCandidates = /* @__PURE__ */ new Set(); + let rounds = 0; + let bodyError = null; + let succeeded = false; + const knownPids = () => [.../* @__PURE__ */ new Set([...memberReceipts.keys(), ...unverifiedCandidates])]; try { - exitCode = await dependencies.kill(pid); - } catch { - commandFailed = true; + const deadline = now() + Math.max(0, killWaitMs); + for (; ; ) { + rounds += 1; + const rows = await transaction.enumerate(); + const resolution = await resolveRound(rows); + if (resolution.kind === "refuse") { + throw new WindowsTreeCleanupError(resolution.code, `${resolution.code}: process tree cleanup refused`, knownPids()); + } + if (resolution.kind === "success") break; + const members2 = resolution.members.map((m) => ({ pid: m.pid, parentPid: m.parentPid, startTicks: m.startTicks, creationFileTime: m.creationFileTime })); + const livePids = new Set(resolution.members.map((m) => m.pid)); + for (const previous of memberReceipts.keys()) { + if (!livePids.has(previous) && !killed.has(previous) && !naturalGone.has(previous)) { + naturalGone.add(previous); + naturalGoneCreation.set(previous, memberReceipts.get(previous).creationFileTime); + } + } + for (const member of members2) { + unverifiedCandidates.delete(member.pid); + if (!memberReceipts.has(member.pid)) { + memberReceipts.set(member.pid, { pid: member.pid, fingerprint: member.startTicks, creationFileTime: member.creationFileTime }); + } + } + const ordered = orderChildrenFirst(members2); + const remaining = Math.max(120, deadline - now()); + const perMemberWait = Math.min(2e3, Math.max(120, Math.floor(remaining / Math.max(1, ordered.length)))); + for (const member of ordered) { + const heldInfo = held.get(member.pid); + const kill = await transaction.kill(member.pid, member.startTicks, perMemberWait); + killReceipts.push({ pid: member.pid, status: kill.status, exitFileTime: kill.exitFileTime }); + if (kill.status === "signaled" || kill.status === "already_exited") { + killed.set(member.pid, { + pid: member.pid, + creationFileTime: heldInfo?.creationFileTime ?? member.creationFileTime, + exitFileTime: kill.exitFileTime + }); + } else if (kill.status === "identity_changed" || kill.status === "not_held") { + naturalGone.add(member.pid); + naturalGoneCreation.set(member.pid, member.creationFileTime); + } + } + if (now() >= deadline) { + throw new WindowsTreeCleanupError( + "process_tree_cleanup_unverified", + `process_tree_cleanup_unverified: ${String(resolution.members.length)} recorded member(s) not provably exited`, + knownPids() + ); + } + await sleep4(20); + } + succeeded = true; + return { + rootPid: pid, + rounds, + members: [...memberReceipts.values()], + kills: killReceipts, + signaled: killReceipts.filter((k) => k.status === "signaled").map((k) => k.pid) + }; + } catch (error2) { + bodyError = error2; + if (error2 instanceof WindowsTreeCleanupError) { + throw error2.memberPids.length > 0 ? error2 : new WindowsTreeCleanupError(error2.code, error2.message, knownPids()); + } + const detail = error2 instanceof Error ? error2.message : String(error2); + throw new WindowsTreeCleanupError("process_tree_transaction_error", `process_tree_transaction_error: ${detail}`, knownPids()); + } finally { + let closeError = null; + try { + await transaction.close(); + } catch (error2) { + closeError = error2; + } + if (closeError !== null) { + const closeText = closeError instanceof Error ? closeError.message : String(closeError); + if (bodyError !== null) { + const original = bodyError instanceof WindowsTreeCleanupError ? new WindowsTreeCleanupError(bodyError.code, `${bodyError.message}; process_tree_close_failed: ${closeText}`, bodyError.memberPids.length > 0 ? bodyError.memberPids : knownPids()) : new WindowsTreeCleanupError("process_tree_transaction_error", `process_tree_transaction_error: ${String(bodyError instanceof Error ? bodyError.message : bodyError)}; process_tree_close_failed: ${closeText}`, knownPids()); + throw original; + } + throw new WindowsTreeCleanupError("process_tree_close_failed", `process_tree_close_failed: ${closeText}`, knownPids()); + } + } + async function resolveRound(rows) { + let workingRows = [...rows]; + const resolve = () => resolveWindowsTreeMembership({ + rows: workingRows, + rootPid: pid, + expectedRootTicks, + held, + killed, + naturalGonePids: naturalGone, + naturalGoneCreation + }); + let resolution = resolve(); + for (let attempt = 0; attempt < 6 && resolution.kind === "members" && resolution.holdRequired.length > 0; attempt += 1) { + const requested = [...new Set(resolution.holdRequired)]; + for (const candidate of resolution.holdRequired) unverifiedCandidates.add(candidate); + const outcomes = await transaction.hold(requested); + if (outcomes.length !== requested.length) { + throw new WindowsTreeCleanupError( + "process_tree_hold_invalid", + `process_tree_hold_invalid: ${String(outcomes.length)} responses for ${String(requested.length)} requested pids`, + knownPids() + ); + } + const seen = /* @__PURE__ */ new Set(); + for (const outcome of outcomes) { + if (seen.has(outcome.pid)) { + throw new WindowsTreeCleanupError("process_tree_hold_invalid", `process_tree_hold_invalid: duplicate response for pid ${String(outcome.pid)}`, knownPids()); + } + seen.add(outcome.pid); + if (!requested.includes(outcome.pid)) { + throw new WindowsTreeCleanupError("process_tree_hold_invalid", `process_tree_hold_invalid: unexpected pid ${String(outcome.pid)} in hold response`, knownPids()); + } + if (outcome.status === "ok") { + held.set(outcome.pid, { pid: outcome.pid, creationFileTime: outcome.creationFileTime, creationTicks: outcome.creationTicks }); + } else if (outcome.status === "absent") { + throw new WindowsTreeCleanupError( + "process_tree_candidate_absent", + `process_tree_candidate_absent: candidate pid ${String(outcome.pid)} vanished between enumeration and hold; its subtree cannot be attributed`, + [.../* @__PURE__ */ new Set([...requested, ...knownPids()])] + ); + } + } + resolution = resolve(); + } + if (resolution.kind === "members" && resolution.holdRequired.length > 0) { + for (const candidate of resolution.holdRequired) unverifiedCandidates.add(candidate); + const roundMemberPids = resolution.members.map((m) => m.pid); + throw new WindowsTreeCleanupError( + "process_tree_hold_unresolved", + `process_tree_hold_unresolved: cannot prove identity/ownership for pids ${resolution.holdRequired.join(",")}`, + [.../* @__PURE__ */ new Set([...roundMemberPids, ...knownPids()])] + ); + } + return resolution; } - const deadline = performance.now() + Math.max(0, killWaitMs); - for (; ; ) { - const after = await inspect(); - if (after.every((verdict2) => verdict2.state === "exited")) return; - if (performance.now() >= deadline) { - const reason = commandFailed ? "command_error" : exitCode === null ? "timeout" : `exit_${exitCode}`; - throw new Error(`process_tree_cleanup_unverified: taskkill ${reason}; recorded executors remain alive`); +} +function orderChildrenFirst(members2) { + const remaining = [...members2]; + const ordered = []; + let progress = true; + while (remaining.length > 0 && progress) { + progress = false; + for (let index = remaining.length - 1; index >= 0; index -= 1) { + const member = remaining[index]; + const hasPendingChild = remaining.some((other) => other !== member && other.parentPid === member.pid); + if (!hasPendingChild) { + ordered.push(member); + remaining.splice(index, 1); + progress = true; + } + } + } + const leftovers = remaining.sort((a, b) => BigInt(a.creationFileTime) > BigInt(b.creationFileTime) ? -1 : 1); + return [...ordered, ...leftovers]; +} +async function verifyWindowsRecordedTreeClaimants(targets, dependencies) { + const transaction = await dependencies.open(); + let bodyError = null; + const targetPids = targets.map((t) => t.pid).filter((p) => Number.isInteger(p) && p > 0); + try { + const rows = await transaction.enumerate(); + const live = /* @__PURE__ */ new Set(); + for (const row of rows) live.add(row.pid); + const claimantPids = /* @__PURE__ */ new Set(); + const foreignPids = /* @__PURE__ */ new Set(); + const recorded = /* @__PURE__ */ new Set(); + for (const target of targets) { + if (!Number.isInteger(target.pid) || target.pid <= 0) continue; + recorded.add(target.pid); + if (!live.has(target.pid)) continue; + if (target.fingerprint === null) { + claimantPids.add(target.pid); + continue; + } + const outcome = (await transaction.hold([target.pid]))[0]; + if (outcome === void 0) throw new WindowsTreeCleanupError("process_tree_scan_invalid", "recorded-tree scan received no hold outcome"); + if (outcome.status === "ok") { + if (outcome.creationTicks === target.fingerprint) claimantPids.add(target.pid); + else foreignPids.add(target.pid); + } else if (outcome.status === "absent") { + continue; + } else { + claimantPids.add(target.pid); + } + } + for (let changed = true; changed; ) { + changed = false; + for (const row of rows) { + if (claimantPids.has(row.pid) || foreignPids.has(row.pid)) continue; + const parentClaimed = claimantPids.has(row.parentPid); + const parentRecordedDead = recorded.has(row.parentPid) && !live.has(row.parentPid) && !foreignPids.has(row.parentPid); + if (parentClaimed || parentRecordedDead) { + claimantPids.add(row.pid); + changed = true; + } + } + } + const liveList = [...claimantPids].sort((a, b) => a - b); + return { clean: liveList.length === 0, livePids: liveList }; + } catch (error2) { + bodyError = error2; + throw error2; + } finally { + let closeError = null; + try { + await transaction.close(); + } catch (error2) { + closeError = error2; + } + if (closeError !== null) { + const closeText = closeError instanceof Error ? closeError.message : String(closeError); + if (bodyError !== null) { + const originalText = bodyError instanceof Error ? bodyError.message : String(bodyError); + throw new WindowsTreeCleanupError("process_tree_scan_failed", `process_tree_scan_failed: ${originalText}; process_tree_close_failed: ${closeText}`, targetPids); + } + throw new WindowsTreeCleanupError("process_tree_close_failed", `process_tree_close_failed: ${closeText}`, targetPids); } - await new Promise((resolve) => setTimeout(resolve, 20)); } } -function windowsTreeCaptureScript(pid) { +function windowsTreeTransactionScript() { return [ "$ErrorActionPreference = 'Stop'", - "$all = @(Get-CimInstance Win32_Process -ErrorAction Stop)", - `$selected = [System.Collections.Generic.HashSet[int]]::new(); [void]$selected.Add(${pid})`, - "do { $changed = $false; foreach ($item in $all) { if ($selected.Contains([int]$item.ParentProcessId)) { if ($selected.Add([int]$item.ProcessId)) { $changed = $true } } } } while ($changed)", - "'bridge_tree_snapshot_v1'", - "foreach ($candidate in $selected) { $item = Get-Process -Id $candidate -ErrorAction SilentlyContinue; if ($null -eq $item) { '{0}|absent' -f $candidate } else { '{0}|{1}' -f $candidate, $item.StartTime.Ticks } }" - ].join("; "); -} -function parseWindowsTreeCapture(stdout, pid) { - const lines = stdout.trim().split(/\r?\n/u); - if (lines.shift() !== "bridge_tree_snapshot_v1") throw new Error("process_tree_snapshot_invalid: protocol missing"); - const capturedAt = (/* @__PURE__ */ new Date()).toISOString(); - const requests = []; - const seen = /* @__PURE__ */ new Set(); - for (const line of lines) { - const match = /^(\d+)\|(\d+|absent)$/u.exec(line.trim()); - const candidate = Number(match?.[1]); - if (!match || !Number.isSafeInteger(candidate) || candidate <= 0 || seen.has(candidate)) { - throw new Error("process_tree_snapshot_invalid: malformed identity"); - } - seen.add(candidate); - requests.push({ pid: candidate, identity: match[2] === "absent" ? null : { - pid: candidate, - fingerprint: match[2], - fingerprint_precision: "exact", - identity_version: 1, - platform: "win32", - captured_at: capturedAt - } }); - } - if (!seen.has(pid)) throw new Error("process_tree_snapshot_invalid: root missing"); - return requests; + "$ProgressPreference = 'SilentlyContinue'", + "Add-Type -TypeDefinition @'", + "using System;", + "using System.Collections.Generic;", + "using System.Runtime.InteropServices;", + "public static class BridgeTreeTx {", + ' [DllImport("kernel32.dll", SetLastError = true)] static extern IntPtr OpenProcess(uint access, bool inherit, int pid);', + ' [DllImport("kernel32.dll", SetLastError = true)] static extern bool GetProcessTimes(IntPtr h, out long created, out long exited, out long kernel, out long user);', + ' [DllImport("kernel32.dll", SetLastError = true)] static extern bool TerminateProcess(IntPtr h, uint exitCode);', + ' [DllImport("kernel32.dll", SetLastError = true)] static extern uint WaitForSingleObject(IntPtr h, uint milliseconds);', + ' [DllImport("kernel32.dll", SetLastError = true)] static extern bool CloseHandle(IntPtr h);', + " public static readonly Dictionary Held = new Dictionary();", + " public static string Hold(int pid) {", + " IntPtr existing;", + ' if (Held.TryGetValue(pid, out existing) && existing != IntPtr.Zero) { return "already_held|"; }', + " IntPtr h = OpenProcess(0x00101001u, false, pid);", + ' if (h == IntPtr.Zero) { int err = Marshal.GetLastWin32Error(); return (err == 87 ? "absent|" : (err == 5 ? "denied|" : "error|")) + err.ToString(); }', + " long c, e, k, u;", + ' if (!GetProcessTimes(h, out c, out e, out k, out u)) { int err = Marshal.GetLastWin32Error(); CloseHandle(h); return "error|" + err.ToString(); }', + " Held[pid] = h;", + ' return "ok|" + c.ToString() + "|" + DateTime.FromFileTime(c).Ticks.ToString();', + " }", + " public static string Kill(int pid, string expectTicks, int waitMs) {", + " IntPtr h;", + ' if (!Held.TryGetValue(pid, out h) || h == IntPtr.Zero) { return "not_held|"; }', + " long c, e, k, u;", + ' if (!GetProcessTimes(h, out c, out e, out k, out u)) { return "error|" + Marshal.GetLastWin32Error().ToString(); }', + " string current = DateTime.FromFileTime(c).Ticks.ToString();", + ' if (current != expectTicks) { return "identity_changed|" + current; }', + ' if (e != 0) { return "already_exited|" + e.ToString(); }', + ' if (!TerminateProcess(h, 1)) { return "error|" + Marshal.GetLastWin32Error().ToString(); }', + " uint wait = WaitForSingleObject(h, (uint)waitMs);", + ' if (!GetProcessTimes(h, out c, out e, out k, out u)) { return "error|" + Marshal.GetLastWin32Error().ToString(); }', + ' if (wait != 0) { return "wait_timeout|" + e.ToString(); }', + ' return "signaled|" + e.ToString();', + " }", + " public static string CloseAll() {", + " int closed = 0;", + " int failed = 0;", + " foreach (KeyValuePair entry in Held) { if (entry.Value != IntPtr.Zero) { if (CloseHandle(entry.Value)) { closed++; } else { failed++; } } }", + " Held.Clear();", + ' return closed.ToString() + "|" + failed.ToString();', + " }", + "}", + "'@", + "[Console]::Out.WriteLine('READY'); [Console]::Out.Flush()", + "while ($true) {", + " $line = [Console]::In.ReadLine()", + " if ($null -eq $line) { [void][BridgeTreeTx]::CloseAll(); exit 0 }", + " $parts = $line.Trim().Split(' ')", + " if ($parts[0] -eq 'enum') {", + " $all = @(Get-CimInstance Win32_Process -ErrorAction Stop)", + " [Console]::Out.WriteLine('ENUM_BEGIN')", + " $ids = @(); foreach ($p in $all) { $ids += [int]$p.ProcessId }", + " $byId = @{}", + " if ($ids.Count -gt 0) { foreach ($g in (Get-Process -Id $ids -ErrorAction SilentlyContinue)) { $byId[[int]$g.Id] = $g } }", + " foreach ($p in $all) {", + " $ticks = 'unknown'", + " $cim = 'unknown'", + " if ($null -ne $p.CreationDate) { $cim = [string]$p.CreationDate.ToFileTimeUtc() }", + " $g = $byId[[int]$p.ProcessId]", + " if ($null -ne $g) { try { $st = $g.StartTime; if ($null -ne $st) { $ticks = [string]$st.Ticks } } catch { $ticks = 'unknown' } }", + " if ([string]::IsNullOrEmpty($ticks)) { $ticks = 'unknown' }", + " [Console]::Out.WriteLine(('R {0} {1} {2} {3}' -f [int]$p.ProcessId, [int]$p.ParentProcessId, $cim, $ticks))", + " }", + " [Console]::Out.WriteLine('ENUM_END'); [Console]::Out.Flush()", + " } elseif ($parts[0] -eq 'hold') {", + " for ($i = 1; $i -lt $parts.Count; $i++) {", + " $hp = 0", + " if (-not [int]::TryParse($parts[$i], [ref]$hp)) { [Console]::Out.WriteLine('H 0 error 22'); continue }", + " $r = [BridgeTreeTx]::Hold($hp).Split('|')", + " if ($r[0] -eq 'already_held') { [Console]::Out.WriteLine(('H {0} held' -f $hp)) }", + " elseif ($r[0] -eq 'ok') { [Console]::Out.WriteLine(('H {0} ok {1} {2}' -f $hp, $r[1], $r[2])) }", + " else { [Console]::Out.WriteLine(('H {0} {1} {2}' -f $hp, $r[0], $r[1])) }", + " }", + " [Console]::Out.WriteLine('HOLD_END'); [Console]::Out.Flush()", + " } elseif ($parts[0] -eq 'kill') {", + " $kpid = [int]$parts[1]; $kticks = [string]$parts[2]; $kwait = [int]$parts[3]", + " $r = [BridgeTreeTx]::Kill($kpid, $kticks, $kwait).Split('|')", + " if ($r[0] -eq 'signaled' -or $r[0] -eq 'already_exited') { [Console]::Out.WriteLine(('K {0} {1} {2}' -f $kpid, $r[0], $r[1])) }", + " elseif ($r[0] -eq 'identity_changed') { [Console]::Out.WriteLine(('K {0} identity_changed 0' -f $kpid)) }", + " elseif ($r[0] -eq 'wait_timeout') { [Console]::Out.WriteLine(('K {0} wait_timeout {1}' -f $kpid, $r[1])) }", + " elseif ($r[0] -eq 'not_held') { [Console]::Out.WriteLine(('K {0} not_held 0' -f $kpid)) }", + " else { [Console]::Out.WriteLine(('K {0} error {1}' -f $kpid, $r[1])) }", + " [Console]::Out.WriteLine('KILL_END'); [Console]::Out.Flush()", + " } elseif ($parts[0] -eq 'close') {", + " $r = [BridgeTreeTx]::CloseAll().Split('|')", + " [Console]::Out.WriteLine(('CLOSED {0} {1}' -f $r[0], $r[1])); [Console]::Out.Flush()", + " exit 0", + " } else {", + " [Console]::Out.WriteLine('FATAL unknown_command'); [Console]::Out.Flush(); exit 1", + " }", + "}" + ].join("\n"); +} +var TRANSPORT_MAX_LINE_BYTES = 1048576; +var TRANSPORT_MAX_BUFFERED_BYTES = 8388608; +var TRANSPORT_MAX_BUFFERED_LINES = 65536; +var TRANSPORT_MAX_ENUM_ROWS = 4096; +var TRANSPORT_TOTAL_SESSION_MS_DEFAULT = 12e4; +var PowerShellTreeTransaction = class _PowerShellTreeTransaction { + #child; + #bounds; + #lines = []; + #waiters = []; + #exitWaiters = []; + #closed = false; + #bufferedBytes = 0; + #sessionDeadline; + #exitCode = null; + #exitSignal = null; + /** True once the helper's own exit is expected (receipt processed or + * fallback initiated); its close event is then not a transport failure. */ + #closeHandled = false; + #failure = null; + /** Test seam: attach to an externally spawned/controlled child so the + * bounded-stream behaviours are testable with fake streams (no OS). */ + static attach(child, bounds = {}) { + return new _PowerShellTreeTransaction(child, bounds); + } + constructor(child, bounds = {}) { + this.#child = child; + this.#bounds = { + maxLineBytes: bounds.maxLineBytes ?? TRANSPORT_MAX_LINE_BYTES, + maxBufferedBytes: bounds.maxBufferedBytes ?? TRANSPORT_MAX_BUFFERED_BYTES, + maxBufferedLines: bounds.maxBufferedLines ?? TRANSPORT_MAX_BUFFERED_LINES, + maxEnumRows: bounds.maxEnumRows ?? TRANSPORT_MAX_ENUM_ROWS, + totalSessionMs: bounds.totalSessionMs ?? TRANSPORT_TOTAL_SESSION_MS_DEFAULT + }; + this.#sessionDeadline = Date.now() + this.#bounds.totalSessionMs; + child.stdout.setEncoding("utf8"); + let buffer = ""; + child.stdout.on("data", (chunk) => { + if (this.#failure) { + buffer = ""; + return; + } + buffer += chunk; + let newline = buffer.indexOf("\n"); + while (newline >= 0) { + const line = buffer.slice(0, newline).replace(/\r$/u, ""); + buffer = buffer.slice(newline + 1); + const lineBytes = Buffer.byteLength(line, "utf8"); + if (lineBytes > this.#bounds.maxLineBytes) { + this.#failAndDrain(); + this.#fail(new Error(`process_tree_transport_overflow: line of ${String(lineBytes)} bytes exceeds ${String(this.#bounds.maxLineBytes)}`)); + return; + } + this.#bufferedBytes += lineBytes; + if (this.#bufferedBytes > this.#bounds.maxBufferedBytes || this.#lines.length >= this.#bounds.maxBufferedLines) { + this.#failAndDrain(); + this.#fail(new Error(`process_tree_transport_overflow: buffered stream exceeded its byte or line budget (${String(this.#bufferedBytes)} bytes, ${String(this.#lines.length)} lines)`)); + return; + } + const waiter = this.#waiters.shift(); + if (waiter) { + clearTimeout(waiter.timer); + waiter.resolve(line); + this.#bufferedBytes -= lineBytes; + } else { + this.#lines.push(line); + } + newline = buffer.indexOf("\n"); + } + if (Buffer.byteLength(buffer, "utf8") > this.#bounds.maxLineBytes) { + this.#failAndDrain(); + this.#fail(new Error(`process_tree_transport_overflow: unterminated line exceeds ${String(this.#bounds.maxLineBytes)} bytes`)); + } + }); + child.stderr.setEncoding("utf8"); + child.stderr.on("data", (chunk) => { + void chunk; + this.#fail(new Error("process_tree_transport_stderr")); + }); + child.on("error", (error2) => this.#fail(new Error(`process_tree_transport_error: ${String(error2.message)}`))); + child.on("close", (code, signal) => { + this.#closed = true; + this.#exitCode = code; + this.#exitSignal = signal; + if (!this.#closeHandled) this.#fail(new Error("process_tree_transport_closed")); + for (const waiter of this.#exitWaiters.splice(0)) waiter(); + }); + } + /** Clears buffered receive state; used when a budget trips so a failed + * transport neither accumulates nor serves stale lines afterwards. */ + #failAndDrain() { + for (const line of this.#lines) this.#bufferedBytes -= Buffer.byteLength(line, "utf8"); + this.#lines = []; + } + #fail(error2) { + if (this.#failure) return; + this.#failure = error2; + for (const waiter of this.#waiters.splice(0)) { + clearTimeout(waiter.timer); + waiter.reject(error2); + } + } + #nextLine(timeoutMs, what) { + if (this.#failure) return Promise.reject(this.#failure); + const remainingSession = this.#sessionDeadline - Date.now(); + if (remainingSession <= 0) { + const error2 = new Error(`process_tree_transport_session_timeout: ${what}`); + this.#failAndDrain(); + this.#fail(error2); + return Promise.reject(error2); + } + const buffered = this.#lines.shift(); + if (buffered !== void 0) { + this.#bufferedBytes -= Buffer.byteLength(buffered, "utf8"); + return Promise.resolve(buffered); + } + if (this.#closed) return Promise.reject(new Error("process_tree_transport_closed")); + const boundedTimeout = Math.min(timeoutMs, remainingSession); + return new Promise((resolve, reject) => { + const timer = setTimeout(() => { + const index = this.#waiters.findIndex((w) => w.resolve === resolve); + if (index >= 0) this.#waiters.splice(index, 1); + const reason = remainingSession < timeoutMs ? "process_tree_transport_session_timeout" : "process_tree_transport_timeout"; + reject(new Error(`${reason}: ${what}`)); + this.#fail(new Error(`${reason}: ${what}`)); + }, boundedTimeout); + if (typeof timer.unref === "function") timer.unref(); + this.#waiters.push({ resolve, reject, timer }); + }); + } + static async start(timeoutMs = 2e4) { + const encoded = Buffer.from(windowsTreeTransactionScript(), "utf16le").toString("base64"); + const child = spawn3("powershell.exe", ["-NoProfile", "-NonInteractive", "-ExecutionPolicy", "Bypass", "-EncodedCommand", encoded], { + shell: false, + windowsHide: true, + stdio: ["pipe", "pipe", "pipe"] + }); + const transaction = _PowerShellTreeTransaction.attach(child); + try { + const ready = await transaction.#nextLine(timeoutMs, "helper startup"); + if (ready !== "READY") throw new Error("process_tree_transport_protocol: startup marker missing"); + return transaction; + } catch (error2) { + transaction.#closeHandled = true; + try { + child.kill("SIGKILL"); + } catch { + } + const exited = await transaction.#waitForExit(3e3); + if (!exited) { + throw new Error(`${error2 instanceof Error ? error2.message : String(error2)}; process_tree_close_failed: startup helper did not exit within bounds`); + } + throw error2; + } + } + async enumerate() { + this.#child.stdin.write("enum\n"); + const begin = await this.#nextLine(2e4, "enum begin"); + if (begin !== "ENUM_BEGIN") throw new Error("process_tree_snapshot_invalid: protocol missing"); + const rows = []; + for (; ; ) { + const line = await this.#nextLine(2e4, "enum row"); + if (line === "ENUM_END") break; + const match = /^R (\d{1,12}) (\d{1,12}) (\d{1,25}|unknown) (\d{1,25}|unknown)$/u.exec(line); + if (!match) throw new Error("process_tree_snapshot_invalid: malformed identity"); + const pid = Number(match[1]); + const parentPid = Number(match[2]); + if (!Number.isSafeInteger(pid) || pid <= 0 || !Number.isSafeInteger(parentPid) || parentPid < 0) continue; + if (rows.length >= this.#bounds.maxEnumRows) { + throw new Error(`process_tree_snapshot_invalid: enumeration exceeded ${String(this.#bounds.maxEnumRows)} rows`); + } + rows.push({ pid, parentPid, creationFileTime: match[3] === "unknown" ? null : match[3], startTicks: match[4] === "unknown" ? null : match[4] }); + } + return rows; + } + async hold(pids) { + if (pids.length === 0) return []; + this.#child.stdin.write(`hold ${pids.join(" ")} +`); + const outcomes = []; + for (; ; ) { + const line = await this.#nextLine(2e4, "hold response"); + if (line === "HOLD_END") break; + const match = /^H (\d{1,12}) (ok (\d{1,25}) (\d{1,25})|held|absent (\d{1,6})|denied (\d{1,6})|error (\d{1,6}))$/u.exec(line); + if (!match) throw new Error("process_tree_hold_invalid: malformed response"); + const pid = Number(match[1]); + if (match[2].startsWith("ok")) { + outcomes.push({ pid, status: "ok", creationFileTime: match[3], creationTicks: match[4] }); + } else if (match[2] === "held") { + outcomes.push({ pid, status: "error", detail: "already_held" }); + } else if (match[2].startsWith("absent")) { + outcomes.push({ pid, status: "absent", error: Number(match[5] ?? 0) }); + } else if (match[2].startsWith("denied")) { + outcomes.push({ pid, status: "denied", error: Number(match[6] ?? 0) }); + } else { + outcomes.push({ pid, status: "error", error: Number(match[7] ?? 0) }); + } + } + return outcomes; + } + async kill(pid, expectedTicks, waitMs) { + this.#child.stdin.write(`kill ${String(pid)} ${expectedTicks} ${String(Math.max(0, Math.floor(waitMs)))} +`); + const line = await this.#nextLine(waitMs + 25e3, "kill response"); + const end = await this.#nextLine(5e3, "kill terminator"); + if (end !== "KILL_END") throw new Error("process_tree_kill_invalid: terminator missing"); + const match = /^K (\d{1,12}) (signaled|identity_changed|not_held|wait_timeout|already_exited|error) (\d{1,25})$/u.exec(line); + if (!match || Number(match[1]) !== pid) throw new Error("process_tree_kill_invalid: malformed response"); + const status = match[2]; + const value = match[3]; + return { + status, + exitFileTime: status === "signaled" || status === "already_exited" || status === "wait_timeout" ? value === "0" ? null : value : null, + detail: status === "identity_changed" ? value : void 0 + }; + } + async close() { + let failure2 = this.#failure; + if (!failure2 && !this.#closed) { + try { + this.#child.stdin.write("close\n"); + const ack = await this.#nextLine(1e4, "close acknowledgement"); + const match = /^CLOSED (\d{1,9}) (\d{1,9})$/u.exec(ack); + if (!match) throw new Error(`process_tree_close_invalid: unexpected close receipt ${JSON.stringify(ack.slice(0, 60))}`); + const failedHandles = Number(match[2]); + if (failedHandles > 0) throw new Error(`process_tree_close_failed: ${String(failedHandles)} CloseHandle call(s) failed`); + this.#closeHandled = true; + } catch (error2) { + failure2 = error2 instanceof Error ? error2 : new Error(String(error2)); + this.#closeHandled = true; + } + } + const exited = await this.#waitForExit(5e3); + if (!exited) { + this.#closeHandled = true; + try { + this.#child.kill("SIGKILL"); + } catch { + } + try { + this.#child.stdin.end(); + } catch { + } + const exitedAfterKill = await this.#waitForExit(3e3); + if (!exitedAfterKill) { + throw new Error(failure2 ? `${failure2.message}; process_tree_close_failed: helper did not exit within bounds` : "process_tree_close_failed: helper did not exit within bounds"); + } + throw new Error(failure2 ? `${failure2.message}; process_tree_close_forced: helper exited only after a forced kill` : "process_tree_close_forced: helper exited only after a forced kill"); + } + if (this.#exitCode !== 0 || this.#exitSignal !== null) { + const abnormal = `process_tree_close_abnormal: helper exited with code ${String(this.#exitCode)}${this.#exitSignal ? `, signal ${this.#exitSignal}` : ""}`; + throw new Error(failure2 ? `${failure2.message}; ${abnormal}` : abnormal); + } + if (failure2) throw failure2; + if (this.#failure) throw this.#failure; + } + #waitForExit(timeoutMs) { + if (this.#closed) return Promise.resolve(true); + return new Promise((resolve) => { + let settled = false; + const finish = (value) => { + if (settled) return; + settled = true; + clearTimeout(timer); + const index = this.#exitWaiters.indexOf(waiter); + if (index >= 0) this.#exitWaiters.splice(index, 1); + resolve(value); + }; + const waiter = () => finish(true); + const timer = setTimeout(() => finish(false), timeoutMs); + if (typeof timer.unref === "function") timer.unref(); + this.#exitWaiters.push(waiter); + }); + } +}; +async function openPowerShellTreeTransaction() { + return PowerShellTreeTransaction.start(); } // src/adapters/process-spawn.ts -function appendBounded(current, chunk, maxBytes) { - if (!(maxBytes > 0)) return { value: current.value + chunk, truncated: current.truncated }; - const remaining = maxBytes - Buffer.byteLength(current.value, "utf8"); - if (remaining <= 0) return chunk ? { value: current.value, truncated: true } : current; - const bytes = Buffer.from(chunk, "utf8"); - if (bytes.length <= remaining) { - return { value: current.value + chunk, truncated: current.truncated }; - } - return { - value: current.value + bytes.subarray(0, remaining).toString("utf8"), - truncated: true - }; -} function isProcessRunning(pid) { if (!Number.isInteger(pid) || pid <= 0) return false; try { @@ -23984,27 +24814,14 @@ var terminateProcessTree = async (pid, options = {}) => { throw new Error(`A positive integer PID is required, got: ${pid}`); } if (process.platform === "win32") { - const probe = createPlatformProbe(); - await terminateWindowsProcessTree(pid, options.killWaitMs ?? 2e3, { - capture: async (root) => { - let snapshot; - try { - snapshot = await runCommand("powershell.exe", ["-NoProfile", "-NonInteractive", "-Command", windowsTreeCaptureScript(root)], { - timeoutMs: 5e3, - maxOutputBytes: 128e3 - }); - } catch { - throw new Error("process_tree_snapshot_unverified: query failed"); - } - if (snapshot.code !== 0 || snapshot.stdoutTruncated || snapshot.stderr.length > 0) { - throw new Error("process_tree_snapshot_unverified: query failed or incomplete"); - } - return parseWindowsTreeCapture(snapshot.stdout, root); - }, - probe: (requests) => probe.probe(requests), - kill: async (root) => (await runCommand("taskkill", ["/PID", String(root), "/T", "/F"], { timeoutMs: 15e3 })).code + const expected = options.expectedRootFingerprint ?? null; + if (expected === null) { + throw new Error("process_tree_root_identity_required: Windows process-tree termination requires a caller-bound root identity"); + } + const receipt = await terminateWindowsProcessTree(pid, expected, options.killWaitMs ?? 2e3, { + open: openPowerShellTreeTransaction }); - return { pid, signal: "SIGKILL", verified: true }; + return { pid, signal: "SIGKILL", verified: true, windows: receipt }; } signalProcessTree(pid, "SIGTERM"); if (await waitForProcessTreeExit(pid, options.graceMs ?? 500)) { @@ -24016,47 +24833,39 @@ var terminateProcessTree = async (pid, options = {}) => { } return { pid, signal: "SIGKILL", verified: true }; }; -function runCommand(command, args, options) { - return new Promise((resolve, reject) => { - const child = spawn3(command, args, { - shell: false, - windowsHide: true, - stdio: ["ignore", "pipe", "pipe"] - }); - let stdout = { value: "", truncated: false }; - let stderr = { value: "", truncated: false }; - child.stdout?.setEncoding("utf8"); - child.stderr?.setEncoding("utf8"); - child.stdout?.on("data", (chunk) => { - stdout = appendBounded(stdout, chunk, options.maxOutputBytes ?? 1e6); - }); - child.stderr?.on("data", (chunk) => { - stderr = appendBounded(stderr, chunk, options.maxOutputBytes ?? 1e6); - }); - let timedOut = false; - const timer = options.timeoutMs && options.timeoutMs > 0 ? setTimeout(() => { - timedOut = true; - try { - child.kill("SIGKILL"); - } catch { - } - }, options.timeoutMs) : null; - timer?.unref(); - child.on("error", (error2) => { - if (timer) clearTimeout(timer); - reject(error2); - }); - child.on("close", (code, signal) => { - if (timer) clearTimeout(timer); - resolve({ - code: timedOut ? null : code, - signal: timedOut ? "SIGKILL" : signal, - stdout: stdout.value, - stderr: stderr.value, - stdoutTruncated: stdout.truncated - }); - }); - }); +async function verifyProcessTreeClaimantsClean(targets) { + if (process.platform === "win32") { + return verifyWindowsRecordedTreeClaimants(targets, { open: openPowerShellTreeTransaction }); + } + const live = targets.filter((t) => Number.isInteger(t.pid) && t.pid > 0 && isProcessTreeRunning(t.pid)).map((t) => t.pid); + return { clean: live.length === 0, livePids: live }; +} +function childStillOnSpawnGeneration(child) { + if (typeof child.pid !== "number" || !Number.isInteger(child.pid) || child.pid <= 0) return false; + if (child.exitSeen === true) return false; + return child.exitCode === null && child.signalCode === null; +} +async function captureSpawnGenerationIdentity(child, probe, options = {}) { + const pid = child.pid; + if (typeof pid !== "number" || !Number.isInteger(pid) || pid <= 0) return null; + const read = async () => { + let identity; + try { + identity = await probe.identityOf(pid); + } catch { + return null; + } + if (!childStillOnSpawnGeneration(child)) return null; + if (typeof identity.fingerprint !== "string" || identity.fingerprint.length === 0) return null; + if (identity.fingerprint_precision !== "exact") return null; + if (child.pid !== pid) return null; + return { pid, fingerprint: identity.fingerprint }; + }; + const first = await read(); + if (first !== null) return first; + await new Promise((resolve) => setTimeout(resolve, options.retryDelayMs ?? 250)); + if (!childStillOnSpawnGeneration(child)) return null; + return read(); } function signalProcessTree(pid, signal) { try { @@ -24383,6 +25192,7 @@ var BridgeTaskManager = class _BridgeTaskManager { #spawnWorker; #isProcessRunning; #terminateProcessTree; + #verifyTreeClean; #probe; #now; #dataRoot; @@ -24401,6 +25211,7 @@ var BridgeTaskManager = class _BridgeTaskManager { this.#spawnWorker = options.spawnWorker ?? defaultSpawnWorker; this.#isProcessRunning = options.isProcessRunning ?? isProcessRunning; this.#terminateProcessTree = options.terminateProcessTree ?? terminateProcessTree; + this.#verifyTreeClean = options.verifyTreeClean ?? verifyProcessTreeClaimantsClean; this.#probe = options.probe ?? null; this.#now = options.now ?? (() => /* @__PURE__ */ new Date()); this.#dataRoot = options.store.dataRoot; @@ -24537,6 +25348,17 @@ var BridgeTaskManager = class _BridgeTaskManager { } return; } + const failureRecordState = this.#cleanupFailureRecordState(taskId, status.attempt); + if (failureRecordState !== "none") { + const occurrence = this.#counters.record("terminal_cleanup_latched", failureRecordState); + if (this.#counters.shouldEmit("terminal_cleanup_latched", failureRecordState)) { + this.#store.appendEvent(taskId, "cleanup_unverified", "A recorded tree-cleanup failure with known members keeps the terminal task occupied; cancel runs the recorded-tree scan", { + record_state: failureRecordState, + count: occurrence.count + }, this.#now().toISOString()); + } + return; + } await this.#exclusive(async () => { const current = this.#safeStatus(taskId); if (!current || !isTerminalStatus(current.status) || current.cleanup_unverified !== true) return; @@ -24617,8 +25439,145 @@ var BridgeTaskManager = class _BridgeTaskManager { } catch { } } + /** Latches a failed termination (ADR-009): occupancy is retained via + * cleanup_unverified and the member pids observed by the failed + * transaction are persisted so later claimant scans cover them. The + * complete member set is persisted structurally (cleanup-failure.json); + * the bounded [cleanup-members: …] marker in the status text is a durable + * fallback that is only trusted when complete — a truncated marker (or a + * missing/corrupt record) fails later releases closed instead of silently + * narrowing the scan (review-round-1 R3 + addendum). */ + #recordCleanupFailure(taskId, attempt, error2) { + const message = boundedErrorMessage(error2, 300); + const freshPids = Array.isArray(error2.memberPids) ? error2.memberPids.filter((pid) => Number.isSafeInteger(pid) && pid > 0) : []; + let metaPids = []; + let metaKnownIncomplete = false; + try { + const previous = this.#store.readAttemptMeta(taskId, attempt, "cleanup-failure.json"); + if (previous !== null) { + metaPids = this.#validMemberPids(previous.member_pids); + metaKnownIncomplete = previous.complete === false; + } + } catch { + } + const priorMarker = this.#parseCleanupMembersMarker(this.#safeStatus(taskId)?.error); + const priorMarkerIncomplete = priorMarker !== null && !priorMarker.complete; + const previousPids = [.../* @__PURE__ */ new Set([...metaPids, ...priorMarker?.pids ?? []])]; + const mergedPids = [.../* @__PURE__ */ new Set([...previousPids, ...freshPids])]; + const joined = mergedPids.join(","); + const fitsWithoutTruncation = joined.length <= 1600 && !(metaKnownIncomplete || priorMarkerIncomplete); + const memberMarker = mergedPids.length > 0 ? fitsWithoutTruncation ? ` [cleanup-members: ${joined}]` : ` [cleanup-members: ${mergedPids.slice(0, 200).join(",")},\u2026]` : metaKnownIncomplete || priorMarkerIncomplete ? " [cleanup-members: ,\u2026]" : ""; + const recordComplete = fitsWithoutTruncation; + try { + this.#store.writeAttemptMeta(taskId, attempt, "cleanup-failure.json", { + member_pids: mergedPids, + complete: recordComplete, + last_error: message + }); + } catch { + } + try { + this.#store.writeStatus(taskId, { + cleanup_unverified: true, + error: `cancellation could not be verified: ${boundedErrorMessage(error2, 200)}${memberMarker}` + }); + this.#store.appendEvent(taskId, "cancel_failed", message, mergedPids.length > 0 ? { member_pids: mergedPids } : void 0, this.#now().toISOString()); + } catch { + } + } + /** Validates a persisted member_pids value (review-round-1 addendum R3): a + * PRESENT record without the member_pids field, a null, or a non-array is + * corrupt and fails closed — "missing" must never read as "no members". */ + #validMemberPids(value) { + if (value === void 0 || value === null) { + throw new TaskManagerError("CANCEL_FAILED", "cleanup failure record is corrupt: member pid list is missing"); + } + if (!Array.isArray(value) || !value.every((pid) => Number.isSafeInteger(pid) && pid > 0)) { + throw new TaskManagerError("CANCEL_FAILED", "cleanup failure record is corrupt: member pid list is not a valid array of positive integers"); + } + return value; + } + /** Parses the durable [cleanup-members: …] marker from a latched status + * error text. A TRUNCATED marker (…) is reported as incomplete: it must + * never be treated as the full member set (addendum R3 case 2). */ + #parseCleanupMembersMarker(errorText2) { + const marker = /\[cleanup-members: ([0-9,]+)(,…)?\]/u.exec(errorText2 ?? ""); + if (!marker) return null; + const pids = marker[1].split(",").map((part) => Number(part)).filter((pid) => Number.isSafeInteger(pid) && pid > 0); + return { pids, complete: marker[2] === void 0 }; + } + /** Classification of the persisted cleanup-failure state for an attempt: + * "none" (no record and no known-member marker — legacy latch), "empty" + * (explicitly member-less record), "members" (known member pids must stay + * covered, including via a complete latch marker), or "corrupt" (fail + * closed). A read error is treated as corrupt. */ + #cleanupFailureRecordState(taskId, attempt) { + let meta2 = null; + try { + meta2 = this.#store.readAttemptMeta(taskId, attempt, "cleanup-failure.json") ?? null; + } catch { + return "corrupt"; + } + const marker = this.#parseCleanupMembersMarker(this.#safeStatus(taskId)?.error); + if (meta2 === null) { + if (marker === null) return "none"; + return marker.complete ? "members" : "corrupt"; + } + let memberPids; + try { + memberPids = this.#validMemberPids(meta2.member_pids); + } catch { + return "corrupt"; + } + if (meta2.complete === false) return "corrupt"; + return memberPids.length > 0 ? "members" : "empty"; + } + /** Recorded pids (with persisted fingerprints when available) that later + * claimant scans must cover after a termination failure. The durable + * [cleanup-members: …] marker inside the latched status text is ALWAYS + * unioned in (R3): a missing or emptied failure record can never narrow + * the scan below what the latch itself recorded; a corrupt record — or a + * TRUNCATED marker that cannot restore the full set — fails closed before + * any scan runs. */ + #claimantTargets(taskId, attempt, probeSnapshot) { + const targets = []; + const addPid = (pid, fingerprint) => { + if (Number.isSafeInteger(pid) && pid > 0 && !targets.some((t) => t.pid === pid)) targets.push({ pid, fingerprint }); + }; + const workerFingerprint = typeof probeSnapshot.identity?.worker?.fingerprint === "string" ? probeSnapshot.identity.worker.fingerprint : null; + const runtimeFingerprint = typeof probeSnapshot.identity?.runtime?.fingerprint === "string" ? probeSnapshot.identity.runtime.fingerprint : null; + if (probeSnapshot.worker_pid !== null) addPid(probeSnapshot.worker_pid, workerFingerprint); + if (probeSnapshot.zcode_pid !== null) addPid(probeSnapshot.zcode_pid, runtimeFingerprint); + const marker = this.#parseCleanupMembersMarker(this.#safeStatus(taskId)?.error); + if (marker !== null && !marker.complete) { + throw new TaskManagerError("CANCEL_FAILED", "cleanup member record is incomplete: the latched member list was truncated and cannot be reconstructed; the task stays occupied"); + } + if (marker !== null) { + for (const pid of marker.pids) addPid(pid, null); + } + let meta2 = null; + try { + meta2 = this.#store.readAttemptMeta(taskId, attempt, "cleanup-failure.json") ?? null; + } catch { + } + if (meta2 === null) return targets; + if (meta2.complete === false) { + throw new TaskManagerError("CANCEL_FAILED", "cleanup member record is incomplete: the structured member list is marked partial; the task stays occupied"); + } + for (const pid of this.#validMemberPids(meta2.member_pids)) addPid(pid, null); + return targets; + } /** Applies confirmed probe verdicts to one unchanged attempt snapshot. */ #applyReconcileVerdicts(taskId, status, snapshot, verdicts) { + if (status.cleanup_unverified === true) { + const occurrence = this.#counters.record("reconcile_latched_cleanup", "cleanup_unverified"); + if (this.#counters.shouldEmit("reconcile_latched_cleanup", "cleanup_unverified")) { + this.#store.appendEvent(taskId, "cleanup_unverified", "Reconciliation skipped: a recorded tree-cleanup failure keeps the task occupied until a verified cancel releases it", { + count: occurrence.count + }, this.#now().toISOString()); + } + return; + } const persistedResult = this.#store.readResult(taskId); if (persistedResult && persistedResult.attempt === status.attempt) { this.#store.writeStatus(taskId, { @@ -25093,6 +26052,8 @@ var BridgeTaskManager = class _BridgeTaskManager { identity }; const verdicts = await this.#probeExecutors(probeSnapshot); + const workerFingerprint = typeof probeSnapshot.identity?.worker?.fingerprint === "string" ? probeSnapshot.identity.worker.fingerprint : null; + const runtimeFingerprint = typeof probeSnapshot.identity?.runtime?.fingerprint === "string" ? probeSnapshot.identity.runtime.fingerprint : null; if (status.cleanup_unverified) { if (probeSnapshot.worker_pid === null && probeSnapshot.zcode_pid === null) { throw new TaskManagerError("CANCEL_FAILED", "No worker or ZCode process identity is available to verify cleanup"); @@ -25103,32 +26064,32 @@ var BridgeTaskManager = class _BridgeTaskManager { if (probeSnapshot.worker_pid !== null && verdicts.worker.state === "unknown") { throw new TaskManagerError("CANCEL_FAILED", `Worker cleanup could not be verified: worker probe returned ${verdicts.worker.reason_code}`); } + const verifiedRoots = /* @__PURE__ */ new Set(); try { if (probeSnapshot.zcode_pid !== null && verdicts.runtime.state === "alive") { - await this.#terminateProcessTree(probeSnapshot.zcode_pid, { graceMs: 500, killWaitMs: 5e3 }); + await this.#terminateProcessTree(probeSnapshot.zcode_pid, { graceMs: 500, killWaitMs: 5e3, expectedRootFingerprint: runtimeFingerprint }); const after = (await this.#probeExecutors(probeSnapshot)).runtime; if (after.state === "alive") throw new Error("runtime still alive after termination"); if (after.state === "unknown") throw new Error(`runtime exit could not be verified (${after.reason_code})`); + verifiedRoots.add(probeSnapshot.zcode_pid); } if (probeSnapshot.worker_pid !== null && verdicts.worker.state === "alive") { - await this.#terminateProcessTree(probeSnapshot.worker_pid, { graceMs: 500, killWaitMs: 5e3 }); + await this.#terminateProcessTree(probeSnapshot.worker_pid, { graceMs: 500, killWaitMs: 5e3, expectedRootFingerprint: workerFingerprint }); + verifiedRoots.add(probeSnapshot.worker_pid); } } catch (error2) { - let recheck; - try { - recheck = await this.#probeExecutors(probeSnapshot); - } catch (recheckError) { - throw new TaskManagerError("CANCEL_FAILED", `ZCode cleanup could not be verified: ${String(error2)}; re-verification failed: ${String(recheckError)}`); - } - const runtimeRecorded = probeSnapshot.zcode_pid !== null; - const workerRecorded = probeSnapshot.worker_pid !== null; - if (runtimeRecorded && recheck.runtime.state !== "exited" || workerRecorded && recheck.worker.state !== "exited") { - throw new TaskManagerError("CANCEL_FAILED", `ZCode cleanup could not be verified: ${String(error2)}; re-verification observed runtime ${recheck.runtime.state} (${recheck.runtime.reason_code})${workerRecorded ? ` and worker ${recheck.worker.state} (${recheck.worker.reason_code})` : ""}`); + this.#recordCleanupFailure(taskId, status.attempt, error2); + throw new TaskManagerError("CANCEL_FAILED", `ZCode cleanup could not be verified: ${error2 instanceof Error ? error2.message : String(error2)}`); + } + const scanTargets = this.#claimantTargets(taskId, status.attempt, probeSnapshot).filter((target) => !verifiedRoots.has(target.pid)); + if (scanTargets.length > 0) { + const scan = await this.#verifyTreeClean(scanTargets).catch((error2) => { + throw new TaskManagerError("CANCEL_FAILED", `ZCode cleanup verification failed: ${error2 instanceof Error ? error2.message : String(error2)}`); + }); + if (!scan.clean) { + this.#store.appendEvent(taskId, "cleanup_unverified", "Recorded-tree scan still observes live claimants; the task stays occupied", { live_pids: [...scan.livePids] }, this.#now().toISOString()); + throw new TaskManagerError("CANCEL_FAILED", `ZCode cleanup could not be verified: ${String(scan.livePids.length)} recorded-tree process(es) still alive`); } - this.#store.appendEvent(taskId, "cleanup_race_resolved", "Termination raced a natural exit; a fresh identity probe confirmed the recorded processes exited", { - runtime_reason_code: recheck.runtime.reason_code, - worker_reason_code: recheck.worker.reason_code - }, this.#now().toISOString()); } if (status.status === "running") { const finishedAt2 = this.#now().toISOString(); @@ -25261,22 +26222,17 @@ var BridgeTaskManager = class _BridgeTaskManager { try { if (!await waitForSafeExit()) { if (process.platform !== "win32" && status.zcode_pid && this.#isProcessRunning(status.zcode_pid)) { - await this.#terminateProcessTree(status.zcode_pid, { graceMs: 500, killWaitMs: 5e3 }); + await this.#terminateProcessTree(status.zcode_pid, { graceMs: 500, killWaitMs: 5e3, expectedRootFingerprint: runtimeFingerprint }); } - await this.#terminateProcessTree(pid, { graceMs: 500, killWaitMs: 5e3 }); + await this.#terminateProcessTree(pid, { graceMs: 500, killWaitMs: 5e3, expectedRootFingerprint: workerFingerprint }); } } catch (error2) { - if (!await waitForSafeExit()) { - const message = error2 instanceof Error ? error2.message : String(error2); - this.#store.writeStatus(taskId, { - error: `cancellation could not be verified: ${message}` - }); - this.#store.appendEvent(taskId, "cancel_failed", message, void 0, this.#now().toISOString()); - throw new TaskManagerError( - "CANCEL_FAILED", - `process-tree termination for task ${taskId} (pid ${pid}) could not be verified: ${message}` - ); - } + this.#recordCleanupFailure(taskId, status.attempt, error2); + const message = error2 instanceof Error ? error2.message : String(error2); + throw new TaskManagerError( + "CANCEL_FAILED", + `process-tree termination for task ${taskId} (pid ${pid}) could not be verified: ${message}` + ); } const raced = this.#store.readResult(taskId); if (raced) { @@ -27236,6 +28192,30 @@ function startAppServer(config2, cwd, env) { let stderr = ""; let nextId = 0; let closed = false; + let childExitSeen = false; + child.once("exit", () => { + childExitSeen = true; + }); + const generationChild = { + get pid() { + return child.pid; + }, + get exitCode() { + return child.exitCode; + }, + get signalCode() { + return child.signalCode; + }, + get exitSeen() { + return childExitSeen; + } + }; + let childIdentity = null; + if (process.platform === "win32" && child.pid) { + void captureSpawnGenerationIdentity(generationChild, createPlatformProbe()).then((identity) => { + childIdentity = identity; + }); + } child.stdout.setEncoding("utf8"); child.stderr.setEncoding("utf8"); const write = (message) => { @@ -27326,7 +28306,14 @@ function startAppServer(config2, cwd, env) { }); }); } - if (child.pid && (!closed || process.platform !== "win32")) await terminateProcessTree(child.pid); + if (child.pid && (!closed || process.platform !== "win32")) { + if (process.platform === "win32" && !childStillOnSpawnGeneration(generationChild)) return; + await terminateProcessTree(child.pid, { + graceMs: 500, + killWaitMs: 5e3, + expectedRootFingerprint: childIdentity?.pid === child.pid ? childIdentity.fingerprint : null + }); + } }; return { child, request, close, get stderr() { return stderr; diff --git a/plugins/codex-zcode-bridge/worker/worker-main.mjs b/plugins/codex-zcode-bridge/worker/worker-main.mjs index 7ac8178..1253b0f 100644 --- a/plugins/codex-zcode-bridge/worker/worker-main.mjs +++ b/plugins/codex-zcode-bridge/worker/worker-main.mjs @@ -659,9 +659,6 @@ function errorText(error62) { return message.length > 300 ? `${message.slice(0, 300)}\u2026` : message; } -// src/adapters/process-spawn.ts -import { spawn as spawn2 } from "node:child_process"; - // src/runtime/process-probe.ts import { readFile } from "node:fs/promises"; import { spawn } from "node:child_process"; @@ -875,91 +872,771 @@ function createPlatformProbe(options = {}) { } // src/adapters/windows-process-cleanup.ts -import { performance } from "node:perf_hooks"; -async function terminateWindowsProcessTree(pid, killWaitMs, dependencies) { - const requests = await dependencies.capture(pid); - if (!requests.some((request) => request.pid === pid)) throw new Error("process_tree_snapshot_invalid: root missing"); - const inspect = async () => { - const verdicts = await dependencies.probe(requests); - if (verdicts.length !== requests.length || verdicts.some((verdict2) => verdict2.state === "unknown")) { - throw new Error("process_tree_probe_unverified: recorded executor exit is unknown"); - } - return verdicts; - }; - const before = await inspect(); - if (before.every((verdict2) => verdict2.state === "exited")) return; - const rootIndex = requests.findIndex((request) => request.pid === pid); - if (before[rootIndex]?.state !== "alive") { - throw new Error("process_tree_descendants_alive: root exited before termination"); - } - let exitCode = null; - let commandFailed = false; +import { spawn as spawn2 } from "node:child_process"; +var CIM_REPRESENTATION_INTERVAL_TICKS = 10n; +function withinCimInterval(exactFileTime, cimFileTime) { + const exact = BigInt(exactFileTime); + const cim = BigInt(cimFileTime); + return exact >= cim && exact <= cim + CIM_REPRESENTATION_INTERVAL_TICKS; +} +function cimIntervalWithin(cimFileTime, lowExact, highExact) { + const cim = BigInt(cimFileTime); + return cim >= BigInt(lowExact) && cim + CIM_REPRESENTATION_INTERVAL_TICKS <= BigInt(highExact); +} +function cimClearlyBefore(cimFileTime, exactFileTime) { + return BigInt(cimFileTime) + CIM_REPRESENTATION_INTERVAL_TICKS < BigInt(exactFileTime); +} +function cimClearlyAfter(cimFileTime, exactFileTime) { + return BigInt(cimFileTime) > BigInt(exactFileTime); +} +function acceptMember(row, input2, held, members2, memberPids, decidedIrrelevant, holdRequired, refuse) { + if (row.startTicks === null || row.creationFileTime === null) { + refuse("process_tree_member_identity_unknown", row.pid); + return; + } + const child = held.get(row.pid); + if (!child) { + holdRequired.add(row.pid); + return; + } + if (!withinCimInterval(child.creationFileTime, row.creationFileTime)) { + refuse("process_tree_generation_mismatch", row.pid); + return; + } + if (child.creationTicks !== row.startTicks) { + refuse("process_tree_member_generation_mismatch", row.pid); + return; + } + memberPids.add(row.pid); + members2.push({ pid: row.pid, parentPid: row.parentPid, creationFileTime: row.creationFileTime, startTicks: row.startTicks }); +} +function resolveWindowsTreeMembership(input2) { + const rowsByPid = /* @__PURE__ */ new Map(); + for (const row of input2.rows) { + if (rowsByPid.has(row.pid)) return { kind: "refuse", code: "process_tree_snapshot_invalid", pid: row.pid }; + rowsByPid.set(row.pid, row); + } + const members2 = []; + const memberPids = /* @__PURE__ */ new Set(); + const decidedIrrelevant = /* @__PURE__ */ new Set(); + const holdRequired = /* @__PURE__ */ new Set(); + const state = { refusal: null }; + const refuse = (code, pid) => { + if (!state.refusal) state.refusal = { code, pid }; + }; + const rootRow = rowsByPid.get(input2.rootPid); + if (rootRow) { + if (input2.expectedRootTicks === null) { + return { kind: "refuse", code: "process_tree_root_identity_required", pid: input2.rootPid }; + } + if (rootRow.startTicks === null || rootRow.creationFileTime === null) { + return { kind: "refuse", code: "process_tree_root_identity_unknown", pid: input2.rootPid }; + } + if (rootRow.startTicks !== input2.expectedRootTicks) { + return { kind: "refuse", code: "process_tree_root_identity_changed", pid: input2.rootPid }; + } + const heldRoot = input2.held.get(input2.rootPid); + if (!heldRoot) { + holdRequired.add(input2.rootPid); + } else if (heldRoot.creationTicks !== rootRow.startTicks || !withinCimInterval(heldRoot.creationFileTime, rootRow.creationFileTime)) { + return { kind: "refuse", code: "process_tree_root_generation_mismatch", pid: input2.rootPid }; + } else { + memberPids.add(rootRow.pid); + members2.push({ pid: rootRow.pid, parentPid: rootRow.parentPid, creationFileTime: rootRow.creationFileTime, startTicks: rootRow.startTicks }); + } + } else if (!input2.killed.has(input2.rootPid)) { + if (input2.expectedRootTicks === null) { + return { kind: "refuse", code: "process_tree_root_identity_required", pid: input2.rootPid }; + } + return { kind: "refuse", code: "process_tree_root_exited", pid: input2.rootPid }; + } + for (let changed = true; changed && !state.refusal; ) { + changed = false; + for (const row of input2.rows) { + if (state.refusal) break; + if (memberPids.has(row.pid) || decidedIrrelevant.has(row.pid)) continue; + const parentPid = row.parentPid; + if (parentPid === row.pid) { + decidedIrrelevant.add(row.pid); + changed = true; + continue; + } + const parentRow = memberPids.has(parentPid) ? rowsByPid.get(parentPid) : void 0; + if (parentRow) { + if (row.creationFileTime === null) { + refuse("process_tree_row_creation_unknown", row.pid); + continue; + } + if (parentRow.creationFileTime === null) { + refuse("process_tree_row_creation_unknown", parentPid); + continue; + } + const parent = input2.held.get(parentPid); + if (!parent) { + holdRequired.add(parentPid); + continue; + } + if (cimClearlyBefore(row.creationFileTime, parent.creationFileTime)) { + decidedIrrelevant.add(row.pid); + changed = true; + continue; + } + const child = input2.held.get(row.pid); + if (!child) { + holdRequired.add(row.pid); + continue; + } + if (!withinCimInterval(child.creationFileTime, row.creationFileTime) || !withinCimInterval(parent.creationFileTime, parentRow.creationFileTime)) { + refuse("process_tree_generation_mismatch", row.pid); + continue; + } + const childCreation = BigInt(child.creationFileTime); + const parentCreation = BigInt(parent.creationFileTime); + if (childCreation === parentCreation) { + refuse("process_tree_edge_indistinguishable", row.pid); + continue; + } + if (childCreation < parentCreation) { + decidedIrrelevant.add(row.pid); + changed = true; + continue; + } + acceptMember(row, input2, input2.held, members2, memberPids, decidedIrrelevant, holdRequired, refuse); + if (memberPids.has(row.pid)) changed = true; + continue; + } + const killedParent = input2.killed.get(parentPid); + if (killedParent) { + if (row.creationFileTime === null) { + refuse("process_tree_row_creation_unknown", row.pid); + continue; + } + if (cimClearlyBefore(row.creationFileTime, killedParent.creationFileTime)) { + decidedIrrelevant.add(row.pid); + changed = true; + continue; + } + if (killedParent.exitFileTime === null) { + refuse("process_tree_parent_exit_unconfirmed", parentPid); + continue; + } + if (cimClearlyAfter(row.creationFileTime, killedParent.exitFileTime)) { + decidedIrrelevant.add(row.pid); + changed = true; + continue; + } + if (cimIntervalWithin(row.creationFileTime, killedParent.creationFileTime, killedParent.exitFileTime)) { + acceptMember(row, input2, input2.held, members2, memberPids, decidedIrrelevant, holdRequired, refuse); + if (memberPids.has(row.pid)) changed = true; + continue; + } + const child = input2.held.get(row.pid); + if (!child) { + holdRequired.add(row.pid); + continue; + } + if (!withinCimInterval(child.creationFileTime, row.creationFileTime)) { + refuse("process_tree_generation_mismatch", row.pid); + continue; + } + const creation = BigInt(child.creationFileTime); + if (creation >= BigInt(killedParent.creationFileTime) && creation <= BigInt(killedParent.exitFileTime)) { + acceptMember(row, input2, input2.held, members2, memberPids, decidedIrrelevant, holdRequired, refuse); + if (memberPids.has(row.pid)) changed = true; + } else { + decidedIrrelevant.add(row.pid); + changed = true; + } + continue; + } + if (input2.naturalGonePids.has(parentPid)) { + if (row.creationFileTime === null) { + refuse("process_tree_row_creation_unknown", row.pid); + continue; + } + const lastCim = input2.naturalGoneCreation.get(parentPid) ?? null; + if (lastCim !== null && cimClearlyBefore(row.creationFileTime, lastCim)) { + decidedIrrelevant.add(row.pid); + changed = true; + continue; + } + refuse("process_tree_parent_unprovable", parentPid); + continue; + } + const parentUndecidedLiveRow = rowsByPid.has(parentPid) && !memberPids.has(parentPid) && !decidedIrrelevant.has(parentPid); + if (parentUndecidedLiveRow || parentPid === input2.rootPid) continue; + decidedIrrelevant.add(row.pid); + changed = true; + } + } + if (state.refusal) return { kind: "refuse", code: state.refusal.code, pid: state.refusal.pid }; + if (holdRequired.size > 0) return { kind: "members", members: members2, holdRequired: [...holdRequired] }; + if (members2.length === 0) return { kind: "success" }; + return { kind: "members", members: members2, holdRequired: [] }; +} +var WindowsTreeCleanupError = class extends Error { + code; + memberPids; + constructor(code, message, memberPids = []) { + super(message); + this.name = "WindowsTreeCleanupError"; + this.code = code; + this.memberPids = memberPids; + } +}; +async function terminateWindowsProcessTree(pid, expectedRootTicks, killWaitMs, dependencies) { + if (!Number.isInteger(pid) || pid <= 0) { + throw new WindowsTreeCleanupError("process_tree_invalid_pid", `a positive integer PID is required, got: ${String(pid)}`); + } + const now = dependencies.now ?? Date.now; + const sleep3 = dependencies.sleep ?? ((ms) => new Promise((resolve) => setTimeout(resolve, ms))); + const transaction = await dependencies.open(); + const killed = /* @__PURE__ */ new Map(); + const held = /* @__PURE__ */ new Map(); + const naturalGone = /* @__PURE__ */ new Set(); + const naturalGoneCreation = /* @__PURE__ */ new Map(); + const memberReceipts = /* @__PURE__ */ new Map(); + const killReceipts = []; + const unverifiedCandidates = /* @__PURE__ */ new Set(); + let rounds = 0; + let bodyError = null; + let succeeded = false; + const knownPids = () => [.../* @__PURE__ */ new Set([...memberReceipts.keys(), ...unverifiedCandidates])]; try { - exitCode = await dependencies.kill(pid); - } catch { - commandFailed = true; + const deadline = now() + Math.max(0, killWaitMs); + for (; ; ) { + rounds += 1; + const rows = await transaction.enumerate(); + const resolution = await resolveRound(rows); + if (resolution.kind === "refuse") { + throw new WindowsTreeCleanupError(resolution.code, `${resolution.code}: process tree cleanup refused`, knownPids()); + } + if (resolution.kind === "success") break; + const members2 = resolution.members.map((m) => ({ pid: m.pid, parentPid: m.parentPid, startTicks: m.startTicks, creationFileTime: m.creationFileTime })); + const livePids = new Set(resolution.members.map((m) => m.pid)); + for (const previous of memberReceipts.keys()) { + if (!livePids.has(previous) && !killed.has(previous) && !naturalGone.has(previous)) { + naturalGone.add(previous); + naturalGoneCreation.set(previous, memberReceipts.get(previous).creationFileTime); + } + } + for (const member of members2) { + unverifiedCandidates.delete(member.pid); + if (!memberReceipts.has(member.pid)) { + memberReceipts.set(member.pid, { pid: member.pid, fingerprint: member.startTicks, creationFileTime: member.creationFileTime }); + } + } + const ordered = orderChildrenFirst(members2); + const remaining = Math.max(120, deadline - now()); + const perMemberWait = Math.min(2e3, Math.max(120, Math.floor(remaining / Math.max(1, ordered.length)))); + for (const member of ordered) { + const heldInfo = held.get(member.pid); + const kill = await transaction.kill(member.pid, member.startTicks, perMemberWait); + killReceipts.push({ pid: member.pid, status: kill.status, exitFileTime: kill.exitFileTime }); + if (kill.status === "signaled" || kill.status === "already_exited") { + killed.set(member.pid, { + pid: member.pid, + creationFileTime: heldInfo?.creationFileTime ?? member.creationFileTime, + exitFileTime: kill.exitFileTime + }); + } else if (kill.status === "identity_changed" || kill.status === "not_held") { + naturalGone.add(member.pid); + naturalGoneCreation.set(member.pid, member.creationFileTime); + } + } + if (now() >= deadline) { + throw new WindowsTreeCleanupError( + "process_tree_cleanup_unverified", + `process_tree_cleanup_unverified: ${String(resolution.members.length)} recorded member(s) not provably exited`, + knownPids() + ); + } + await sleep3(20); + } + succeeded = true; + return { + rootPid: pid, + rounds, + members: [...memberReceipts.values()], + kills: killReceipts, + signaled: killReceipts.filter((k) => k.status === "signaled").map((k) => k.pid) + }; + } catch (error62) { + bodyError = error62; + if (error62 instanceof WindowsTreeCleanupError) { + throw error62.memberPids.length > 0 ? error62 : new WindowsTreeCleanupError(error62.code, error62.message, knownPids()); + } + const detail = error62 instanceof Error ? error62.message : String(error62); + throw new WindowsTreeCleanupError("process_tree_transaction_error", `process_tree_transaction_error: ${detail}`, knownPids()); + } finally { + let closeError = null; + try { + await transaction.close(); + } catch (error62) { + closeError = error62; + } + if (closeError !== null) { + const closeText = closeError instanceof Error ? closeError.message : String(closeError); + if (bodyError !== null) { + const original = bodyError instanceof WindowsTreeCleanupError ? new WindowsTreeCleanupError(bodyError.code, `${bodyError.message}; process_tree_close_failed: ${closeText}`, bodyError.memberPids.length > 0 ? bodyError.memberPids : knownPids()) : new WindowsTreeCleanupError("process_tree_transaction_error", `process_tree_transaction_error: ${String(bodyError instanceof Error ? bodyError.message : bodyError)}; process_tree_close_failed: ${closeText}`, knownPids()); + throw original; + } + throw new WindowsTreeCleanupError("process_tree_close_failed", `process_tree_close_failed: ${closeText}`, knownPids()); + } + } + async function resolveRound(rows) { + let workingRows = [...rows]; + const resolve = () => resolveWindowsTreeMembership({ + rows: workingRows, + rootPid: pid, + expectedRootTicks, + held, + killed, + naturalGonePids: naturalGone, + naturalGoneCreation + }); + let resolution = resolve(); + for (let attempt = 0; attempt < 6 && resolution.kind === "members" && resolution.holdRequired.length > 0; attempt += 1) { + const requested = [...new Set(resolution.holdRequired)]; + for (const candidate of resolution.holdRequired) unverifiedCandidates.add(candidate); + const outcomes = await transaction.hold(requested); + if (outcomes.length !== requested.length) { + throw new WindowsTreeCleanupError( + "process_tree_hold_invalid", + `process_tree_hold_invalid: ${String(outcomes.length)} responses for ${String(requested.length)} requested pids`, + knownPids() + ); + } + const seen = /* @__PURE__ */ new Set(); + for (const outcome of outcomes) { + if (seen.has(outcome.pid)) { + throw new WindowsTreeCleanupError("process_tree_hold_invalid", `process_tree_hold_invalid: duplicate response for pid ${String(outcome.pid)}`, knownPids()); + } + seen.add(outcome.pid); + if (!requested.includes(outcome.pid)) { + throw new WindowsTreeCleanupError("process_tree_hold_invalid", `process_tree_hold_invalid: unexpected pid ${String(outcome.pid)} in hold response`, knownPids()); + } + if (outcome.status === "ok") { + held.set(outcome.pid, { pid: outcome.pid, creationFileTime: outcome.creationFileTime, creationTicks: outcome.creationTicks }); + } else if (outcome.status === "absent") { + throw new WindowsTreeCleanupError( + "process_tree_candidate_absent", + `process_tree_candidate_absent: candidate pid ${String(outcome.pid)} vanished between enumeration and hold; its subtree cannot be attributed`, + [.../* @__PURE__ */ new Set([...requested, ...knownPids()])] + ); + } + } + resolution = resolve(); + } + if (resolution.kind === "members" && resolution.holdRequired.length > 0) { + for (const candidate of resolution.holdRequired) unverifiedCandidates.add(candidate); + const roundMemberPids = resolution.members.map((m) => m.pid); + throw new WindowsTreeCleanupError( + "process_tree_hold_unresolved", + `process_tree_hold_unresolved: cannot prove identity/ownership for pids ${resolution.holdRequired.join(",")}`, + [.../* @__PURE__ */ new Set([...roundMemberPids, ...knownPids()])] + ); + } + return resolution; } - const deadline = performance.now() + Math.max(0, killWaitMs); - for (; ; ) { - const after = await inspect(); - if (after.every((verdict2) => verdict2.state === "exited")) return; - if (performance.now() >= deadline) { - const reason = commandFailed ? "command_error" : exitCode === null ? "timeout" : `exit_${exitCode}`; - throw new Error(`process_tree_cleanup_unverified: taskkill ${reason}; recorded executors remain alive`); +} +function orderChildrenFirst(members2) { + const remaining = [...members2]; + const ordered = []; + let progress = true; + while (remaining.length > 0 && progress) { + progress = false; + for (let index = remaining.length - 1; index >= 0; index -= 1) { + const member = remaining[index]; + const hasPendingChild = remaining.some((other) => other !== member && other.parentPid === member.pid); + if (!hasPendingChild) { + ordered.push(member); + remaining.splice(index, 1); + progress = true; + } } - await new Promise((resolve) => setTimeout(resolve, 20)); } + const leftovers = remaining.sort((a, b) => BigInt(a.creationFileTime) > BigInt(b.creationFileTime) ? -1 : 1); + return [...ordered, ...leftovers]; } -function windowsTreeCaptureScript(pid) { +function windowsTreeTransactionScript() { return [ "$ErrorActionPreference = 'Stop'", - "$all = @(Get-CimInstance Win32_Process -ErrorAction Stop)", - `$selected = [System.Collections.Generic.HashSet[int]]::new(); [void]$selected.Add(${pid})`, - "do { $changed = $false; foreach ($item in $all) { if ($selected.Contains([int]$item.ParentProcessId)) { if ($selected.Add([int]$item.ProcessId)) { $changed = $true } } } } while ($changed)", - "'bridge_tree_snapshot_v1'", - "foreach ($candidate in $selected) { $item = Get-Process -Id $candidate -ErrorAction SilentlyContinue; if ($null -eq $item) { '{0}|absent' -f $candidate } else { '{0}|{1}' -f $candidate, $item.StartTime.Ticks } }" - ].join("; "); -} -function parseWindowsTreeCapture(stdout, pid) { - const lines = stdout.trim().split(/\r?\n/u); - if (lines.shift() !== "bridge_tree_snapshot_v1") throw new Error("process_tree_snapshot_invalid: protocol missing"); - const capturedAt = (/* @__PURE__ */ new Date()).toISOString(); - const requests = []; - const seen = /* @__PURE__ */ new Set(); - for (const line of lines) { - const match = /^(\d+)\|(\d+|absent)$/u.exec(line.trim()); - const candidate = Number(match?.[1]); - if (!match || !Number.isSafeInteger(candidate) || candidate <= 0 || seen.has(candidate)) { - throw new Error("process_tree_snapshot_invalid: malformed identity"); - } - seen.add(candidate); - requests.push({ pid: candidate, identity: match[2] === "absent" ? null : { - pid: candidate, - fingerprint: match[2], - fingerprint_precision: "exact", - identity_version: 1, - platform: "win32", - captured_at: capturedAt - } }); - } - if (!seen.has(pid)) throw new Error("process_tree_snapshot_invalid: root missing"); - return requests; + "$ProgressPreference = 'SilentlyContinue'", + "Add-Type -TypeDefinition @'", + "using System;", + "using System.Collections.Generic;", + "using System.Runtime.InteropServices;", + "public static class BridgeTreeTx {", + ' [DllImport("kernel32.dll", SetLastError = true)] static extern IntPtr OpenProcess(uint access, bool inherit, int pid);', + ' [DllImport("kernel32.dll", SetLastError = true)] static extern bool GetProcessTimes(IntPtr h, out long created, out long exited, out long kernel, out long user);', + ' [DllImport("kernel32.dll", SetLastError = true)] static extern bool TerminateProcess(IntPtr h, uint exitCode);', + ' [DllImport("kernel32.dll", SetLastError = true)] static extern uint WaitForSingleObject(IntPtr h, uint milliseconds);', + ' [DllImport("kernel32.dll", SetLastError = true)] static extern bool CloseHandle(IntPtr h);', + " public static readonly Dictionary Held = new Dictionary();", + " public static string Hold(int pid) {", + " IntPtr existing;", + ' if (Held.TryGetValue(pid, out existing) && existing != IntPtr.Zero) { return "already_held|"; }', + " IntPtr h = OpenProcess(0x00101001u, false, pid);", + ' if (h == IntPtr.Zero) { int err = Marshal.GetLastWin32Error(); return (err == 87 ? "absent|" : (err == 5 ? "denied|" : "error|")) + err.ToString(); }', + " long c, e, k, u;", + ' if (!GetProcessTimes(h, out c, out e, out k, out u)) { int err = Marshal.GetLastWin32Error(); CloseHandle(h); return "error|" + err.ToString(); }', + " Held[pid] = h;", + ' return "ok|" + c.ToString() + "|" + DateTime.FromFileTime(c).Ticks.ToString();', + " }", + " public static string Kill(int pid, string expectTicks, int waitMs) {", + " IntPtr h;", + ' if (!Held.TryGetValue(pid, out h) || h == IntPtr.Zero) { return "not_held|"; }', + " long c, e, k, u;", + ' if (!GetProcessTimes(h, out c, out e, out k, out u)) { return "error|" + Marshal.GetLastWin32Error().ToString(); }', + " string current = DateTime.FromFileTime(c).Ticks.ToString();", + ' if (current != expectTicks) { return "identity_changed|" + current; }', + ' if (e != 0) { return "already_exited|" + e.ToString(); }', + ' if (!TerminateProcess(h, 1)) { return "error|" + Marshal.GetLastWin32Error().ToString(); }', + " uint wait = WaitForSingleObject(h, (uint)waitMs);", + ' if (!GetProcessTimes(h, out c, out e, out k, out u)) { return "error|" + Marshal.GetLastWin32Error().ToString(); }', + ' if (wait != 0) { return "wait_timeout|" + e.ToString(); }', + ' return "signaled|" + e.ToString();', + " }", + " public static string CloseAll() {", + " int closed = 0;", + " int failed = 0;", + " foreach (KeyValuePair entry in Held) { if (entry.Value != IntPtr.Zero) { if (CloseHandle(entry.Value)) { closed++; } else { failed++; } } }", + " Held.Clear();", + ' return closed.ToString() + "|" + failed.ToString();', + " }", + "}", + "'@", + "[Console]::Out.WriteLine('READY'); [Console]::Out.Flush()", + "while ($true) {", + " $line = [Console]::In.ReadLine()", + " if ($null -eq $line) { [void][BridgeTreeTx]::CloseAll(); exit 0 }", + " $parts = $line.Trim().Split(' ')", + " if ($parts[0] -eq 'enum') {", + " $all = @(Get-CimInstance Win32_Process -ErrorAction Stop)", + " [Console]::Out.WriteLine('ENUM_BEGIN')", + " $ids = @(); foreach ($p in $all) { $ids += [int]$p.ProcessId }", + " $byId = @{}", + " if ($ids.Count -gt 0) { foreach ($g in (Get-Process -Id $ids -ErrorAction SilentlyContinue)) { $byId[[int]$g.Id] = $g } }", + " foreach ($p in $all) {", + " $ticks = 'unknown'", + " $cim = 'unknown'", + " if ($null -ne $p.CreationDate) { $cim = [string]$p.CreationDate.ToFileTimeUtc() }", + " $g = $byId[[int]$p.ProcessId]", + " if ($null -ne $g) { try { $st = $g.StartTime; if ($null -ne $st) { $ticks = [string]$st.Ticks } } catch { $ticks = 'unknown' } }", + " if ([string]::IsNullOrEmpty($ticks)) { $ticks = 'unknown' }", + " [Console]::Out.WriteLine(('R {0} {1} {2} {3}' -f [int]$p.ProcessId, [int]$p.ParentProcessId, $cim, $ticks))", + " }", + " [Console]::Out.WriteLine('ENUM_END'); [Console]::Out.Flush()", + " } elseif ($parts[0] -eq 'hold') {", + " for ($i = 1; $i -lt $parts.Count; $i++) {", + " $hp = 0", + " if (-not [int]::TryParse($parts[$i], [ref]$hp)) { [Console]::Out.WriteLine('H 0 error 22'); continue }", + " $r = [BridgeTreeTx]::Hold($hp).Split('|')", + " if ($r[0] -eq 'already_held') { [Console]::Out.WriteLine(('H {0} held' -f $hp)) }", + " elseif ($r[0] -eq 'ok') { [Console]::Out.WriteLine(('H {0} ok {1} {2}' -f $hp, $r[1], $r[2])) }", + " else { [Console]::Out.WriteLine(('H {0} {1} {2}' -f $hp, $r[0], $r[1])) }", + " }", + " [Console]::Out.WriteLine('HOLD_END'); [Console]::Out.Flush()", + " } elseif ($parts[0] -eq 'kill') {", + " $kpid = [int]$parts[1]; $kticks = [string]$parts[2]; $kwait = [int]$parts[3]", + " $r = [BridgeTreeTx]::Kill($kpid, $kticks, $kwait).Split('|')", + " if ($r[0] -eq 'signaled' -or $r[0] -eq 'already_exited') { [Console]::Out.WriteLine(('K {0} {1} {2}' -f $kpid, $r[0], $r[1])) }", + " elseif ($r[0] -eq 'identity_changed') { [Console]::Out.WriteLine(('K {0} identity_changed 0' -f $kpid)) }", + " elseif ($r[0] -eq 'wait_timeout') { [Console]::Out.WriteLine(('K {0} wait_timeout {1}' -f $kpid, $r[1])) }", + " elseif ($r[0] -eq 'not_held') { [Console]::Out.WriteLine(('K {0} not_held 0' -f $kpid)) }", + " else { [Console]::Out.WriteLine(('K {0} error {1}' -f $kpid, $r[1])) }", + " [Console]::Out.WriteLine('KILL_END'); [Console]::Out.Flush()", + " } elseif ($parts[0] -eq 'close') {", + " $r = [BridgeTreeTx]::CloseAll().Split('|')", + " [Console]::Out.WriteLine(('CLOSED {0} {1}' -f $r[0], $r[1])); [Console]::Out.Flush()", + " exit 0", + " } else {", + " [Console]::Out.WriteLine('FATAL unknown_command'); [Console]::Out.Flush(); exit 1", + " }", + "}" + ].join("\n"); +} +var TRANSPORT_MAX_LINE_BYTES = 1048576; +var TRANSPORT_MAX_BUFFERED_BYTES = 8388608; +var TRANSPORT_MAX_BUFFERED_LINES = 65536; +var TRANSPORT_MAX_ENUM_ROWS = 4096; +var TRANSPORT_TOTAL_SESSION_MS_DEFAULT = 12e4; +var PowerShellTreeTransaction = class _PowerShellTreeTransaction { + #child; + #bounds; + #lines = []; + #waiters = []; + #exitWaiters = []; + #closed = false; + #bufferedBytes = 0; + #sessionDeadline; + #exitCode = null; + #exitSignal = null; + /** True once the helper's own exit is expected (receipt processed or + * fallback initiated); its close event is then not a transport failure. */ + #closeHandled = false; + #failure = null; + /** Test seam: attach to an externally spawned/controlled child so the + * bounded-stream behaviours are testable with fake streams (no OS). */ + static attach(child, bounds = {}) { + return new _PowerShellTreeTransaction(child, bounds); + } + constructor(child, bounds = {}) { + this.#child = child; + this.#bounds = { + maxLineBytes: bounds.maxLineBytes ?? TRANSPORT_MAX_LINE_BYTES, + maxBufferedBytes: bounds.maxBufferedBytes ?? TRANSPORT_MAX_BUFFERED_BYTES, + maxBufferedLines: bounds.maxBufferedLines ?? TRANSPORT_MAX_BUFFERED_LINES, + maxEnumRows: bounds.maxEnumRows ?? TRANSPORT_MAX_ENUM_ROWS, + totalSessionMs: bounds.totalSessionMs ?? TRANSPORT_TOTAL_SESSION_MS_DEFAULT + }; + this.#sessionDeadline = Date.now() + this.#bounds.totalSessionMs; + child.stdout.setEncoding("utf8"); + let buffer = ""; + child.stdout.on("data", (chunk) => { + if (this.#failure) { + buffer = ""; + return; + } + buffer += chunk; + let newline = buffer.indexOf("\n"); + while (newline >= 0) { + const line = buffer.slice(0, newline).replace(/\r$/u, ""); + buffer = buffer.slice(newline + 1); + const lineBytes = Buffer.byteLength(line, "utf8"); + if (lineBytes > this.#bounds.maxLineBytes) { + this.#failAndDrain(); + this.#fail(new Error(`process_tree_transport_overflow: line of ${String(lineBytes)} bytes exceeds ${String(this.#bounds.maxLineBytes)}`)); + return; + } + this.#bufferedBytes += lineBytes; + if (this.#bufferedBytes > this.#bounds.maxBufferedBytes || this.#lines.length >= this.#bounds.maxBufferedLines) { + this.#failAndDrain(); + this.#fail(new Error(`process_tree_transport_overflow: buffered stream exceeded its byte or line budget (${String(this.#bufferedBytes)} bytes, ${String(this.#lines.length)} lines)`)); + return; + } + const waiter = this.#waiters.shift(); + if (waiter) { + clearTimeout(waiter.timer); + waiter.resolve(line); + this.#bufferedBytes -= lineBytes; + } else { + this.#lines.push(line); + } + newline = buffer.indexOf("\n"); + } + if (Buffer.byteLength(buffer, "utf8") > this.#bounds.maxLineBytes) { + this.#failAndDrain(); + this.#fail(new Error(`process_tree_transport_overflow: unterminated line exceeds ${String(this.#bounds.maxLineBytes)} bytes`)); + } + }); + child.stderr.setEncoding("utf8"); + child.stderr.on("data", (chunk) => { + void chunk; + this.#fail(new Error("process_tree_transport_stderr")); + }); + child.on("error", (error62) => this.#fail(new Error(`process_tree_transport_error: ${String(error62.message)}`))); + child.on("close", (code, signal) => { + this.#closed = true; + this.#exitCode = code; + this.#exitSignal = signal; + if (!this.#closeHandled) this.#fail(new Error("process_tree_transport_closed")); + for (const waiter of this.#exitWaiters.splice(0)) waiter(); + }); + } + /** Clears buffered receive state; used when a budget trips so a failed + * transport neither accumulates nor serves stale lines afterwards. */ + #failAndDrain() { + for (const line of this.#lines) this.#bufferedBytes -= Buffer.byteLength(line, "utf8"); + this.#lines = []; + } + #fail(error62) { + if (this.#failure) return; + this.#failure = error62; + for (const waiter of this.#waiters.splice(0)) { + clearTimeout(waiter.timer); + waiter.reject(error62); + } + } + #nextLine(timeoutMs, what) { + if (this.#failure) return Promise.reject(this.#failure); + const remainingSession = this.#sessionDeadline - Date.now(); + if (remainingSession <= 0) { + const error62 = new Error(`process_tree_transport_session_timeout: ${what}`); + this.#failAndDrain(); + this.#fail(error62); + return Promise.reject(error62); + } + const buffered = this.#lines.shift(); + if (buffered !== void 0) { + this.#bufferedBytes -= Buffer.byteLength(buffered, "utf8"); + return Promise.resolve(buffered); + } + if (this.#closed) return Promise.reject(new Error("process_tree_transport_closed")); + const boundedTimeout = Math.min(timeoutMs, remainingSession); + return new Promise((resolve, reject) => { + const timer = setTimeout(() => { + const index = this.#waiters.findIndex((w) => w.resolve === resolve); + if (index >= 0) this.#waiters.splice(index, 1); + const reason = remainingSession < timeoutMs ? "process_tree_transport_session_timeout" : "process_tree_transport_timeout"; + reject(new Error(`${reason}: ${what}`)); + this.#fail(new Error(`${reason}: ${what}`)); + }, boundedTimeout); + if (typeof timer.unref === "function") timer.unref(); + this.#waiters.push({ resolve, reject, timer }); + }); + } + static async start(timeoutMs = 2e4) { + const encoded = Buffer.from(windowsTreeTransactionScript(), "utf16le").toString("base64"); + const child = spawn2("powershell.exe", ["-NoProfile", "-NonInteractive", "-ExecutionPolicy", "Bypass", "-EncodedCommand", encoded], { + shell: false, + windowsHide: true, + stdio: ["pipe", "pipe", "pipe"] + }); + const transaction = _PowerShellTreeTransaction.attach(child); + try { + const ready = await transaction.#nextLine(timeoutMs, "helper startup"); + if (ready !== "READY") throw new Error("process_tree_transport_protocol: startup marker missing"); + return transaction; + } catch (error62) { + transaction.#closeHandled = true; + try { + child.kill("SIGKILL"); + } catch { + } + const exited = await transaction.#waitForExit(3e3); + if (!exited) { + throw new Error(`${error62 instanceof Error ? error62.message : String(error62)}; process_tree_close_failed: startup helper did not exit within bounds`); + } + throw error62; + } + } + async enumerate() { + this.#child.stdin.write("enum\n"); + const begin = await this.#nextLine(2e4, "enum begin"); + if (begin !== "ENUM_BEGIN") throw new Error("process_tree_snapshot_invalid: protocol missing"); + const rows = []; + for (; ; ) { + const line = await this.#nextLine(2e4, "enum row"); + if (line === "ENUM_END") break; + const match = /^R (\d{1,12}) (\d{1,12}) (\d{1,25}|unknown) (\d{1,25}|unknown)$/u.exec(line); + if (!match) throw new Error("process_tree_snapshot_invalid: malformed identity"); + const pid = Number(match[1]); + const parentPid = Number(match[2]); + if (!Number.isSafeInteger(pid) || pid <= 0 || !Number.isSafeInteger(parentPid) || parentPid < 0) continue; + if (rows.length >= this.#bounds.maxEnumRows) { + throw new Error(`process_tree_snapshot_invalid: enumeration exceeded ${String(this.#bounds.maxEnumRows)} rows`); + } + rows.push({ pid, parentPid, creationFileTime: match[3] === "unknown" ? null : match[3], startTicks: match[4] === "unknown" ? null : match[4] }); + } + return rows; + } + async hold(pids) { + if (pids.length === 0) return []; + this.#child.stdin.write(`hold ${pids.join(" ")} +`); + const outcomes = []; + for (; ; ) { + const line = await this.#nextLine(2e4, "hold response"); + if (line === "HOLD_END") break; + const match = /^H (\d{1,12}) (ok (\d{1,25}) (\d{1,25})|held|absent (\d{1,6})|denied (\d{1,6})|error (\d{1,6}))$/u.exec(line); + if (!match) throw new Error("process_tree_hold_invalid: malformed response"); + const pid = Number(match[1]); + if (match[2].startsWith("ok")) { + outcomes.push({ pid, status: "ok", creationFileTime: match[3], creationTicks: match[4] }); + } else if (match[2] === "held") { + outcomes.push({ pid, status: "error", detail: "already_held" }); + } else if (match[2].startsWith("absent")) { + outcomes.push({ pid, status: "absent", error: Number(match[5] ?? 0) }); + } else if (match[2].startsWith("denied")) { + outcomes.push({ pid, status: "denied", error: Number(match[6] ?? 0) }); + } else { + outcomes.push({ pid, status: "error", error: Number(match[7] ?? 0) }); + } + } + return outcomes; + } + async kill(pid, expectedTicks, waitMs) { + this.#child.stdin.write(`kill ${String(pid)} ${expectedTicks} ${String(Math.max(0, Math.floor(waitMs)))} +`); + const line = await this.#nextLine(waitMs + 25e3, "kill response"); + const end = await this.#nextLine(5e3, "kill terminator"); + if (end !== "KILL_END") throw new Error("process_tree_kill_invalid: terminator missing"); + const match = /^K (\d{1,12}) (signaled|identity_changed|not_held|wait_timeout|already_exited|error) (\d{1,25})$/u.exec(line); + if (!match || Number(match[1]) !== pid) throw new Error("process_tree_kill_invalid: malformed response"); + const status = match[2]; + const value = match[3]; + return { + status, + exitFileTime: status === "signaled" || status === "already_exited" || status === "wait_timeout" ? value === "0" ? null : value : null, + detail: status === "identity_changed" ? value : void 0 + }; + } + async close() { + let failure2 = this.#failure; + if (!failure2 && !this.#closed) { + try { + this.#child.stdin.write("close\n"); + const ack = await this.#nextLine(1e4, "close acknowledgement"); + const match = /^CLOSED (\d{1,9}) (\d{1,9})$/u.exec(ack); + if (!match) throw new Error(`process_tree_close_invalid: unexpected close receipt ${JSON.stringify(ack.slice(0, 60))}`); + const failedHandles = Number(match[2]); + if (failedHandles > 0) throw new Error(`process_tree_close_failed: ${String(failedHandles)} CloseHandle call(s) failed`); + this.#closeHandled = true; + } catch (error62) { + failure2 = error62 instanceof Error ? error62 : new Error(String(error62)); + this.#closeHandled = true; + } + } + const exited = await this.#waitForExit(5e3); + if (!exited) { + this.#closeHandled = true; + try { + this.#child.kill("SIGKILL"); + } catch { + } + try { + this.#child.stdin.end(); + } catch { + } + const exitedAfterKill = await this.#waitForExit(3e3); + if (!exitedAfterKill) { + throw new Error(failure2 ? `${failure2.message}; process_tree_close_failed: helper did not exit within bounds` : "process_tree_close_failed: helper did not exit within bounds"); + } + throw new Error(failure2 ? `${failure2.message}; process_tree_close_forced: helper exited only after a forced kill` : "process_tree_close_forced: helper exited only after a forced kill"); + } + if (this.#exitCode !== 0 || this.#exitSignal !== null) { + const abnormal = `process_tree_close_abnormal: helper exited with code ${String(this.#exitCode)}${this.#exitSignal ? `, signal ${this.#exitSignal}` : ""}`; + throw new Error(failure2 ? `${failure2.message}; ${abnormal}` : abnormal); + } + if (failure2) throw failure2; + if (this.#failure) throw this.#failure; + } + #waitForExit(timeoutMs) { + if (this.#closed) return Promise.resolve(true); + return new Promise((resolve) => { + let settled = false; + const finish = (value) => { + if (settled) return; + settled = true; + clearTimeout(timer); + const index = this.#exitWaiters.indexOf(waiter); + if (index >= 0) this.#exitWaiters.splice(index, 1); + resolve(value); + }; + const waiter = () => finish(true); + const timer = setTimeout(() => finish(false), timeoutMs); + if (typeof timer.unref === "function") timer.unref(); + this.#exitWaiters.push(waiter); + }); + } +}; +async function openPowerShellTreeTransaction() { + return PowerShellTreeTransaction.start(); } // src/adapters/process-spawn.ts -function appendBounded(current, chunk, maxBytes) { - if (!(maxBytes > 0)) return { value: current.value + chunk, truncated: current.truncated }; - const remaining = maxBytes - Buffer.byteLength(current.value, "utf8"); - if (remaining <= 0) return chunk ? { value: current.value, truncated: true } : current; - const bytes = Buffer.from(chunk, "utf8"); - if (bytes.length <= remaining) { - return { value: current.value + chunk, truncated: current.truncated }; - } - return { - value: current.value + bytes.subarray(0, remaining).toString("utf8"), - truncated: true - }; -} function isProcessRunning(pid) { if (!Number.isInteger(pid) || pid <= 0) return false; try { @@ -976,27 +1653,14 @@ var terminateProcessTree = async (pid, options = {}) => { throw new Error(`A positive integer PID is required, got: ${pid}`); } if (process.platform === "win32") { - const probe = createPlatformProbe(); - await terminateWindowsProcessTree(pid, options.killWaitMs ?? 2e3, { - capture: async (root) => { - let snapshot; - try { - snapshot = await runCommand("powershell.exe", ["-NoProfile", "-NonInteractive", "-Command", windowsTreeCaptureScript(root)], { - timeoutMs: 5e3, - maxOutputBytes: 128e3 - }); - } catch { - throw new Error("process_tree_snapshot_unverified: query failed"); - } - if (snapshot.code !== 0 || snapshot.stdoutTruncated || snapshot.stderr.length > 0) { - throw new Error("process_tree_snapshot_unverified: query failed or incomplete"); - } - return parseWindowsTreeCapture(snapshot.stdout, root); - }, - probe: (requests) => probe.probe(requests), - kill: async (root) => (await runCommand("taskkill", ["/PID", String(root), "/T", "/F"], { timeoutMs: 15e3 })).code + const expected = options.expectedRootFingerprint ?? null; + if (expected === null) { + throw new Error("process_tree_root_identity_required: Windows process-tree termination requires a caller-bound root identity"); + } + const receipt = await terminateWindowsProcessTree(pid, expected, options.killWaitMs ?? 2e3, { + open: openPowerShellTreeTransaction }); - return { pid, signal: "SIGKILL", verified: true }; + return { pid, signal: "SIGKILL", verified: true, windows: receipt }; } signalProcessTree(pid, "SIGTERM"); if (await waitForProcessTreeExit(pid, options.graceMs ?? 500)) { @@ -1008,47 +1672,32 @@ var terminateProcessTree = async (pid, options = {}) => { } return { pid, signal: "SIGKILL", verified: true }; }; -function runCommand(command, args, options) { - return new Promise((resolve, reject) => { - const child = spawn2(command, args, { - shell: false, - windowsHide: true, - stdio: ["ignore", "pipe", "pipe"] - }); - let stdout = { value: "", truncated: false }; - let stderr = { value: "", truncated: false }; - child.stdout?.setEncoding("utf8"); - child.stderr?.setEncoding("utf8"); - child.stdout?.on("data", (chunk) => { - stdout = appendBounded(stdout, chunk, options.maxOutputBytes ?? 1e6); - }); - child.stderr?.on("data", (chunk) => { - stderr = appendBounded(stderr, chunk, options.maxOutputBytes ?? 1e6); - }); - let timedOut = false; - const timer = options.timeoutMs && options.timeoutMs > 0 ? setTimeout(() => { - timedOut = true; - try { - child.kill("SIGKILL"); - } catch { - } - }, options.timeoutMs) : null; - timer?.unref(); - child.on("error", (error62) => { - if (timer) clearTimeout(timer); - reject(error62); - }); - child.on("close", (code, signal) => { - if (timer) clearTimeout(timer); - resolve({ - code: timedOut ? null : code, - signal: timedOut ? "SIGKILL" : signal, - stdout: stdout.value, - stderr: stderr.value, - stdoutTruncated: stdout.truncated - }); - }); - }); +function childStillOnSpawnGeneration(child) { + if (typeof child.pid !== "number" || !Number.isInteger(child.pid) || child.pid <= 0) return false; + if (child.exitSeen === true) return false; + return child.exitCode === null && child.signalCode === null; +} +async function captureSpawnGenerationIdentity(child, probe, options = {}) { + const pid = child.pid; + if (typeof pid !== "number" || !Number.isInteger(pid) || pid <= 0) return null; + const read = async () => { + let identity; + try { + identity = await probe.identityOf(pid); + } catch { + return null; + } + if (!childStillOnSpawnGeneration(child)) return null; + if (typeof identity.fingerprint !== "string" || identity.fingerprint.length === 0) return null; + if (identity.fingerprint_precision !== "exact") return null; + if (child.pid !== pid) return null; + return { pid, fingerprint: identity.fingerprint }; + }; + const first = await read(); + if (first !== null) return first; + await new Promise((resolve) => setTimeout(resolve, options.retryDelayMs ?? 250)); + if (!childStillOnSpawnGeneration(child)) return null; + return read(); } function signalProcessTree(pid, signal) { try { @@ -22112,6 +22761,7 @@ var ZCodeAppServerAdapter = class { #turnBindingCompatMs; #replayIdleMs; #browser; + #probe; #runs = /* @__PURE__ */ new Map(); #workspaceByTask = /* @__PURE__ */ new Map(); constructor(options = {}) { @@ -22122,6 +22772,7 @@ var ZCodeAppServerAdapter = class { this.#homeDir = options.homeDir ?? homedir3(); this.#host = validateHostProfile(options.host ?? codexHostProfile(this.#homeDir)); this.#now = options.now ?? (() => /* @__PURE__ */ new Date()); + this.#probe = options.probe ?? createPlatformProbe(); this.#resolveInteraction = options.resolveInteraction; this.#onOutcomeCheckpoint = options.onOutcomeCheckpoint; this.#onInteractionState = options.onInteractionState; @@ -22197,6 +22848,8 @@ var ZCodeAppServerAdapter = class { const entry = { handle, child: null, + childIdentity: null, + childExitSeen: false, client: null, sessionId: resumeSessionId, finished: false, @@ -22688,7 +23341,7 @@ var ZCodeAppServerAdapter = class { await this.#closeBrowserHost(entry); try { if (entry.client) await entry.client.close(); - else if (entry.child?.pid) await terminateProcessTree(entry.child.pid); + else if (entry.child?.pid) await this.#terminateOwnedChild(entry); } catch (cleanupError) { const browserSuffix = entry.browserCleanupError ? ` Managed browser cleanup is also unverified: ${entry.browserCleanupError}` : ""; throw new BridgeError("cleanup_failed", `ZCode process termination could not be verified; workspace remains occupied.${browserSuffix}`, { cause: cleanupError }); @@ -22728,6 +23381,28 @@ var ZCodeAppServerAdapter = class { let stdoutBuffer = ""; let rpcId = 0; let closed = false; + child.once("exit", () => { + entry.childExitSeen = true; + }); + const generationChild = { + get pid() { + return child.pid; + }, + get exitCode() { + return child.exitCode; + }, + get signalCode() { + return child.signalCode; + }, + get exitSeen() { + return entry.childExitSeen; + } + }; + if (process.platform === "win32" && child.pid) { + void captureSpawnGenerationIdentity(generationChild, this.#probe).then((identity) => { + entry.childIdentity = identity; + }); + } const pending = /* @__PURE__ */ new Map(); let protocolFatal = null; const failProtocol = (code, frameBytes, cause, detail) => { @@ -22869,12 +23544,31 @@ var ZCodeAppServerAdapter = class { if (process.platform !== "win32" && child.pid) await terminateProcessTree(child.pid); return; } - if (child.pid && isProcessRunning(child.pid)) await terminateProcessTree(child.pid); + if (child.pid && isProcessRunning(child.pid)) { + if (entry.childExitSeen) return; + await terminateProcessTree(child.pid, { + graceMs: 500, + killWaitMs: 5e3, + expectedRootFingerprint: entry.childIdentity?.pid === child.pid ? entry.childIdentity.fingerprint : null + }); + } }; return { child, request, replayEvents, close, get stderr() { return stderr; } }; } + /** ADR-009: terminate the adapter's own spawned child with the identity + * captured while its native handle was still held. Windows refuses to + * signal without it (a bare current PID is never trusted), and a child + * whose 'exit' already fired is never signalled again. POSIX keeps its + * existing process-group behaviour. */ + async #terminateOwnedChild(entry) { + const pid = entry.child?.pid; + if (!pid) return; + if (entry.childExitSeen) return; + const fingerprint = entry.childIdentity?.pid === pid ? entry.childIdentity.fingerprint : null; + await terminateProcessTree(pid, { graceMs: 500, killWaitMs: 5e3, expectedRootFingerprint: fingerprint }); + } #handleMessage(message, entry, pending, config2, write) { if (message.method === "session/requestRuntimePreferences") { const id = message.id; @@ -23563,11 +24257,25 @@ import { existsSync as existsSync5, mkdirSync, readFileSync as readFileSync3, re import { randomUUID } from "node:crypto"; import path6 from "node:path"; var selfIdentity = null; +var TRANSIENT_LOCK_FS_CODES = /* @__PURE__ */ new Set(["EEXIST", "EPERM", "EACCES", "EBUSY", "ENOTEMPTY"]); +var DESTINATION_PRESENT_CONTENTION_CODES = /* @__PURE__ */ new Set(["EEXIST", "ENOTEMPTY"]); +var PUBLICATION_RETRY_ATTEMPTS = 3; +var WITHDRAW_RETRY_MS = 2e3; +var SHARED_GUARD_RETRY_MS = 2e3; +var PRIVATE_CLEANUP_RETRY_MS = 1e3; +var RETRY_PAUSE_MS = 10; +function isTransientLockFsError(error62) { + return TRANSIENT_LOCK_FS_CODES.has(error62?.code ?? ""); +} +function pauseBriefly() { + Atomics.wait(new Int32Array(new SharedArrayBuffer(4)), 0, 0, RETRY_PAUSE_MS); +} function tryAcquireProcessLock(directory) { return tryAcquireProcessLockWithRetry(directory, 0); } function tryAcquireProcessLockWithRetry(directory, retries) { const token = randomUUID(); + if (existsSync5(directory)) return acquireExistingLock(directory); const staging = `${directory}.${process.pid}.${token}.pending`; const ownerRecord = { pid: process.pid, @@ -23584,14 +24292,16 @@ function tryAcquireProcessLockWithRetry(directory, retries) { renameSync(staging, directory); } catch (error62) { removeStagedLock(staging); - if (!existsSync5(directory)) { - const code = error62.code; - if (retries < 3 && (code === "EEXIST" || code === "EPERM" || code === "EACCES")) { - return tryAcquireProcessLockWithRetry(directory, retries + 1); - } - throw error62; + const code = error62.code; + if (!isTransientLockFsError(error62)) throw error62; + if (DESTINATION_PRESENT_CONTENTION_CODES.has(code ?? "") || existsSync5(directory)) { + return acquireExistingLock(directory); + } + if (retries < PUBLICATION_RETRY_ATTEMPTS) { + pauseBriefly(); + return tryAcquireProcessLockWithRetry(directory, retries + 1); } - return acquireExistingLock(directory); + throw error62; } } catch (error62) { removeStagedLock(staging); @@ -23601,38 +24311,72 @@ function tryAcquireProcessLockWithRetry(directory, retries) { const owner = JSON.parse(readFileSync3(path6.join(directory, "owner.json"), "utf8")); if (owner.token !== token) throw new Error("process lock ownership changed"); const retired = `${directory}.${token}.retired`; - renameSync(directory, retired); + renameWithTransientRetry(directory, retired); removeStagedLock(retired); }; } +function renameWithTransientRetry(source, target) { + const deadline = Date.now() + WITHDRAW_RETRY_MS; + while (true) { + try { + renameSync(source, target); + return; + } catch (error62) { + if (!isTransientLockFsError(error62) || Date.now() >= deadline) throw error62; + pauseBriefly(); + } + } +} function acquireExistingLock(directory) { + let owner; try { - const owner = JSON.parse(readFileSync3(path6.join(directory, "owner.json"), "utf8")); + owner = JSON.parse(readFileSync3(path6.join(directory, "owner.json"), "utf8")); if (!Number.isSafeInteger(owner.pid) || owner.pid <= 0) throw new Error("invalid lock owner"); - try { - process.kill(owner.pid, 0); - } catch (failure2) { - if (failure2.code === "ESRCH") { - if (reclaimDeadOwner(directory)) return tryAcquireProcessLock(directory); - } - } } catch { try { if (Date.now() - statSync2(directory).mtimeMs > 3e4) throw new Error(`unreadable lock owner: ${directory}`); } catch (failure2) { if (failure2.code !== "ENOENT") throw failure2; } + return null; + } + try { + process.kill(owner.pid, 0); + } catch (failure2) { + if (failure2.code === "ESRCH") { + if (reclaimDeadOwner(directory)) return tryAcquireProcessLock(directory); + } } return null; } function removeStagedLock(directory) { - try { - unlinkSync(path6.join(directory, "owner.json")); - } catch { + removePrivateFile(path6.join(directory, "owner.json")); + removePrivateDirectory(directory); +} +function removePrivateFile(file2) { + const deadline = Date.now() + PRIVATE_CLEANUP_RETRY_MS; + while (true) { + try { + unlinkSync(file2); + return; + } catch (error62) { + const code = error62?.code; + if (code === "ENOENT" || !isTransientLockFsError(error62) || Date.now() >= deadline) return; + pauseBriefly(); + } } - try { - rmdirSync(directory); - } catch { +} +function removePrivateDirectory(directory) { + const deadline = Date.now() + PRIVATE_CLEANUP_RETRY_MS; + while (true) { + try { + rmdirSync(directory); + return; + } catch (error62) { + const code = error62?.code; + if (code === "ENOENT" || !isTransientLockFsError(error62) || Date.now() >= deadline) return; + pauseBriefly(); + } } } function reclaimDeadOwner(directory) { @@ -23643,24 +24387,56 @@ function reclaimDeadOwner(directory) { if (error62.code === "EEXIST") return false; throw error62; } + let outcome; try { const owner = JSON.parse(readFileSync3(path6.join(directory, "owner.json"), "utf8")); - if (!Number.isSafeInteger(owner.pid) || owner.pid <= 0) return false; + if (!Number.isSafeInteger(owner.pid) || owner.pid <= 0) { + outcome = { kind: "busy" }; + } else { + try { + process.kill(owner.pid, 0); + outcome = { kind: "busy" }; + } catch (failure2) { + if (failure2.code !== "ESRCH") { + outcome = { kind: "busy" }; + } else { + const retired = `${directory}.${randomUUID()}.retired`; + renameWithTransientRetry(directory, retired); + removeStagedLock(retired); + outcome = { kind: "reclaimed" }; + } + } + } + } catch (error62) { + outcome = error62.code === "ENOENT" ? { kind: "busy" } : { kind: "error", error: error62 }; + } + let guardCleanupError = null; + try { + removeSharedGuard(guard); + } catch (error62) { + guardCleanupError = error62; + } + if (outcome.kind === "error") throw outcome.error; + if (guardCleanupError !== null) throw guardCleanupError; + return outcome.kind === "reclaimed"; +} +function removeSharedGuard(guard) { + const deadline = Date.now() + SHARED_GUARD_RETRY_MS; + while (true) { try { - process.kill(owner.pid, 0); - return false; + rmdirSync(guard); + return; } catch (error62) { - if (error62.code !== "ESRCH") return false; + const code = error62?.code; + if (code === "ENOENT") return; + if (!isTransientLockFsError(error62) || Date.now() >= deadline) { + throw new Error( + `could not remove the lock reclaim guard at ${guard} (${code ?? "unknown error"}); a leftover guard blocks all future dead-owner reclaims of this lock and needs manual diagnosis`, + { cause: error62 } + ); + } + pauseBriefly(); } - const retired = `${directory}.${randomUUID()}.retired`; - renameSync(directory, retired); - removeStagedLock(retired); - return true; - } catch (error62) { - if (error62.code !== "ENOENT") throw error62; - return false; - } finally { - rmdirSync(guard); } } @@ -23854,7 +24630,14 @@ var TaskStore = class { this.assertWorkerAttempt(taskId2, attempt); this.#writeJsonAtomic(path7.join(this.taskDir(taskId2), "result.json"), result); const current = this.readStatus(taskId2); - this.#writeJsonAtomic(path7.join(this.taskDir(taskId2), "status.json"), { ...current, ...patch, task_id: taskId2, attempt, updated_at: (/* @__PURE__ */ new Date()).toISOString() }); + let effective = patch; + if (current.cleanup_unverified === true) { + effective = { ...patch, cleanup_unverified: true }; + if (typeof current.error === "string" && current.error.includes("[cleanup-members:")) { + effective = { ...effective, error: current.error }; + } + } + this.#writeJsonAtomic(path7.join(this.taskDir(taskId2), "status.json"), { ...current, ...effective, task_id: taskId2, attempt, updated_at: (/* @__PURE__ */ new Date()).toISOString() }); }); } patchRunningAttempt(taskId2, attempt, patch) { @@ -24857,18 +25640,18 @@ async function runWorkerTask(options) { } }; if (initialStatus.cancel_requested === true) { - const finishedAt2 = now().toISOString(); - const result2 = buildTaskResult({ + const finishedAt = now().toISOString(); + const result = buildTaskResult({ task, attempt, startedAt: initialStatus.started_at, - finishedAt: finishedAt2, + finishedAt, outcome: null, failure: { code: "cancelled", message: "cancelled by request before the worker started the agent" }, cancelled: true }); - store.commitWorkerResult(taskId2, attempt, result2, { status: "cancelled", finished_at: finishedAt2, worker_pid: null }); - return { status: result2.status, result: result2 }; + store.commitWorkerResult(taskId2, attempt, result, { status: "cancelled", finished_at: finishedAt, worker_pid: null }); + return { status: result.status, result }; } const startedAt = initialStatus.started_at ?? now().toISOString(); store.writeStatus(taskId2, { status: "running", started_at: startedAt, worker_pid: process.pid }, attempt); @@ -24878,8 +25661,22 @@ async function runWorkerTask(options) { started_at: startedAt }); writePhase(); - void probe.selfIdentity().then( + const IDENTITY_WRITE_REAP_MS = 2e3; + const identityWrites = /* @__PURE__ */ new Set(); + let identityWritesClosed = false; + const trackIdentityWrite = (write) => { + identityWrites.add(write); + void write.then(() => identityWrites.delete(write), () => identityWrites.delete(write)); + }; + const reapIdentityWrites = async () => { + if (identityWritesClosed) return; + const deadline = Date.now() + IDENTITY_WRITE_REAP_MS; + while (identityWrites.size > 0 && Date.now() < deadline) await sleep2(25); + identityWritesClosed = true; + }; + trackIdentityWrite(probe.selfIdentity().then( (identity) => { + if (identityWritesClosed) return; store.writeExecutorIdentity(taskId2, attempt, { worker: identity }); observationRevision += 1; try { @@ -24888,421 +25685,429 @@ async function runWorkerTask(options) { } }, () => void 0 - ); - const heartbeat = { - attempt, - worker_pid: process.pid, - started_at: startedAt, - heartbeat_at: now().toISOString(), - heartbeat_seq: 0, - session_id: initialStatus.zcode_session_id ?? null, - turn_id: null, - last_event_seq: 0, - last_event_type: "worker_running", - zcode_event_seq: 0 - }; - const persistHeartbeat = () => { - heartbeat.heartbeat_at = now().toISOString(); - heartbeat.heartbeat_seq += 1; - store.writeWorkerHeartbeat(taskId2, attempt, heartbeat); - writePhase(); - }; - persistHeartbeat(); - const heartbeatTimer = setInterval(() => { - try { - persistHeartbeat(); - } catch { - } - }, 3e3); - heartbeatTimer.unref(); - const continueSpec = store.readAttemptMeta(taskId2, attempt, "continue.json"); - const previousResult = continueSpec ? store.readArchivedResult(taskId2, continueSpec.previous_attempt ?? attempt - 1) : null; - const promptText = continueSpec ? buildContinuePrompt({ - task, - feedback: continueSpec.feedback, - additionalRequirements: continueSpec.additional_requirements ?? [], - previousSessionId: continueSpec.previous_session_id, - previousResult - }) : buildTaskPrompt(task); - store.writeAttemptFile(taskId2, attempt, "prompt.txt", promptText); - const correlation = { - rpc_accepted_at: null, - turn_started_at: null, - turn_completed_at: null, - turn_id: null, - session_id: initialStatus.zcode_session_id ?? null - }; - const persistCorrelation = () => { - try { - store.writeAttemptMeta(taskId2, attempt, "turn-correlation.json", { ...correlation }); - } catch { - } - }; - const persistCorrelationTimer = setInterval(persistCorrelation, 5e3); - persistCorrelationTimer.unref(); - const modelProfile = { - executor: "zcode", - provider_id: null, - model_id: null, - requested_model: task.model ? `${task.model.provider_id}/${task.model.model_id}` : null, - requested_reasoning_level: task.model?.reasoning_level ?? null, - effective_reasoning_level: null, - effective_reasoning_level_source: "not_reported", - selection_source: null, - effective_at: null, - session_id: initialStatus.zcode_session_id ?? null, - turn_id: null - }; - let outcome = null; - let failure2 = null; - let pendingModelOutput = ""; - let lastModelOutputAt = 0; - const flushModelOutput = () => { - if (!pendingModelOutput) return; - store.assertWorkerAttempt(taskId2, attempt); - for (let offset = 0; offset < pendingModelOutput.length; offset += 2e3) store.appendEvent(taskId2, "model_output", pendingModelOutput.slice(offset, offset + 2e3)); - pendingModelOutput = ""; - lastModelOutputAt = Date.now(); - }; + )); + let heartbeatTimer; + let persistCorrelationTimer; try { - if (options.resolver) { - await options.resolver.resolve(); - } - const adapter = options.adapter ?? new ZCodeAppServerAdapter({ - host: options.host, - ...options.resolver ? { resolver: options.resolver } : {}, - onEvent: (event) => { - store.assertWorkerAttempt(taskId2, attempt); - if (event.type === "model_output") { - pendingModelOutput += event.summary; - if (pendingModelOutput.length >= 4e3 || Date.now() - lastModelOutputAt >= 500) flushModelOutput(); - return; - } - flushModelOutput(); - const persistedEvent = store.appendEvent(taskId2, event.type, event.summary, event.details); - if (persistedEvent) { - heartbeat.last_event_seq = persistedEvent.seq; - heartbeat.last_event_type = event.type; - } - const sessionId = event.type === "session_ready" ? event.details?.["session_id"] : void 0; - if (typeof sessionId === "string") { - heartbeat.session_id = sessionId; - correlation.session_id = sessionId; - modelProfile.session_id = sessionId; - store.writeStatus(taskId2, { zcode_session_id: sessionId }, attempt); - } - if (event.type === "turn_started") { - correlation.rpc_accepted_at = correlation.rpc_accepted_at ?? now().toISOString(); - correlation.turn_started_at = now().toISOString(); - if (typeof event.details?.["turn_id"] === "string") { - heartbeat.turn_id = event.details["turn_id"]; - correlation.turn_id = event.details["turn_id"]; - modelProfile.turn_id = event.details["turn_id"]; - } - activityPhase = "executing"; - writePhase(); - } - if (event.type === "turn_completed") { - correlation.turn_completed_at = now().toISOString(); - activityPhase = "finalizing"; - writePhase(); - } - if (event.type === "model_selected") { - if (typeof event.details?.["provider_id"] === "string") modelProfile.provider_id = event.details["provider_id"]; - if (typeof event.details?.["model_id"] === "string") modelProfile.model_id = event.details["model_id"]; - if (typeof event.details?.["selected_model"] === "string" && modelProfile.model_id === null) { - modelProfile.model_id = event.details["selected_model"]; + const heartbeat = { + attempt, + worker_pid: process.pid, + started_at: startedAt, + heartbeat_at: now().toISOString(), + heartbeat_seq: 0, + session_id: initialStatus.zcode_session_id ?? null, + turn_id: null, + last_event_seq: 0, + last_event_type: "worker_running", + zcode_event_seq: 0 + }; + const persistHeartbeat = () => { + heartbeat.heartbeat_at = now().toISOString(); + heartbeat.heartbeat_seq += 1; + store.writeWorkerHeartbeat(taskId2, attempt, heartbeat); + writePhase(); + }; + persistHeartbeat(); + heartbeatTimer = setInterval(() => { + try { + persistHeartbeat(); + } catch { + } + }, 3e3); + heartbeatTimer.unref(); + const continueSpec = store.readAttemptMeta(taskId2, attempt, "continue.json"); + const previousResult = continueSpec ? store.readArchivedResult(taskId2, continueSpec.previous_attempt ?? attempt - 1) : null; + const promptText = continueSpec ? buildContinuePrompt({ + task, + feedback: continueSpec.feedback, + additionalRequirements: continueSpec.additional_requirements ?? [], + previousSessionId: continueSpec.previous_session_id, + previousResult + }) : buildTaskPrompt(task); + store.writeAttemptFile(taskId2, attempt, "prompt.txt", promptText); + const correlation = { + rpc_accepted_at: null, + turn_started_at: null, + turn_completed_at: null, + turn_id: null, + session_id: initialStatus.zcode_session_id ?? null + }; + const persistCorrelation = () => { + try { + store.writeAttemptMeta(taskId2, attempt, "turn-correlation.json", { ...correlation }); + } catch { + } + }; + persistCorrelationTimer = setInterval(persistCorrelation, 5e3); + persistCorrelationTimer.unref(); + const modelProfile = { + executor: "zcode", + provider_id: null, + model_id: null, + requested_model: task.model ? `${task.model.provider_id}/${task.model.model_id}` : null, + requested_reasoning_level: task.model?.reasoning_level ?? null, + effective_reasoning_level: null, + effective_reasoning_level_source: "not_reported", + selection_source: null, + effective_at: null, + session_id: initialStatus.zcode_session_id ?? null, + turn_id: null + }; + let outcome = null; + let failure2 = null; + let pendingModelOutput = ""; + let lastModelOutputAt = 0; + const flushModelOutput = () => { + if (!pendingModelOutput) return; + store.assertWorkerAttempt(taskId2, attempt); + for (let offset = 0; offset < pendingModelOutput.length; offset += 2e3) store.appendEvent(taskId2, "model_output", pendingModelOutput.slice(offset, offset + 2e3)); + pendingModelOutput = ""; + lastModelOutputAt = Date.now(); + }; + try { + if (options.resolver) { + await options.resolver.resolve(); + } + const adapter = options.adapter ?? new ZCodeAppServerAdapter({ + host: options.host, + ...options.resolver ? { resolver: options.resolver } : {}, + onEvent: (event) => { + store.assertWorkerAttempt(taskId2, attempt); + if (event.type === "model_output") { + pendingModelOutput += event.summary; + if (pendingModelOutput.length >= 4e3 || Date.now() - lastModelOutputAt >= 500) flushModelOutput(); + return; } - if (typeof event.details?.["reasoning_level"] === "string") { - modelProfile.effective_reasoning_level = event.details["reasoning_level"]; - modelProfile.effective_reasoning_level_source = "runtime"; + flushModelOutput(); + const persistedEvent = store.appendEvent(taskId2, event.type, event.summary, event.details); + if (persistedEvent) { + heartbeat.last_event_seq = persistedEvent.seq; + heartbeat.last_event_type = event.type; } - if (typeof event.details?.["model_source"] === "string") modelProfile.selection_source = event.details["model_source"]; - modelProfile.effective_at = now().toISOString(); - } - if (typeof event.details?.["event_seq"] === "number") heartbeat.zcode_event_seq = event.details["event_seq"]; - if (event.type === "app_server_started" && typeof event.details?.["pid"] === "number") { - store.writeStatus(taskId2, { zcode_pid: event.details["pid"] }, attempt); - void probe.identityOf(event.details["pid"]).then( - (identity) => { - try { - store.writeExecutorIdentity(taskId2, attempt, { runtime: identity }); - observationRevision += 1; - store.writeWorkerObservation(taskId2, attempt, process.pid, { runtime_identity: identity }, observationRevision, now().toISOString()); - } catch { - } - }, - () => void 0 - ); - } - }, - onInteractionState: (state) => { - activityPhase = state === "waiting" ? "waiting_for_permission" : "executing"; - writePhase(); - }, - onOutcomeCheckpoint: (checkpoint) => { - store.writeAttemptMeta(taskId2, attempt, "outcome-checkpoint.json", { - recorded_at: now().toISOString(), - exit_code: checkpoint.exitCode, - signal: checkpoint.signal, - session_id: checkpoint.sessionId, - response: checkpoint.response, - usage: checkpoint.usage, - error_code: checkpoint.errorCode, - report_error: checkpoint.reportError, - agent_report: checkpoint.agentReport, - report_candidate: checkpoint.reportCandidate, - cleanup_error: checkpoint.cleanupError ?? null, - cleanup_verified: checkpoint.cleanupVerified === true, - usage_normalized: normalizeUsage(checkpoint.usage, { source: "zcode_runtime_turn", scope: checkpoint.sessionId ? `session:${checkpoint.sessionId}` : null, observedAt: now().toISOString() }), - model_profile: modelProfile - }); - }, - resolveInteraction: async (request, signal) => { - store.assertWorkerAttempt(taskId2, attempt); - const safeRequest = sanitizeInteractionRequest({ ...request, request_id: `${attempt}:${request.request_id}` }); - let record2; - let created = false; - try { - const written = store.writeInteractionRequest(taskId2, safeRequest, now().toISOString()); - record2 = written.record; - created = written.created; - } catch (error62) { - if (error62 instanceof InteractionRequestTooLargeError) { - store.appendEvent(taskId2, "interaction_request_oversized", `ZCode requested approval of a tool input too large to deliver: ${error62.actualBytes} UTF-8 bytes, over the ${error62.limitBytes}-byte limit; the request was declined and nothing was executed`, { - error_code: "interaction_params_over_limit", - request_id: safeRequest.request_id, - method: safeRequest.method, - actual_bytes: error62.actualBytes, - limit_bytes: error62.limitBytes - }, now().toISOString()); - return interactionDecline2( - request.method, - `Bridge cannot deliver this approval request: the tool input is ${error62.actualBytes} UTF-8 bytes, over the ${error62.limitBytes}-byte limit. Split the input into smaller parts (for example write the file in multiple Write or append steps) and retry. Nothing was executed.` - ); + const sessionId = event.type === "session_ready" ? event.details?.["session_id"] : void 0; + if (typeof sessionId === "string") { + heartbeat.session_id = sessionId; + correlation.session_id = sessionId; + modelProfile.session_id = sessionId; + store.writeStatus(taskId2, { zcode_session_id: sessionId }, attempt); } - store.appendEvent(taskId2, "interaction_request_undeliverable", "ZCode requested approval but the Bridge could not persist the request for the calling host; the request was declined and nothing was executed", { - error_code: "interaction_persist_failed", - request_id: safeRequest.request_id, - method: safeRequest.method - }, now().toISOString()); - return interactionDecline2( - request.method, - `Bridge could not persist this approval request for the calling host (${safeErrorSummary(error62)}); the request was declined and nothing was executed. Split the input into smaller parts and retry.` - ); - } - activityPhase = request.method === "interaction/requestPermission" ? "waiting_for_permission" : "waiting_for_user"; - writePhase(); - if (created) { - const published = publishInteractionRequest(store, taskId2, safeRequest, record2); - if (!published.event) { - store.appendEvent(taskId2, "interaction_request_undeliverable", `ZCode requested approval but the Bridge could not publish it to the calling host (${published.declineReason}); the request was declined and nothing was executed`, { - error_code: "interaction_publish_failed", - request_id: safeRequest.request_id, - method: safeRequest.method - }, now().toISOString()); - const fallback = interactionDecline2(request.method, published.declineReason); - store.answerInteractionRequest(taskId2, safeRequest.request_id, fallback, now().toISOString()); + if (event.type === "turn_started") { + correlation.rpc_accepted_at = correlation.rpc_accepted_at ?? now().toISOString(); + correlation.turn_started_at = now().toISOString(); + if (typeof event.details?.["turn_id"] === "string") { + heartbeat.turn_id = event.details["turn_id"]; + correlation.turn_id = event.details["turn_id"]; + modelProfile.turn_id = event.details["turn_id"]; + } activityPhase = "executing"; writePhase(); - return fallback; } - } - let answer = null; - while (!signal.aborted) { + if (event.type === "turn_completed") { + correlation.turn_completed_at = now().toISOString(); + activityPhase = "finalizing"; + writePhase(); + } + if (event.type === "model_selected") { + if (typeof event.details?.["provider_id"] === "string") modelProfile.provider_id = event.details["provider_id"]; + if (typeof event.details?.["model_id"] === "string") modelProfile.model_id = event.details["model_id"]; + if (typeof event.details?.["selected_model"] === "string" && modelProfile.model_id === null) { + modelProfile.model_id = event.details["selected_model"]; + } + if (typeof event.details?.["reasoning_level"] === "string") { + modelProfile.effective_reasoning_level = event.details["reasoning_level"]; + modelProfile.effective_reasoning_level_source = "runtime"; + } + if (typeof event.details?.["model_source"] === "string") modelProfile.selection_source = event.details["model_source"]; + modelProfile.effective_at = now().toISOString(); + } + if (typeof event.details?.["event_seq"] === "number") heartbeat.zcode_event_seq = event.details["event_seq"]; + if (event.type === "app_server_started" && typeof event.details?.["pid"] === "number") { + store.writeStatus(taskId2, { zcode_pid: event.details["pid"] }, attempt); + trackIdentityWrite(probe.identityOf(event.details["pid"]).then( + (identity) => { + if (identityWritesClosed) return; + try { + store.writeExecutorIdentity(taskId2, attempt, { runtime: identity }); + observationRevision += 1; + store.writeWorkerObservation(taskId2, attempt, process.pid, { runtime_identity: identity }, observationRevision, now().toISOString()); + } catch { + } + }, + () => void 0 + )); + } + }, + onInteractionState: (state) => { + activityPhase = state === "waiting" ? "waiting_for_permission" : "executing"; + writePhase(); + }, + onOutcomeCheckpoint: (checkpoint) => { + store.writeAttemptMeta(taskId2, attempt, "outcome-checkpoint.json", { + recorded_at: now().toISOString(), + exit_code: checkpoint.exitCode, + signal: checkpoint.signal, + session_id: checkpoint.sessionId, + response: checkpoint.response, + usage: checkpoint.usage, + error_code: checkpoint.errorCode, + report_error: checkpoint.reportError, + agent_report: checkpoint.agentReport, + report_candidate: checkpoint.reportCandidate, + cleanup_error: checkpoint.cleanupError ?? null, + cleanup_verified: checkpoint.cleanupVerified === true, + usage_normalized: normalizeUsage(checkpoint.usage, { source: "zcode_runtime_turn", scope: checkpoint.sessionId ? `session:${checkpoint.sessionId}` : null, observedAt: now().toISOString() }), + model_profile: modelProfile + }); + }, + resolveInteraction: async (request, signal) => { store.assertWorkerAttempt(taskId2, attempt); - let current; + const safeRequest = sanitizeInteractionRequest({ ...request, request_id: `${attempt}:${request.request_id}` }); + let record2; + let created = false; try { - current = store.readInteractionRequest(taskId2, safeRequest.request_id); + const written = store.writeInteractionRequest(taskId2, safeRequest, now().toISOString()); + record2 = written.record; + created = written.created; } catch (error62) { - activityPhase = "executing"; - writePhase(); - store.appendEvent(taskId2, "interaction_request_undeliverable", "The persisted ZCode interaction request became unreadable while waiting for the calling host; the request was declined and nothing was executed", { - error_code: "interaction_record_unreadable", + if (error62 instanceof InteractionRequestTooLargeError) { + store.appendEvent(taskId2, "interaction_request_oversized", `ZCode requested approval of a tool input too large to deliver: ${error62.actualBytes} UTF-8 bytes, over the ${error62.limitBytes}-byte limit; the request was declined and nothing was executed`, { + error_code: "interaction_params_over_limit", + request_id: safeRequest.request_id, + method: safeRequest.method, + actual_bytes: error62.actualBytes, + limit_bytes: error62.limitBytes + }, now().toISOString()); + return interactionDecline2( + request.method, + `Bridge cannot deliver this approval request: the tool input is ${error62.actualBytes} UTF-8 bytes, over the ${error62.limitBytes}-byte limit. Split the input into smaller parts (for example write the file in multiple Write or append steps) and retry. Nothing was executed.` + ); + } + store.appendEvent(taskId2, "interaction_request_undeliverable", "ZCode requested approval but the Bridge could not persist the request for the calling host; the request was declined and nothing was executed", { + error_code: "interaction_persist_failed", request_id: safeRequest.request_id, method: safeRequest.method }, now().toISOString()); return interactionDecline2( request.method, - `Bridge can no longer read the full content of this approval request (${safeErrorSummary(error62)}); the request was declined and nothing was executed.` + `Bridge could not persist this approval request for the calling host (${safeErrorSummary(error62)}); the request was declined and nothing was executed. Split the input into smaller parts and retry.` ); } - if (!current) { - activityPhase = "executing"; - writePhase(); - store.appendEvent(taskId2, "interaction_request_undeliverable", "The persisted ZCode interaction request disappeared while waiting for the calling host; the request was declined and nothing was executed", { - error_code: "interaction_record_missing", - request_id: safeRequest.request_id, - method: safeRequest.method - }, now().toISOString()); - return interactionDecline2( - request.method, - "Bridge can no longer read the full content of this approval request (the persisted record disappeared); the request was declined and nothing was executed." - ); + activityPhase = request.method === "interaction/requestPermission" ? "waiting_for_permission" : "waiting_for_user"; + writePhase(); + if (created) { + const published = publishInteractionRequest(store, taskId2, safeRequest, record2); + if (!published.event) { + store.appendEvent(taskId2, "interaction_request_undeliverable", `ZCode requested approval but the Bridge could not publish it to the calling host (${published.declineReason}); the request was declined and nothing was executed`, { + error_code: "interaction_publish_failed", + request_id: safeRequest.request_id, + method: safeRequest.method + }, now().toISOString()); + const fallback = interactionDecline2(request.method, published.declineReason); + store.answerInteractionRequest(taskId2, safeRequest.request_id, fallback, now().toISOString()); + activityPhase = "executing"; + writePhase(); + return fallback; + } } - if (current.state === "answered" && current.answer) { - if (current.method !== safeRequest.method || stableJson(current.params) !== stableJson(safeRequest.params)) { + let answer = null; + while (!signal.aborted) { + store.assertWorkerAttempt(taskId2, attempt); + let current; + try { + current = store.readInteractionRequest(taskId2, safeRequest.request_id); + } catch (error62) { activityPhase = "executing"; writePhase(); - store.appendEvent(taskId2, "interaction_request_undeliverable", "The persisted ZCode interaction request no longer matches the request ZCode sent; the request was declined and nothing was executed", { - error_code: "interaction_record_mismatch", + store.appendEvent(taskId2, "interaction_request_undeliverable", "The persisted ZCode interaction request became unreadable while waiting for the calling host; the request was declined and nothing was executed", { + error_code: "interaction_record_unreadable", request_id: safeRequest.request_id, method: safeRequest.method }, now().toISOString()); return interactionDecline2( request.method, - "Bridge can no longer read the full content of this approval request (the persisted record no longer matches the request ZCode sent); the request was declined and nothing was executed." + `Bridge can no longer read the full content of this approval request (${safeErrorSummary(error62)}); the request was declined and nothing was executed.` ); } - answer = current.answer; - break; + if (!current) { + activityPhase = "executing"; + writePhase(); + store.appendEvent(taskId2, "interaction_request_undeliverable", "The persisted ZCode interaction request disappeared while waiting for the calling host; the request was declined and nothing was executed", { + error_code: "interaction_record_missing", + request_id: safeRequest.request_id, + method: safeRequest.method + }, now().toISOString()); + return interactionDecline2( + request.method, + "Bridge can no longer read the full content of this approval request (the persisted record disappeared); the request was declined and nothing was executed." + ); + } + if (current.state === "answered" && current.answer) { + if (current.method !== safeRequest.method || stableJson(current.params) !== stableJson(safeRequest.params)) { + activityPhase = "executing"; + writePhase(); + store.appendEvent(taskId2, "interaction_request_undeliverable", "The persisted ZCode interaction request no longer matches the request ZCode sent; the request was declined and nothing was executed", { + error_code: "interaction_record_mismatch", + request_id: safeRequest.request_id, + method: safeRequest.method + }, now().toISOString()); + return interactionDecline2( + request.method, + "Bridge can no longer read the full content of this approval request (the persisted record no longer matches the request ZCode sent); the request was declined and nothing was executed." + ); + } + answer = current.answer; + break; + } + await sleep2(250); } - await sleep2(250); + activityPhase = "executing"; + writePhase(); + if (answer) return answer; + return interactionDecline2(request.method, "The task attempt ended before the calling host replied; nothing was executed"); } - activityPhase = "executing"; - writePhase(); - if (answer) return answer; - return interactionDecline2(request.method, "The task attempt ended before the calling host replied; nothing was executed"); + }); + const workspaceRef = store.readWorkspaceRef(taskId2) ?? { + requestedPath: task.workspace, + canonicalPath: task.worktree_path ?? task.workspace, + mode: task.worktree_path ? "worktree" : "direct", + ...task.worktree_path ? { sourcePath: task.workspace } : {} + }; + const handle = continueSpec ? await adapter.continueTask({ + task, + workspace: workspaceRef, + attempt, + feedback: continueSpec.feedback, + additionalRequirements: [...continueSpec.additional_requirements ?? []], + previousSessionId: continueSpec.previous_session_id, + previousResult + }) : await adapter.startTask({ task, workspace: workspaceRef, attempt }); + let cancelTimer; + const cancellation = new Promise((_resolve, reject) => { + cancelTimer = setInterval(() => { + try { + const current = store.readStatus(taskId2); + if (current.attempt !== attempt || !current.cancel_requested) return; + clearInterval(cancelTimer); + void adapter.cancelTask(handle).catch(reject); + } catch (error62) { + clearInterval(cancelTimer); + reject(error62); + } + }, 50); + }); + try { + outcome = await Promise.race([adapter.getResult(handle), cancellation]); + } finally { + clearInterval(cancelTimer); } - }); - const workspaceRef = store.readWorkspaceRef(taskId2) ?? { - requestedPath: task.workspace, - canonicalPath: task.worktree_path ?? task.workspace, - mode: task.worktree_path ? "worktree" : "direct", - ...task.worktree_path ? { sourcePath: task.workspace } : {} + if (outcome.cleanupError) failure2 = { code: "cleanup_failed", message: `ZCode returned a turn result, but process cleanup could not be verified: ${outcome.cleanupError}` }; + flushModelOutput(); + } catch (error62) { + flushModelOutput(); + failure2 = { + code: error62 instanceof BridgeError ? error62.code : "worker_error", + message: error62 instanceof Error ? error62.message : String(error62) + }; + } + await reapIdentityWrites(); + activityPhase = "finalizing"; + writePhase(); + const finishedAt = now().toISOString(); + store.assertWorkerAttempt(taskId2, attempt); + const stdoutLog = store.appendLog(taskId2, "stdout", outcome?.stdout ? `${outcome.stdout} +` : ""); + const stderrLog = store.appendLog( + taskId2, + "stderr", + outcome?.stderr ? `${outcome.stderr} +` : failure2 ? `${failure2.code}: ${failure2.message} +` : "" + ); + if (modelProfile.model_id === null && outcome?.modelProfile?.model_id) { + Object.assign(modelProfile, outcome.modelProfile); + } + correlation.rpc_accepted_at = correlation.rpc_accepted_at ?? outcome?.phaseTimestamps?.rpc_accepted_at ?? null; + correlation.turn_started_at = correlation.turn_started_at ?? outcome?.phaseTimestamps?.turn_started_at ?? null; + correlation.turn_completed_at = correlation.turn_completed_at ?? outcome?.phaseTimestamps?.turn_completed_at ?? null; + if (modelProfile.turn_id === null && correlation.turn_id) modelProfile.turn_id = correlation.turn_id; + const timingNotes = []; + const timing = { + queued_ms: phaseDuration(initialStatus.created_at, startedAt, timingNotes), + execution_ms: phaseDuration(startedAt, finishedAt, timingNotes), + turn_ms: phaseDuration(correlation.turn_started_at, correlation.turn_completed_at, timingNotes), + finalize_ms: phaseDuration(correlation.turn_completed_at, finishedAt, timingNotes), + wall_ms: phaseDuration(startedAt, finishedAt, timingNotes), + derived: false, + notes: timingNotes }; - const handle = continueSpec ? await adapter.continueTask({ + const usage = normalizeUsage(outcome?.usage ?? null, { + source: "zcode_runtime_turn", + scope: outcome?.sessionId ? `session:${outcome.sessionId}` : null, + observedAt: finishedAt + }); + persistCorrelation(); + store.writeAttemptMeta(taskId2, attempt, "outcome.json", { + started_at: startedAt, + finished_at: finishedAt, + failure: failure2, + outcome: outcome ? { + exitCode: outcome.exitCode, + signal: outcome.signal, + sessionId: outcome.sessionId, + response: outcome.response, + usage: outcome.usage, + timedOut: outcome.timedOut, + cancelled: outcome.cancelled, + attempts: outcome.attempts, + errorCode: outcome.errorCode, + reportError: outcome.reportError, + stdoutTruncated: outcome.stdoutTruncated, + stderrTruncated: outcome.stderrTruncated, + agentReport: outcome.agentReport, + reportCandidate: outcome.reportCandidate, + cleanupError: outcome.cleanupError ?? null, + cleanupVerified: outcome.cleanupVerified === true + } : null, + usage_normalized: usage, + model_profile: modelProfile, + timing, + correlation: { ...correlation }, + logs: { stdout_truncated: stdoutLog.truncated, stderr_truncated: stderrLog.truncated } + }); + const result = buildTaskResult({ task, - workspace: workspaceRef, attempt, - feedback: continueSpec.feedback, - additionalRequirements: [...continueSpec.additional_requirements ?? []], - previousSessionId: continueSpec.previous_session_id, - previousResult - }) : await adapter.startTask({ task, workspace: workspaceRef, attempt }); - let cancelTimer; - const cancellation = new Promise((_resolve, reject) => { - cancelTimer = setInterval(() => { - try { - const current = store.readStatus(taskId2); - if (current.attempt !== attempt || !current.cancel_requested) return; - clearInterval(cancelTimer); - void adapter.cancelTask(handle).catch(reject); - } catch (error62) { - clearInterval(cancelTimer); - reject(error62); - } - }, 50); + startedAt, + finishedAt, + outcome, + failure: failure2, + cancelled: outcome?.cancelled === true || failure2?.code === "cancelled", + sessionId: store.readStatus(taskId2).zcode_session_id ?? continueSpec?.previous_session_id, + usage, + model: modelProfile, + timing }); - try { - outcome = await Promise.race([adapter.getResult(handle), cancellation]); - } finally { - clearInterval(cancelTimer); - } - if (outcome.cleanupError) failure2 = { code: "cleanup_failed", message: `ZCode returned a turn result, but process cleanup could not be verified: ${outcome.cleanupError}` }; - flushModelOutput(); - } catch (error62) { - flushModelOutput(); - failure2 = { - code: error62 instanceof BridgeError ? error62.code : "worker_error", - message: error62 instanceof Error ? error62.message : String(error62) - }; + store.appendEvent(taskId2, "task_finished", `Task reached terminal status: ${result.status}`, { + status: result.status, + needs_master_decision: result.needs_master_decision + }, finishedAt); + store.commitWorkerResult(taskId2, attempt, result, { + status: result.status, + finished_at: finishedAt, + exit_code: result.exit_code, + zcode_session_id: result.session_id, + error_code: result.error_code ?? null, + error: result.status === "failed" ? result.summary : null, + worker_pid: null, + cleanup_unverified: failure2?.code === "cleanup_failed", + zcode_pid: failure2?.code === "cleanup_failed" ? store.readStatus(taskId2).zcode_pid : null + }); + return { status: result.status, result }; + } finally { + if (heartbeatTimer) clearInterval(heartbeatTimer); + if (persistCorrelationTimer) clearInterval(persistCorrelationTimer); + await reapIdentityWrites(); } - clearInterval(heartbeatTimer); - clearInterval(persistCorrelationTimer); - activityPhase = "finalizing"; - writePhase(); - const finishedAt = now().toISOString(); - store.assertWorkerAttempt(taskId2, attempt); - const stdoutLog = store.appendLog(taskId2, "stdout", outcome?.stdout ? `${outcome.stdout} -` : ""); - const stderrLog = store.appendLog( - taskId2, - "stderr", - outcome?.stderr ? `${outcome.stderr} -` : failure2 ? `${failure2.code}: ${failure2.message} -` : "" - ); - if (modelProfile.model_id === null && outcome?.modelProfile?.model_id) { - Object.assign(modelProfile, outcome.modelProfile); - } - correlation.rpc_accepted_at = correlation.rpc_accepted_at ?? outcome?.phaseTimestamps?.rpc_accepted_at ?? null; - correlation.turn_started_at = correlation.turn_started_at ?? outcome?.phaseTimestamps?.turn_started_at ?? null; - correlation.turn_completed_at = correlation.turn_completed_at ?? outcome?.phaseTimestamps?.turn_completed_at ?? null; - if (modelProfile.turn_id === null && correlation.turn_id) modelProfile.turn_id = correlation.turn_id; - const timingNotes = []; - const timing = { - queued_ms: phaseDuration(initialStatus.created_at, startedAt, timingNotes), - execution_ms: phaseDuration(startedAt, finishedAt, timingNotes), - turn_ms: phaseDuration(correlation.turn_started_at, correlation.turn_completed_at, timingNotes), - finalize_ms: phaseDuration(correlation.turn_completed_at, finishedAt, timingNotes), - wall_ms: phaseDuration(startedAt, finishedAt, timingNotes), - derived: false, - notes: timingNotes - }; - const usage = normalizeUsage(outcome?.usage ?? null, { - source: "zcode_runtime_turn", - scope: outcome?.sessionId ? `session:${outcome.sessionId}` : null, - observedAt: finishedAt - }); - persistCorrelation(); - store.writeAttemptMeta(taskId2, attempt, "outcome.json", { - started_at: startedAt, - finished_at: finishedAt, - failure: failure2, - outcome: outcome ? { - exitCode: outcome.exitCode, - signal: outcome.signal, - sessionId: outcome.sessionId, - response: outcome.response, - usage: outcome.usage, - timedOut: outcome.timedOut, - cancelled: outcome.cancelled, - attempts: outcome.attempts, - errorCode: outcome.errorCode, - reportError: outcome.reportError, - stdoutTruncated: outcome.stdoutTruncated, - stderrTruncated: outcome.stderrTruncated, - agentReport: outcome.agentReport, - reportCandidate: outcome.reportCandidate, - cleanupError: outcome.cleanupError ?? null, - cleanupVerified: outcome.cleanupVerified === true - } : null, - usage_normalized: usage, - model_profile: modelProfile, - timing, - correlation: { ...correlation }, - logs: { stdout_truncated: stdoutLog.truncated, stderr_truncated: stderrLog.truncated } - }); - const result = buildTaskResult({ - task, - attempt, - startedAt, - finishedAt, - outcome, - failure: failure2, - cancelled: outcome?.cancelled === true || failure2?.code === "cancelled", - sessionId: store.readStatus(taskId2).zcode_session_id ?? continueSpec?.previous_session_id, - usage, - model: modelProfile, - timing - }); - store.appendEvent(taskId2, "task_finished", `Task reached terminal status: ${result.status}`, { - status: result.status, - needs_master_decision: result.needs_master_decision - }, finishedAt); - store.commitWorkerResult(taskId2, attempt, result, { - status: result.status, - finished_at: finishedAt, - exit_code: result.exit_code, - zcode_session_id: result.session_id, - error_code: result.error_code ?? null, - error: result.status === "failed" ? result.summary : null, - worker_pid: null, - cleanup_unverified: failure2?.code === "cleanup_failed", - zcode_pid: failure2?.code === "cleanup_failed" ? store.readStatus(taskId2).zcode_pid : null - }); - return { status: result.status, result }; } function interactionSummary(request) { const params = request.params; diff --git a/scripts/lib/repo-root-pattern.mjs b/scripts/lib/repo-root-pattern.mjs index b4b35ee..675c151 100644 --- a/scripts/lib/repo-root-pattern.mjs +++ b/scripts/lib/repo-root-pattern.mjs @@ -30,9 +30,74 @@ // path. The delimiters still apply: "file:///app%20Backup/src" extends the // segment and must not match, while "file:///app/src" must. Scheme-less // percent-encoded spellings are not covered by this check. +// +// Filesystem roots are rejected up front (2026-10-17): a POSIX root would +// match every "//" line comment and every file URI in the bundle, a drive or +// share root every same-volume absolute path — the check could only produce +// guaranteed false positives, never a meaningful verdict. Rejecting loudly +// keeps validate-plugin from silently degrading into a no-op. +import path from "node:path"; import { pathToFileURL } from "node:url"; +const POSIX_ROOT = /^\/+$/u; +const DRIVE_ROOT = /^[a-zA-Z]:[\\/]?$/u; +const EXTENDED_DRIVE_ROOT = /^\\\\[.?]\\[a-zA-Z]:[\\/]?$/u; +const UNC_SHARE_ROOT = /^\\\\(?:\?\\UNC\\)?[^\\/]+[\\/][^\\/]+[\\/]?$/u; + +/** Root predicates are separator-agnostic: a root consists only of the + * volume designator plus separators, so testing the backslash-mapped spelling + * with the slash-based patterns covers both spellings at once. */ +const asRootLike = (value) => value.replace(/\\/gu, "/"); + +const isPosixRoot = (value) => POSIX_ROOT.test(asRootLike(value)); +const isDriveRoot = (value) => DRIVE_ROOT.test(asRootLike(value)); +const isExtendedDriveRoot = (value) => EXTENDED_DRIVE_ROOT.test(value); +const isUncShareRoot = (value) => UNC_SHARE_ROOT.test(value); +const isFilesystemRootSpelling = (value) => + isPosixRoot(value) || isDriveRoot(value) || isExtendedDriveRoot(value) || isUncShareRoot(value); + +/** Throws an explanatory error when `root` names an entire filesystem, + * volume, or share instead of a repository checkout. Public so callers and + * tests can share the exact rejection semantics. + * + * Lexically equivalent spellings count (2026-10-17 review): "/./", "/alpha/.." + * and "C:/alpha/.." normalize to roots, and a pattern built from them would + * degenerate exactly like the bare root (the file-URI alternative especially). + * The guard therefore runs BOTH platform normalizers on every input and tests + * every candidate with separator-agnostic root predicates; the ORIGINAL + * string is still what gets embedded in the message. Consequences verified + * by tests: a named extended-length checkout ("\\?\C:\repo") survives because + * its normalized candidates are not roots, while win32.normalize DOES + * collapse ".." after an extended prefix ("\\?\C:\repo\.." becomes "\\?\C:", + * a drive root) — such a spelling is rejected as the root it collapses to. */ +export function assertNotFilesystemRoot(root) { + if (typeof root !== "string" || root.trim() === "") { + throw new Error( + "buildRepoRootLeakPattern requires a non-empty repository root string; refusing to build a meaningless leak pattern", + ); + } + const normalizedCandidates = new Set([root]); + for (const normalize of [path.win32.normalize, path.posix.normalize]) { + try { normalizedCandidates.add(normalize(root)); } catch { /* platform-specific parse failures keep the original */ } + } + const rootCandidate = [...normalizedCandidates].find(isFilesystemRootSpelling); + if (!rootCandidate) return; + const extendedDrive = /^\\\\[.?]\\([a-zA-Z]):/u.exec(rootCandidate); + const kind = isPosixRoot(rootCandidate) ? "a POSIX filesystem root" + : isDriveRoot(rootCandidate) ? `the root of drive ${asRootLike(rootCandidate).slice(0, 2)}` + : extendedDrive ? `the root of drive ${extendedDrive[1]}: (extended-length spelling)` + : "the root of a UNC share"; + const normalizedNote = rootCandidate === root ? "" : ` (lexically equivalent to ${JSON.stringify(rootCandidate)})`; + throw new Error( + `buildRepoRootLeakPattern refuses the filesystem root ${JSON.stringify(root)}${normalizedNote} (${kind}): ` + + "every absolute path under it would match, so the leak check could only report false positives " + + '(a POSIX root even flags ordinary "//" comments), never a usable verdict. ' + + "Run validation from a checkout located under a named directory.", + ); +} + export function buildRepoRootLeakPattern(root) { + assertNotFilesystemRoot(root); const escapedRoot = root .replace(/[.*+?^${}()|[\]\\]/gu, "\\$&") .replace(/\\\\/gu, "[/\\\\]{1,2}"); @@ -49,7 +114,13 @@ export function buildRepoRootLeakPattern(root) { // backtick is expressed as \u0060 so it can live inside these templates. const startContext = String.raw`[^\s"'=(:,;\\/\u0060]`; const terminator = String.raw`[^/\\'"\n\r\t\u0060]`; - const httpUrlToken = String.raw`(? void): void; on(event: "error", listener: (error: Error) => void): void; kill(signal?: NodeJS.Signals | number): boolean; @@ -39,7 +53,8 @@ export type SpawnFunction = ( export const defaultSpawnFunction: SpawnFunction = (file, args, options) => // detached on POSIX so the child leads its own process group and the whole - // tree can be signalled; Windows uses taskkill /T instead. + // tree can be signalled; Windows termination is identity-bound per member + // through held handles (ADR-009) — no taskkill /T tree expansion. spawn(file, args, { ...options, detached: process.platform !== "win32", @@ -81,43 +96,47 @@ export interface TerminationResult { readonly pid: number; readonly signal: "SIGTERM" | "SIGKILL"; readonly verified: true; + /** Windows-only receipt of the identity-bound transaction (ADR-009). */ + readonly windows?: WindowsTerminationReceipt; +} + +export interface TerminateOptions { + graceMs?: number; + killWaitMs?: number; + /** Caller-bound root identity (startup fingerprint). REQUIRED on Windows: + * the terminator never builds trust from a bare current PID. POSIX keeps + * its process-group semantics and does not alter behavior for it. */ + expectedRootFingerprint?: string | null; } export type TerminateProcessTree = ( pid: number, - options?: { graceMs?: number; killWaitMs?: number }, + options?: TerminateOptions, ) => Promise; /** * Terminates a process tree and only resolves after termination is verified. - * Windows: `taskkill /PID /T /F`, followed by startup-identity checks - * of the captured root and descendants, regardless of its exit code. - * POSIX: SIGTERM to the process group, then - * SIGKILL, verified by group/pid liveness polls. + * Windows (ADR-009): one PowerShell transaction helper holds a handle per + * verified member; every signal is issued on a held handle after + * GetProcessTimes on that same handle confirms the recorded startup + * identity; ownership edges are proven from exact kernel creation/exit + * times; anything unprovable throws and keeps occupancy. POSIX: SIGTERM to + * the process group, then SIGKILL, verified by group/pid liveness polls + * (unchanged semantics). */ export const terminateProcessTree: TerminateProcessTree = async (pid, options = {}) => { if (!Number.isInteger(pid) || pid <= 0) { throw new Error(`A positive integer PID is required, got: ${pid}`); } if (process.platform === "win32") { - const probe = createPlatformProbe(); - await terminateWindowsProcessTree(pid, options.killWaitMs ?? 2_000, { - capture: async (root) => { - let snapshot: CommandResult; - try { - snapshot = await runCommand("powershell.exe", ["-NoProfile", "-NonInteractive", "-Command", windowsTreeCaptureScript(root)], { - timeoutMs: 5_000, maxOutputBytes: 128_000, - }); - } catch { throw new Error("process_tree_snapshot_unverified: query failed"); } - if (snapshot.code !== 0 || snapshot.stdoutTruncated || snapshot.stderr.length > 0) { - throw new Error("process_tree_snapshot_unverified: query failed or incomplete"); - } - return parseWindowsTreeCapture(snapshot.stdout, root); - }, - probe: (requests) => probe.probe(requests), - kill: async (root) => (await runCommand("taskkill", ["/PID", String(root), "/T", "/F"], { timeoutMs: 15_000 })).code, + const expected = options.expectedRootFingerprint ?? null; + if (expected === null) { + throw new Error("process_tree_root_identity_required: Windows process-tree termination requires a caller-bound root identity"); + } + const receipt = await terminateWindowsProcessTree(pid, expected, options.killWaitMs ?? 2_000, { + open: openPowerShellTreeTransaction, }); - return { pid, signal: "SIGKILL", verified: true }; + return { pid, signal: "SIGKILL", verified: true, windows: receipt }; } signalProcessTree(pid, "SIGTERM"); if (await waitForProcessTreeExit(pid, options.graceMs ?? 500)) { @@ -130,64 +149,95 @@ export const terminateProcessTree: TerminateProcessTree = async (pid, options = return { pid, signal: "SIGKILL", verified: true }; }; -interface CommandResult { - code: number | null; - signal: string | null; - stdout: string; - stderr: string; - stdoutTruncated: boolean; +/** + * Read-only verification used by the manager after a failed termination: + * reports live processes that currently claim any recorded pid as their + * parent (transitively), plus recorded pids still alive under their recorded + * generation. A live recorded pid whose current generation differs is a + * proven foreign owner and its subtree is excluded. It proves "nothing + * provably-ours-or-unattributable is alive right now"; unobserved + * intermediate ancestors remain outside any proof and callers must log that + * boundary instead of claiming completeness. + */ +export interface TreeClaimantTarget { + readonly pid: number; + readonly fingerprint: string | null; } -/** Minimal shell-free runner (used for taskkill). Not for model invocation. */ -function runCommand( - command: string, - args: string[], - options: { timeoutMs?: number; maxOutputBytes?: number }, -): Promise { - return new Promise((resolve, reject) => { - const child = spawn(command, args, { - shell: false, - windowsHide: true, - stdio: ["ignore", "pipe", "pipe"], - }); - let stdout: BoundedText = { value: "", truncated: false }; - let stderr: BoundedText = { value: "", truncated: false }; - child.stdout?.setEncoding("utf8"); - child.stderr?.setEncoding("utf8"); - child.stdout?.on("data", (chunk: string) => { - stdout = appendBounded(stdout, chunk, options.maxOutputBytes ?? 1_000_000); - }); - child.stderr?.on("data", (chunk: string) => { - stderr = appendBounded(stderr, chunk, options.maxOutputBytes ?? 1_000_000); - }); - let timedOut = false; - const timer = - options.timeoutMs && options.timeoutMs > 0 - ? setTimeout(() => { - timedOut = true; - try { - child.kill("SIGKILL"); - } catch { - // process already gone - } - }, options.timeoutMs) - : null; - timer?.unref(); - child.on("error", (error) => { - if (timer) clearTimeout(timer); - reject(error); - }); - child.on("close", (code, signal) => { - if (timer) clearTimeout(timer); - resolve({ - code: timedOut ? null : code, - signal: timedOut ? "SIGKILL" : signal, - stdout: stdout.value, - stderr: stderr.value, - stdoutTruncated: stdout.truncated, - }); - }); - }); +export async function verifyProcessTreeClaimantsClean(targets: readonly TreeClaimantTarget[]): Promise<{ clean: boolean; livePids: readonly number[] }> { + if (process.platform === "win32") { + return verifyWindowsRecordedTreeClaimants(targets, { open: openPowerShellTreeTransaction }); + } + const live = targets.filter((t) => Number.isInteger(t.pid) && t.pid > 0 && isProcessTreeRunning(t.pid)).map((t) => t.pid); + return { clean: live.length === 0, livePids: live }; +} + +// -- Spawn-generation continuity (ADR-009) ---------------------------------- +// +// libuv closes the child's native process handle immediately before +// delivering Node's 'exit' event, and Node sets exitCode/signalCode inside +// that same exit callback — before any other JS (including promise +// continuations) can run. Therefore, while exitCode and signalCode are both +// null, either the process is still alive or libuv still holds its handle; +// in both states the PID cannot have been recycled and the child is provably +// still on its original spawn generation. A close/stdio flag alone is NOT +// sufficient: Node delivers 'exit' before stdio 'close', and between the two +// the PID may already be recycled. + +export interface SpawnGenerationChildState { + readonly pid?: number; + /** Exit state as exposed by a real ChildProcess from Node's exit callback. + * Missing fields (legacy fakes) prove nothing and must fail closed. */ + readonly exitCode?: number | null; + readonly signalCode?: string | null; + /** Optional exit-event flag from the owner; extra defense, not required. */ + readonly exitSeen?: boolean; +} + +/** True while `child` is provably still on its original spawn generation. + * Missing exit state (undefined) is NOT treated as "not exited": unknown + * fields never become trusted evidence of continuity. */ +export function childStillOnSpawnGeneration(child: SpawnGenerationChildState): boolean { + if (typeof child.pid !== "number" || !Number.isInteger(child.pid) || child.pid <= 0) return false; + if (child.exitSeen === true) return false; + return child.exitCode === null && child.signalCode === null; +} + +/** + * Captures the child's startup identity for as long as it stays on its + * original spawn generation. The generation is re-verified AFTER every + * asynchronous probe await and the child's pid must still match at return; a + * fingerprint read after the child exited — possibly describing a recycled + * PID generation — is discarded, never stored. Only exact-precision + * fingerprints qualify (coarse ones can collide across generations). + * Resolves null when the identity cannot be proven; Windows termination + * must then refuse to signal. + */ +export async function captureSpawnGenerationIdentity( + child: SpawnGenerationChildState, + probe: { identityOf(pid: number): Promise<{ readonly fingerprint: string | null; readonly fingerprint_precision?: string | null }> }, + options: { retryDelayMs?: number } = {}, +): Promise<{ pid: number; fingerprint: string } | null> { + const pid = child.pid; + if (typeof pid !== "number" || !Number.isInteger(pid) || pid <= 0) return null; + const read = async (): Promise<{ pid: number; fingerprint: string } | null> => { + let identity: { readonly fingerprint: string | null; readonly fingerprint_precision?: string | null }; + try { + identity = await probe.identityOf(pid); + } catch { + return null; + } + if (!childStillOnSpawnGeneration(child)) return null; + if (typeof identity.fingerprint !== "string" || identity.fingerprint.length === 0) return null; + if (identity.fingerprint_precision !== "exact") return null; + if (child.pid !== pid) return null; + return { pid, fingerprint: identity.fingerprint }; + }; + const first = await read(); + if (first !== null) return first; + await new Promise((resolve) => setTimeout(resolve, options.retryDelayMs ?? 250)); + if (!childStillOnSpawnGeneration(child)) return null; + return read(); } function signalProcessTree(pid: number, signal: NodeJS.Signals): void { diff --git a/src/adapters/windows-process-cleanup.ts b/src/adapters/windows-process-cleanup.ts index 816378e..1ae2bd3 100644 --- a/src/adapters/windows-process-cleanup.ts +++ b/src/adapters/windows-process-cleanup.ts @@ -1,80 +1,1143 @@ -import { performance } from "node:perf_hooks"; -import type { ProbeRequest, ProbeVerdict } from "../runtime/process-probe.js"; +// Windows process-tree ownership and termination (ADR-009). +// +// Strategy: one hidden PowerShell helper process runs the WHOLE termination +// transaction. It Add-Type's a Win32 helper that opens one process handle +// per verified member and keeps every handle open from identity verification +// until the final scan. Per the documented Windows contract (Process Handles +// and Identifiers / PROCESS_INFORMATION), a PID cannot be reused while any +// handle to the process object remains open, so held PIDs are +// generation-stable for the transaction's lifetime. Signals are issued only +// through those held handles, after GetProcessTimes on the SAME handle +// confirms the recorded startup identity; TerminateProcess therefore can +// never reach a recycled generation. Tree ownership (parent/child edges) is +// decided from exact kernel creation/exit times read from held handles; CIM +// CreationDate (a microsecond representation) is used only to exclude +// clearly-stale rows without touching them and to check that a handle and +// its enumeration row describe the same generation. Anything that cannot be +// proven — indistinguishable same-tick ordering, naturally-exited parents, +// unknown identities, protocol failures — fails conservatively and keeps +// occupancy; it is never reported as verified cleanup. +import { spawn } from "node:child_process"; + +/** CIM Win32_Process.CreationDate has microsecond representation: the true + * creation time lies in the closed interval [cim, cim + 1us] (observed + * truncation, never rounding up; see ADR-009 verification notes). */ +export const CIM_REPRESENTATION_INTERVAL_TICKS = 10n; + +export interface WindowsProcessRow { + readonly pid: number; + readonly parentPid: number; + /** CIM CreationDate.ToFileTimeUtc(), decimal string (100ns ticks since + * 1601 UTC); null when the CIM value was unreadable (protected system + * processes expose a null CreationDate). */ + readonly creationFileTime: string | null; + /** Get-Process StartTime.Ticks, decimal string; null when unreadable. */ + readonly startTicks: string | null; +} + +export interface HeldProcessInfo { + readonly pid: number; + /** Exact creation FILETIME read from the held handle. */ + readonly creationFileTime: string; + /** The same creation time in DateTime .Ticks frame (matches StartTime.Ticks). */ + readonly creationTicks: string; +} + +export interface KilledProcessRecord { + readonly pid: number; + readonly creationFileTime: string; + /** Kernel exit FILETIME captured on the held handle; null when the exit + * wait timed out (unconfirmed exit). */ + readonly exitFileTime: string | null; +} + +export interface MemberResolution { + readonly pid: number; + readonly parentPid: number; + /** CIM creation of the row (representation-interval anchor). */ + readonly creationFileTime: string; + readonly startTicks: string; +} + +export type MembershipResolution = + | { readonly kind: "success" } + | { + readonly kind: "members"; + readonly members: readonly MemberResolution[]; + readonly holdRequired: readonly number[]; + } + | { readonly kind: "refuse"; readonly code: string; readonly pid?: number }; + +export interface MembershipInput { + readonly rows: readonly WindowsProcessRow[]; + readonly rootPid: number; + /** Caller-bound root identity (persisted/spawn-captured StartTime.Ticks). */ + readonly expectedRootTicks: string | null; + readonly held: ReadonlyMap; + readonly killed: ReadonlyMap; + /** Pids of earlier-round members that vanished without our signal. */ + readonly naturalGonePids: ReadonlySet; + /** Last observed CIM creation for naturally-gone pids (stale-row exclusion). */ + readonly naturalGoneCreation: ReadonlyMap; +} + +function withinCimInterval(exactFileTime: string, cimFileTime: string): boolean { + const exact = BigInt(exactFileTime); + const cim = BigInt(cimFileTime); + return exact >= cim && exact <= cim + CIM_REPRESENTATION_INTERVAL_TICKS; +} + +function cimIntervalWithin(cimFileTime: string, lowExact: string, highExact: string): boolean { + const cim = BigInt(cimFileTime); + return cim >= BigInt(lowExact) && cim + CIM_REPRESENTATION_INTERVAL_TICKS <= BigInt(highExact); +} + +function cimClearlyBefore(cimFileTime: string, exactFileTime: string): boolean { + return BigInt(cimFileTime) + CIM_REPRESENTATION_INTERVAL_TICKS < BigInt(exactFileTime); +} + +function cimClearlyAfter(cimFileTime: string, exactFileTime: string): boolean { + return BigInt(cimFileTime) > BigInt(exactFileTime); +} + +function acceptMember( + row: WindowsProcessRow, + input: MembershipInput, + held: ReadonlyMap, + members: MemberResolution[], + memberPids: Set, + decidedIrrelevant: Set, + holdRequired: Set, + refuse: (code: string, pid?: number) => void, +): void { + if (row.startTicks === null || row.creationFileTime === null) { + refuse("process_tree_member_identity_unknown", row.pid); + return; + } + const child = held.get(row.pid); + if (!child) { + holdRequired.add(row.pid); // the driver holds and re-resolves + return; + } + if (!withinCimInterval(child.creationFileTime, row.creationFileTime)) { + refuse("process_tree_generation_mismatch", row.pid); + return; + } + if (child.creationTicks !== row.startTicks) { + refuse("process_tree_member_generation_mismatch", row.pid); + return; + } + memberPids.add(row.pid); + members.push({ pid: row.pid, parentPid: row.parentPid, creationFileTime: row.creationFileTime, startTicks: row.startTicks }); +} + +/** + * Pure, deterministic membership resolution over one enumeration round. + * Seeds: the root row (identity-checked against the caller's expected + * fingerprint), or — once the transaction itself killed the root — candidate + * rows attributed through killed members' exact lifetime windows. Edges are + * accepted only with proof: + * - live parent: exact handle creation times of both sides, child strictly + * later than parent; equal ticks are indistinguishable and refused; + * - killed parent (exact creation and exit times held from our signal): + * child creation within the closed [creation, exit] window proves + * membership (the PID belonged exclusively to that process during its + * lifetime); creation clearly before the parent (stale record) or after + * its confirmed exit proves the row is not that parent's child; + * - naturally-exited parent: no proof is possible; a live candidate under + * it that does not predate the parent's first observation refuses the + * round (conservative failure). + * Stale rows are excluded by CIM-vs-exact time comparison BEFORE any handle + * is opened on them. + */ +export function resolveWindowsTreeMembership(input: MembershipInput): MembershipResolution { + const rowsByPid = new Map(); + for (const row of input.rows) { + if (rowsByPid.has(row.pid)) return { kind: "refuse", code: "process_tree_snapshot_invalid", pid: row.pid }; + rowsByPid.set(row.pid, row); + } + const members: MemberResolution[] = []; + const memberPids = new Set(); + const decidedIrrelevant = new Set(); + const holdRequired = new Set(); + const state: { refusal: { code: string; pid?: number } | null } = { refusal: null }; + const refuse = (code: string, pid?: number): void => { + if (!state.refusal) state.refusal = { code, pid }; + }; + + const rootRow = rowsByPid.get(input.rootPid); + if (rootRow) { + if (input.expectedRootTicks === null) { + return { kind: "refuse", code: "process_tree_root_identity_required", pid: input.rootPid }; + } + if (rootRow.startTicks === null || rootRow.creationFileTime === null) { + return { kind: "refuse", code: "process_tree_root_identity_unknown", pid: input.rootPid }; + } + if (rootRow.startTicks !== input.expectedRootTicks) { + return { kind: "refuse", code: "process_tree_root_identity_changed", pid: input.rootPid }; + } + const heldRoot = input.held.get(input.rootPid); + if (!heldRoot) { + holdRequired.add(input.rootPid); + } else if (heldRoot.creationTicks !== rootRow.startTicks || !withinCimInterval(heldRoot.creationFileTime, rootRow.creationFileTime)) { + return { kind: "refuse", code: "process_tree_root_generation_mismatch", pid: input.rootPid }; + } else { + memberPids.add(rootRow.pid); + members.push({ pid: rootRow.pid, parentPid: rootRow.parentPid, creationFileTime: rootRow.creationFileTime, startTicks: rootRow.startTicks }); + } + } else if (!input.killed.has(input.rootPid)) { + if (input.expectedRootTicks === null) { + return { kind: "refuse", code: "process_tree_root_identity_required", pid: input.rootPid }; + } + // Fresh transaction and the root is already gone: no provable complete + // tree state exists (children of the dead root cannot be attributed). + return { kind: "refuse", code: "process_tree_root_exited", pid: input.rootPid }; + } + + // Fixpoint: admitting a member can make its own children attributable. + for (let changed = true; changed && !state.refusal;) { + changed = false; + for (const row of input.rows) { + if (state.refusal) break; + if (memberPids.has(row.pid) || decidedIrrelevant.has(row.pid)) continue; + const parentPid = row.parentPid; + if (parentPid === row.pid) { + decidedIrrelevant.add(row.pid); + changed = true; + continue; + } + const parentRow = memberPids.has(parentPid) ? rowsByPid.get(parentPid) : undefined; + if (parentRow) { + if (row.creationFileTime === null) { + refuse("process_tree_row_creation_unknown", row.pid); + continue; + } + if (parentRow.creationFileTime === null) { + refuse("process_tree_row_creation_unknown", parentPid); + continue; + } + const parent = input.held.get(parentPid); + if (!parent) { + holdRequired.add(parentPid); + continue; + } + if (cimClearlyBefore(row.creationFileTime, parent.creationFileTime)) { + decidedIrrelevant.add(row.pid); // predates the parent: stale row + changed = true; + continue; + } + const child = input.held.get(row.pid); + if (!child) { + holdRequired.add(row.pid); + continue; + } + if (!withinCimInterval(child.creationFileTime, row.creationFileTime) || !withinCimInterval(parent.creationFileTime, parentRow.creationFileTime)) { + refuse("process_tree_generation_mismatch", row.pid); + continue; + } + const childCreation = BigInt(child.creationFileTime); + const parentCreation = BigInt(parent.creationFileTime); + if (childCreation === parentCreation) { + refuse("process_tree_edge_indistinguishable", row.pid); + continue; + } + if (childCreation < parentCreation) { + decidedIrrelevant.add(row.pid); // predates the parent + changed = true; + continue; + } + acceptMember(row, input, input.held, members, memberPids, decidedIrrelevant, holdRequired, refuse); + if (memberPids.has(row.pid)) changed = true; + continue; + } + const killedParent = input.killed.get(parentPid); + if (killedParent) { + if (row.creationFileTime === null) { + refuse("process_tree_row_creation_unknown", row.pid); + continue; + } + if (cimClearlyBefore(row.creationFileTime, killedParent.creationFileTime)) { + decidedIrrelevant.add(row.pid); // stale record predating the parent + changed = true; + continue; + } + if (killedParent.exitFileTime === null) { + refuse("process_tree_parent_exit_unconfirmed", parentPid); + continue; + } + if (cimClearlyAfter(row.creationFileTime, killedParent.exitFileTime)) { + decidedIrrelevant.add(row.pid); // created after the confirmed exit + changed = true; + continue; + } + if (cimIntervalWithin(row.creationFileTime, killedParent.creationFileTime, killedParent.exitFileTime)) { + acceptMember(row, input, input.held, members, memberPids, decidedIrrelevant, holdRequired, refuse); + if (memberPids.has(row.pid)) changed = true; + continue; + } + // Interval straddles a window boundary: the exact time decides. + const child = input.held.get(row.pid); + if (!child) { + holdRequired.add(row.pid); + continue; + } + if (!withinCimInterval(child.creationFileTime, row.creationFileTime)) { + refuse("process_tree_generation_mismatch", row.pid); + continue; + } + const creation = BigInt(child.creationFileTime); + if (creation >= BigInt(killedParent.creationFileTime) && creation <= BigInt(killedParent.exitFileTime)) { + acceptMember(row, input, input.held, members, memberPids, decidedIrrelevant, holdRequired, refuse); + if (memberPids.has(row.pid)) changed = true; + } else { + decidedIrrelevant.add(row.pid); // outside the parent's lifetime + changed = true; + } + continue; + } + if (input.naturalGonePids.has(parentPid)) { + if (row.creationFileTime === null) { + refuse("process_tree_row_creation_unknown", row.pid); + continue; + } + const lastCim = input.naturalGoneCreation.get(parentPid) ?? null; + if (lastCim !== null && cimClearlyBefore(row.creationFileTime, lastCim)) { + decidedIrrelevant.add(row.pid); // predates the parent's first observation + changed = true; + continue; + } + refuse("process_tree_parent_unprovable", parentPid); + continue; + } + // Review-round-1 addendum (test-quality): a row whose parent is a + // still-UNDECIDED live row must stay pending for the next fixpoint + // sweep — marking it irrelevant now would drop enumerated descendants + // whenever rows arrive before their parent is admitted (reverse or + // shuffled enumeration order). Only a parent that can provably never + // join this tree (no row, not killed, not naturally gone, not the + // root, or already decided irrelevant) makes the row irrelevant. + const parentUndecidedLiveRow = rowsByPid.has(parentPid) && !memberPids.has(parentPid) && !decidedIrrelevant.has(parentPid); + if (parentUndecidedLiveRow || parentPid === input.rootPid) continue; // pending: re-evaluate after the parent is decided + decidedIrrelevant.add(row.pid); // parent outside this tree + changed = true; + } + } + if (state.refusal) return { kind: "refuse", code: state.refusal.code, pid: state.refusal.pid }; + if (holdRequired.size > 0) return { kind: "members", members, holdRequired: [...holdRequired] }; + if (members.length === 0) return { kind: "success" }; + return { kind: "members", members, holdRequired: [] }; +} + +// -- Transaction transport ------------------------------------------------ + +export interface HoldOutcome { + readonly pid: number; + readonly status: "ok" | "absent" | "denied" | "error"; + readonly creationFileTime?: string; + readonly creationTicks?: string; + readonly error?: number; + readonly detail?: string; +} + +export interface KillOutcome { + readonly status: "signaled" | "identity_changed" | "not_held" | "wait_timeout" | "already_exited" | "error"; + readonly exitFileTime: string | null; + readonly detail?: string; +} + +export interface WindowsTreeTransactionLike { + enumerate(): Promise; + hold(pids: readonly number[]): Promise; + kill(pid: number, expectedTicks: string, waitMs: number): Promise; + close(): Promise; +} export interface WindowsCleanupDependencies { - capture: (pid: number) => Promise; - probe: (requests: readonly ProbeRequest[]) => Promise; - kill: (pid: number) => Promise; + open: () => Promise; + now?: () => number; + sleep?: (ms: number) => Promise; +} + +export class WindowsTreeCleanupError extends Error { + readonly code: string; + readonly memberPids: readonly number[]; + constructor(code: string, message: string, memberPids: readonly number[] = []) { + super(message); + this.name = "WindowsTreeCleanupError"; + this.code = code; + this.memberPids = memberPids; + } +} + +export interface WindowsTerminationReceipt { + readonly rootPid: number; + readonly rounds: number; + readonly members: readonly { pid: number; fingerprint: string; creationFileTime: string }[]; + readonly kills: readonly { pid: number; status: string; exitFileTime: string | null }[]; + readonly signaled: readonly number[]; +} + +interface MemberRecord { + readonly pid: number; + readonly parentPid: number; + readonly startTicks: string; + readonly creationFileTime: string; } -/** Verify recorded executors, not taskkill's localized output or the root alone. */ +/** + * Terminates the process tree rooted at `pid` using ONLY identity-bound + * signals through held handles. `expectedRootTicks` is the caller-bound root + * identity (persisted executor fingerprint or spawn-time capture); the + * Windows path never builds trust from a bare current PID. Resolves with a + * receipt when the closure provably contains no live members; throws + * WindowsTreeCleanupError on any unprovable or failed state (occupancy must + * be retained). EVERY exception exit — including transport/query failures, + * unresolved holds, and helper-close failures — carries the transaction's + * known member pids plus any observed-but-unverified candidate pids, so a + * caller can never lose the set it must keep occupied (review-round-1 R2/R3). + */ export async function terminateWindowsProcessTree( pid: number, + expectedRootTicks: string | null, killWaitMs: number, dependencies: WindowsCleanupDependencies, -): Promise { - const requests = await dependencies.capture(pid); - if (!requests.some((request) => request.pid === pid)) throw new Error("process_tree_snapshot_invalid: root missing"); - const inspect = async (): Promise => { - const verdicts = await dependencies.probe(requests); - if (verdicts.length !== requests.length || verdicts.some((verdict) => verdict.state === "unknown")) { - throw new Error("process_tree_probe_unverified: recorded executor exit is unknown"); - } - return verdicts; - }; - const before = await inspect(); - if (before.every((verdict) => verdict.state === "exited")) return; - const rootIndex = requests.findIndex((request) => request.pid === pid); - if (before[rootIndex]?.state !== "alive") { - // Never signal a recycled root PID. Its remaining children need review. - throw new Error("process_tree_descendants_alive: root exited before termination"); - } - let exitCode: number | null = null; - let commandFailed = false; - try { exitCode = await dependencies.kill(pid); } - catch { commandFailed = true; } - const deadline = performance.now() + Math.max(0, killWaitMs); - for (;;) { - const after = await inspect(); - if (after.every((verdict) => verdict.state === "exited")) return; - if (performance.now() >= deadline) { - const reason = commandFailed ? "command_error" : exitCode === null ? "timeout" : `exit_${exitCode}`; - throw new Error(`process_tree_cleanup_unverified: taskkill ${reason}; recorded executors remain alive`); - } - await new Promise((resolve) => setTimeout(resolve, 20)); - } -} - -/** ASCII-only protocol; locale-dependent stderr never becomes a public error. */ -export function windowsTreeCaptureScript(pid: number): string { +): Promise { + if (!Number.isInteger(pid) || pid <= 0) { + throw new WindowsTreeCleanupError("process_tree_invalid_pid", `a positive integer PID is required, got: ${String(pid)}`); + } + const now = dependencies.now ?? Date.now; + const sleep = dependencies.sleep ?? ((ms: number) => new Promise((resolve) => setTimeout(resolve, ms))); + const transaction = await dependencies.open(); + const killed = new Map(); + const held = new Map(); + const naturalGone = new Set(); + const naturalGoneCreation = new Map(); + const memberReceipts = new Map(); + const killReceipts: { pid: number; status: string; exitFileTime: string | null }[] = []; + /** Candidates observed but never proven (unresolved holds): they stay in + * every failure's pid set until the transaction fails or proves them. */ + const unverifiedCandidates = new Set(); + let rounds = 0; + let bodyError: unknown = null; + let succeeded = false; + const knownPids = (): number[] => [...new Set([...memberReceipts.keys(), ...unverifiedCandidates])]; + try { + const deadline = now() + Math.max(0, killWaitMs); + for (;;) { + rounds += 1; + const rows = await transaction.enumerate(); + const resolution = await resolveRound(rows); + if (resolution.kind === "refuse") { + throw new WindowsTreeCleanupError(resolution.code, `${resolution.code}: process tree cleanup refused`, knownPids()); + } + if (resolution.kind === "success") break; + const members: MemberRecord[] = resolution.members.map((m) => ({ pid: m.pid, parentPid: m.parentPid, startTicks: m.startTicks, creationFileTime: m.creationFileTime })); + const livePids = new Set(resolution.members.map((m) => m.pid)); + for (const previous of memberReceipts.keys()) { + if (!livePids.has(previous) && !killed.has(previous) && !naturalGone.has(previous)) { + naturalGone.add(previous); + naturalGoneCreation.set(previous, memberReceipts.get(previous)!.creationFileTime); + } + } + for (const member of members) { + unverifiedCandidates.delete(member.pid); // proven member this round + if (!memberReceipts.has(member.pid)) { + memberReceipts.set(member.pid, { pid: member.pid, fingerprint: member.startTicks, creationFileTime: member.creationFileTime }); + } + } + const ordered = orderChildrenFirst(members); + const remaining = Math.max(120, deadline - now()); + const perMemberWait = Math.min(2_000, Math.max(120, Math.floor(remaining / Math.max(1, ordered.length)))); + for (const member of ordered) { + const heldInfo = held.get(member.pid); + const kill = await transaction.kill(member.pid, member.startTicks, perMemberWait); + killReceipts.push({ pid: member.pid, status: kill.status, exitFileTime: kill.exitFileTime }); + if (kill.status === "signaled" || kill.status === "already_exited") { + killed.set(member.pid, { + pid: member.pid, + creationFileTime: heldInfo?.creationFileTime ?? member.creationFileTime, + exitFileTime: kill.exitFileTime, + }); + } else if (kill.status === "identity_changed" || kill.status === "not_held") { + // The recorded executor no longer owns the held handle/PID: it + // changed hands outside our signal; further attribution for its + // children is unprovable. + naturalGone.add(member.pid); + naturalGoneCreation.set(member.pid, member.creationFileTime); + } + // wait_timeout / error: the member stays unresolved; the next round + // retries until the deadline refuses. + } + if (now() >= deadline) { + throw new WindowsTreeCleanupError( + "process_tree_cleanup_unverified", + `process_tree_cleanup_unverified: ${String(resolution.members.length)} recorded member(s) not provably exited`, + knownPids(), + ); + } + await sleep(20); + } + succeeded = true; + return { + rootPid: pid, + rounds, + members: [...memberReceipts.values()], + kills: killReceipts, + signaled: killReceipts.filter((k) => k.status === "signaled").map((k) => k.pid), + }; + } catch (error) { + bodyError = error; + // Every exit carries the known member + unverified candidate set; a raw + // transport/query error is wrapped so no caller ever sees a memberless + // failure for a transaction that had observed members (R3). + if (error instanceof WindowsTreeCleanupError) { + throw error.memberPids.length > 0 ? error : new WindowsTreeCleanupError(error.code, error.message, knownPids()); + } + const detail = error instanceof Error ? error.message : String(error); + throw new WindowsTreeCleanupError("process_tree_transaction_error", `process_tree_transaction_error: ${detail}`, knownPids()); + } finally { + // R4: a helper-close failure is never swallowed. On success it fails the + // whole transaction; on failure it is chained into the thrown error so + // both remain visible. + let closeError: unknown = null; + try { + await transaction.close(); + } catch (error) { + closeError = error; + } + if (closeError !== null) { + const closeText = closeError instanceof Error ? closeError.message : String(closeError); + if (bodyError !== null) { + const original = bodyError instanceof WindowsTreeCleanupError + ? new WindowsTreeCleanupError(bodyError.code, `${bodyError.message}; process_tree_close_failed: ${closeText}`, bodyError.memberPids.length > 0 ? bodyError.memberPids : knownPids()) + : new WindowsTreeCleanupError("process_tree_transaction_error", `process_tree_transaction_error: ${String(bodyError instanceof Error ? bodyError.message : bodyError)}; process_tree_close_failed: ${closeText}`, knownPids()); + throw original; + } + throw new WindowsTreeCleanupError("process_tree_close_failed", `process_tree_close_failed: ${closeText}`, knownPids()); + } + } + + async function resolveRound(rows: readonly WindowsProcessRow[]): Promise { + let workingRows = [...rows]; + const resolve = (): MembershipResolution => + resolveWindowsTreeMembership({ + rows: workingRows, + rootPid: pid, + expectedRootTicks, + held, + killed, + naturalGonePids: naturalGone, + naturalGoneCreation, + }); + let resolution = resolve(); + for (let attempt = 0; attempt < 6 && resolution.kind === "members" && resolution.holdRequired.length > 0; attempt += 1) { + const requested = [...new Set(resolution.holdRequired)]; + for (const candidate of resolution.holdRequired) unverifiedCandidates.add(candidate); + const outcomes = await transaction.hold(requested); + // R2: missing, duplicate, or unexpected hold responses are explicit + // protocol failures, never silently ignored members. + if (outcomes.length !== requested.length) { + throw new WindowsTreeCleanupError( + "process_tree_hold_invalid", + `process_tree_hold_invalid: ${String(outcomes.length)} responses for ${String(requested.length)} requested pids`, + knownPids(), + ); + } + const seen = new Set(); + for (const outcome of outcomes) { + if (seen.has(outcome.pid)) { + throw new WindowsTreeCleanupError("process_tree_hold_invalid", `process_tree_hold_invalid: duplicate response for pid ${String(outcome.pid)}`, knownPids()); + } + seen.add(outcome.pid); + if (!requested.includes(outcome.pid)) { + throw new WindowsTreeCleanupError("process_tree_hold_invalid", `process_tree_hold_invalid: unexpected pid ${String(outcome.pid)} in hold response`, knownPids()); + } + if (outcome.status === "ok") { + held.set(outcome.pid, { pid: outcome.pid, creationFileTime: outcome.creationFileTime!, creationTicks: outcome.creationTicks! }); + } else if (outcome.status === "absent") { + // Review-round-1 addendum (R2): an absent candidate is NOT + // forgettable. The enumeration saw it live microseconds ago; its + // own descendants may still be alive and can no longer be + // attributed (the parent vanished before we proved its identity). + // The round fails, keeping the candidate and every known member. + throw new WindowsTreeCleanupError( + "process_tree_candidate_absent", + `process_tree_candidate_absent: candidate pid ${String(outcome.pid)} vanished between enumeration and hold; its subtree cannot be attributed`, + [...new Set([...requested, ...knownPids()])], + ); + } + // denied / error stay unresolved and remain in unverifiedCandidates. + } + resolution = resolve(); + } + if (resolution.kind === "members" && resolution.holdRequired.length > 0) { + // R2: the bounded hold budget is exhausted with unprovable candidates. + // The round is NOT an executable set: fail explicitly, keeping every + // observed-but-unverified candidate (and this round's already-proven + // members) in the failure's pid set. + for (const candidate of resolution.holdRequired) unverifiedCandidates.add(candidate); + const roundMemberPids = resolution.members.map((m) => m.pid); + throw new WindowsTreeCleanupError( + "process_tree_hold_unresolved", + `process_tree_hold_unresolved: cannot prove identity/ownership for pids ${resolution.holdRequired.join(",")}`, + [...new Set([...roundMemberPids, ...knownPids()])], + ); + } + return resolution; + } +} + +function orderChildrenFirst(members: readonly MemberRecord[]): MemberRecord[] { + // True postorder (review-round-1 addendum): a member is emitted only after + // every one of its own descendants inside this round's member set. Members + // whose subtree relation is cyclic (corrupt data) fall back to + // creation-descending order instead of blocking the bounded kill pass. + const remaining = [...members]; + const ordered: MemberRecord[] = []; + let progress = true; + while (remaining.length > 0 && progress) { + progress = false; + for (let index = remaining.length - 1; index >= 0; index -= 1) { + const member = remaining[index]!; + const hasPendingChild = remaining.some((other) => other !== member && other.parentPid === member.pid); + if (!hasPendingChild) { + ordered.push(member); + remaining.splice(index, 1); + progress = true; + } + } + } + const leftovers = remaining.sort((a, b) => (BigInt(a.creationFileTime) > BigInt(b.creationFileTime) ? -1 : 1)); + return [...ordered, ...leftovers]; +} + +/** + * Read-only recorded-tree scan: reports live processes that currently claim + * any of the recorded pids as their parent, transitively. A recorded pid that + * is alive with a MATCHING startup fingerprint is itself a claimant; one + * alive with a different fingerprint is a proven foreign generation — its + * subtree is excluded from the claim. A recorded pid without a fingerprint + * that is alive stays a claimant (unprovable, conservative). Children of + * dead recorded pids count as claimants: their creator is unattributable + * after the fact, and releasing while one is alive could orphan our own + * process. This proves "nothing provably-ours-or-unattributable is alive + * right now"; it does NOT prove no process ever descended from the recorded + * pids (unobserved intermediate ancestors remain outside any proof), and + * callers must log that boundary instead of claiming completeness. + */ +export async function verifyWindowsRecordedTreeClaimants( + targets: readonly { pid: number; fingerprint: string | null }[], + dependencies: WindowsCleanupDependencies, +): Promise<{ clean: boolean; livePids: readonly number[] }> { + const transaction = await dependencies.open(); + let bodyError: unknown = null; + const targetPids = targets.map((t) => t.pid).filter((p) => Number.isInteger(p) && p > 0); + try { + const rows = await transaction.enumerate(); + const live = new Set(); + for (const row of rows) live.add(row.pid); + const claimantPids = new Set(); + const foreignPids = new Set(); + const recorded = new Set(); + for (const target of targets) { + if (!Number.isInteger(target.pid) || target.pid <= 0) continue; + recorded.add(target.pid); + if (!live.has(target.pid)) continue; + if (target.fingerprint === null) { + claimantPids.add(target.pid); + continue; + } + const outcome = (await transaction.hold([target.pid]))[0]; + if (outcome === undefined) throw new WindowsTreeCleanupError("process_tree_scan_invalid", "recorded-tree scan received no hold outcome"); + if (outcome.status === "ok") { + if (outcome.creationTicks === target.fingerprint) claimantPids.add(target.pid); + else foreignPids.add(target.pid); + } else if (outcome.status === "absent") { + continue; // exited between enumeration and hold + } else { + // denied/error: identity unreadable; never treat as provably gone. + claimantPids.add(target.pid); + } + } + for (let changed = true; changed;) { + changed = false; + for (const row of rows) { + if (claimantPids.has(row.pid) || foreignPids.has(row.pid)) continue; + const parentClaimed = claimantPids.has(row.parentPid); + const parentRecordedDead = recorded.has(row.parentPid) && !live.has(row.parentPid) && !foreignPids.has(row.parentPid); + if (parentClaimed || parentRecordedDead) { + claimantPids.add(row.pid); + changed = true; + } + } + } + const liveList = [...claimantPids].sort((a, b) => a - b); + return { clean: liveList.length === 0, livePids: liveList }; + } catch (error) { + bodyError = error; + throw error; + } finally { + // R4: the read-only scan must not claim a result through a silently + // failed helper close either; a close failure is chained onto a body + // failure or fails an otherwise-successful scan. + let closeError: unknown = null; + try { + await transaction.close(); + } catch (error) { + closeError = error; + } + if (closeError !== null) { + const closeText = closeError instanceof Error ? closeError.message : String(closeError); + if (bodyError !== null) { + const originalText = bodyError instanceof Error ? bodyError.message : String(bodyError); + throw new WindowsTreeCleanupError("process_tree_scan_failed", `process_tree_scan_failed: ${originalText}; process_tree_close_failed: ${closeText}`, targetPids); + } + throw new WindowsTreeCleanupError("process_tree_close_failed", `process_tree_close_failed: ${closeText}`, targetPids); + } + } +} + +// -- Real PowerShell transport -------------------------------------------- +// +// One persistent hidden powershell.exe process runs the whole transaction so +// opened handles stay open across rounds (blocking PID reuse for held pids). +// Line protocol over stdio; every response ends with an explicit terminator +// line. Any deviation — unknown command, malformed row, early exit, timeout — +// is a hard failure: the transport throws and the driver reports unverified +// cleanup. Protocol text is ASCII only; localized stderr never surfaces. + +export function windowsTreeTransactionScript(): string { return [ "$ErrorActionPreference = 'Stop'", - "$all = @(Get-CimInstance Win32_Process -ErrorAction Stop)", - `$selected = [System.Collections.Generic.HashSet[int]]::new(); [void]$selected.Add(${pid})`, - "do { $changed = $false; foreach ($item in $all) { if ($selected.Contains([int]$item.ParentProcessId)) { if ($selected.Add([int]$item.ProcessId)) { $changed = $true } } } } while ($changed)", - "'bridge_tree_snapshot_v1'", - "foreach ($candidate in $selected) { $item = Get-Process -Id $candidate -ErrorAction SilentlyContinue; if ($null -eq $item) { '{0}|absent' -f $candidate } else { '{0}|{1}' -f $candidate, $item.StartTime.Ticks } }", - ].join("; "); -} - -export function parseWindowsTreeCapture(stdout: string, pid: number): ProbeRequest[] { - const lines = stdout.trim().split(/\r?\n/u); - if (lines.shift() !== "bridge_tree_snapshot_v1") throw new Error("process_tree_snapshot_invalid: protocol missing"); - const capturedAt = new Date().toISOString(); - const requests: ProbeRequest[] = []; - const seen = new Set(); - for (const line of lines) { - const match = /^(\d+)\|(\d+|absent)$/u.exec(line.trim()); - const candidate = Number(match?.[1]); - if (!match || !Number.isSafeInteger(candidate) || candidate <= 0 || seen.has(candidate)) { - throw new Error("process_tree_snapshot_invalid: malformed identity"); - } - seen.add(candidate); - requests.push({ pid: candidate, identity: match[2] === "absent" ? null : { - pid: candidate, fingerprint: match[2]!, fingerprint_precision: "exact", identity_version: 1, - platform: "win32", captured_at: capturedAt, - } }); - } - if (!seen.has(pid)) throw new Error("process_tree_snapshot_invalid: root missing"); - return requests; + "$ProgressPreference = 'SilentlyContinue'", + "Add-Type -TypeDefinition @'", + "using System;", + "using System.Collections.Generic;", + "using System.Runtime.InteropServices;", + "public static class BridgeTreeTx {", + " [DllImport(\"kernel32.dll\", SetLastError = true)] static extern IntPtr OpenProcess(uint access, bool inherit, int pid);", + " [DllImport(\"kernel32.dll\", SetLastError = true)] static extern bool GetProcessTimes(IntPtr h, out long created, out long exited, out long kernel, out long user);", + " [DllImport(\"kernel32.dll\", SetLastError = true)] static extern bool TerminateProcess(IntPtr h, uint exitCode);", + " [DllImport(\"kernel32.dll\", SetLastError = true)] static extern uint WaitForSingleObject(IntPtr h, uint milliseconds);", + " [DllImport(\"kernel32.dll\", SetLastError = true)] static extern bool CloseHandle(IntPtr h);", + " public static readonly Dictionary Held = new Dictionary();", + " public static string Hold(int pid) {", + " IntPtr existing;", + " if (Held.TryGetValue(pid, out existing) && existing != IntPtr.Zero) { return \"already_held|\"; }", + " IntPtr h = OpenProcess(0x00101001u, false, pid);", + " if (h == IntPtr.Zero) { int err = Marshal.GetLastWin32Error(); return (err == 87 ? \"absent|\" : (err == 5 ? \"denied|\" : \"error|\")) + err.ToString(); }", + " long c, e, k, u;", + " if (!GetProcessTimes(h, out c, out e, out k, out u)) { int err = Marshal.GetLastWin32Error(); CloseHandle(h); return \"error|\" + err.ToString(); }", + " Held[pid] = h;", + " return \"ok|\" + c.ToString() + \"|\" + DateTime.FromFileTime(c).Ticks.ToString();", + " }", + " public static string Kill(int pid, string expectTicks, int waitMs) {", + " IntPtr h;", + " if (!Held.TryGetValue(pid, out h) || h == IntPtr.Zero) { return \"not_held|\"; }", + " long c, e, k, u;", + " if (!GetProcessTimes(h, out c, out e, out k, out u)) { return \"error|\" + Marshal.GetLastWin32Error().ToString(); }", + " string current = DateTime.FromFileTime(c).Ticks.ToString();", + " if (current != expectTicks) { return \"identity_changed|\" + current; }", + " if (e != 0) { return \"already_exited|\" + e.ToString(); }", + " if (!TerminateProcess(h, 1)) { return \"error|\" + Marshal.GetLastWin32Error().ToString(); }", + " uint wait = WaitForSingleObject(h, (uint)waitMs);", + " if (!GetProcessTimes(h, out c, out e, out k, out u)) { return \"error|\" + Marshal.GetLastWin32Error().ToString(); }", + " if (wait != 0) { return \"wait_timeout|\" + e.ToString(); }", + " return \"signaled|\" + e.ToString();", + " }", + " public static string CloseAll() {", + " int closed = 0;", + " int failed = 0;", + " foreach (KeyValuePair entry in Held) { if (entry.Value != IntPtr.Zero) { if (CloseHandle(entry.Value)) { closed++; } else { failed++; } } }", + " Held.Clear();", + " return closed.ToString() + \"|\" + failed.ToString();", + " }", + "}", + "'@", + "[Console]::Out.WriteLine('READY'); [Console]::Out.Flush()", + "while ($true) {", + " $line = [Console]::In.ReadLine()", + " if ($null -eq $line) { [void][BridgeTreeTx]::CloseAll(); exit 0 }", + " $parts = $line.Trim().Split(' ')", + " if ($parts[0] -eq 'enum') {", + " $all = @(Get-CimInstance Win32_Process -ErrorAction Stop)", + " [Console]::Out.WriteLine('ENUM_BEGIN')", + " $ids = @(); foreach ($p in $all) { $ids += [int]$p.ProcessId }", + " $byId = @{}", + " if ($ids.Count -gt 0) { foreach ($g in (Get-Process -Id $ids -ErrorAction SilentlyContinue)) { $byId[[int]$g.Id] = $g } }", + " foreach ($p in $all) {", + " $ticks = 'unknown'", + " $cim = 'unknown'", + " if ($null -ne $p.CreationDate) { $cim = [string]$p.CreationDate.ToFileTimeUtc() }", + " $g = $byId[[int]$p.ProcessId]", + " if ($null -ne $g) { try { $st = $g.StartTime; if ($null -ne $st) { $ticks = [string]$st.Ticks } } catch { $ticks = 'unknown' } }", + " if ([string]::IsNullOrEmpty($ticks)) { $ticks = 'unknown' }", + " [Console]::Out.WriteLine(('R {0} {1} {2} {3}' -f [int]$p.ProcessId, [int]$p.ParentProcessId, $cim, $ticks))", + " }", + " [Console]::Out.WriteLine('ENUM_END'); [Console]::Out.Flush()", + " } elseif ($parts[0] -eq 'hold') {", + " for ($i = 1; $i -lt $parts.Count; $i++) {", + " $hp = 0", + " if (-not [int]::TryParse($parts[$i], [ref]$hp)) { [Console]::Out.WriteLine('H 0 error 22'); continue }", + " $r = [BridgeTreeTx]::Hold($hp).Split('|')", + " if ($r[0] -eq 'already_held') { [Console]::Out.WriteLine(('H {0} held' -f $hp)) }", + " elseif ($r[0] -eq 'ok') { [Console]::Out.WriteLine(('H {0} ok {1} {2}' -f $hp, $r[1], $r[2])) }", + " else { [Console]::Out.WriteLine(('H {0} {1} {2}' -f $hp, $r[0], $r[1])) }", + " }", + " [Console]::Out.WriteLine('HOLD_END'); [Console]::Out.Flush()", + " } elseif ($parts[0] -eq 'kill') {", + " $kpid = [int]$parts[1]; $kticks = [string]$parts[2]; $kwait = [int]$parts[3]", + " $r = [BridgeTreeTx]::Kill($kpid, $kticks, $kwait).Split('|')", + " if ($r[0] -eq 'signaled' -or $r[0] -eq 'already_exited') { [Console]::Out.WriteLine(('K {0} {1} {2}' -f $kpid, $r[0], $r[1])) }", + " elseif ($r[0] -eq 'identity_changed') { [Console]::Out.WriteLine(('K {0} identity_changed 0' -f $kpid)) }", + " elseif ($r[0] -eq 'wait_timeout') { [Console]::Out.WriteLine(('K {0} wait_timeout {1}' -f $kpid, $r[1])) }", + " elseif ($r[0] -eq 'not_held') { [Console]::Out.WriteLine(('K {0} not_held 0' -f $kpid)) }", + " else { [Console]::Out.WriteLine(('K {0} error {1}' -f $kpid, $r[1])) }", + " [Console]::Out.WriteLine('KILL_END'); [Console]::Out.Flush()", + " } elseif ($parts[0] -eq 'close') {", + " $r = [BridgeTreeTx]::CloseAll().Split('|')", + " [Console]::Out.WriteLine(('CLOSED {0} {1}' -f $r[0], $r[1])); [Console]::Out.Flush()", + " exit 0", + " } else {", + " [Console]::Out.WriteLine('FATAL unknown_command'); [Console]::Out.Flush(); exit 1", + " }", + "}", + ].join("\n"); +} + +interface LineWaiter { + resolve: (line: string) => void; + reject: (error: Error) => void; + timer: NodeJS.Timeout; +} + +interface SpawnedChild { + readonly pid?: number; + readonly stdout: NodeJS.ReadableStream & { setEncoding(encoding: string): void }; + readonly stderr: NodeJS.ReadableStream & { setEncoding(encoding: string): void }; + readonly stdin: NodeJS.WritableStream; + on(event: "error", listener: (error: Error) => void): void; + on(event: "close", listener: (code: number | null, signal: string | null) => void): void; + kill(signal?: NodeJS.Signals | number): boolean; +} + +/** Transport bounds (review-round-1 addendum R4): a single protocol line is + * tiny, so 1 MiB is already generous; the total buffered stream is capped so + * an endless writer cannot grow memory forever; an enumeration beyond 4096 + * rows is not a plausible process table and fails the snapshot; and the + * whole session has one total wait budget so repeated small chunks can never + * extend a stalled helper indefinitely. */ +export const TRANSPORT_MAX_LINE_BYTES = 1_048_576; +export const TRANSPORT_MAX_BUFFERED_BYTES = 8_388_608; +export const TRANSPORT_MAX_BUFFERED_LINES = 65_536; +export const TRANSPORT_MAX_ENUM_ROWS = 4_096; +export const TRANSPORT_TOTAL_SESSION_MS_DEFAULT = 120_000; + +export interface TransportBounds { + maxLineBytes?: number; + maxBufferedBytes?: number; + maxBufferedLines?: number; + maxEnumRows?: number; + totalSessionMs?: number; +} + +export class PowerShellTreeTransaction implements WindowsTreeTransactionLike { + readonly #child: SpawnedChild; + readonly #bounds: Required; + #lines: string[] = []; + #waiters: LineWaiter[] = []; + #exitWaiters: Array<() => void> = []; + #closed = false; + #bufferedBytes = 0; + #sessionDeadline: number; + #exitCode: number | null = null; + #exitSignal: string | null = null; + /** True once the helper's own exit is expected (receipt processed or + * fallback initiated); its close event is then not a transport failure. */ + #closeHandled = false; + #failure: Error | null = null; + + /** Test seam: attach to an externally spawned/controlled child so the + * bounded-stream behaviours are testable with fake streams (no OS). */ + static attach(child: SpawnedChild, bounds: TransportBounds = {}): PowerShellTreeTransaction { + return new PowerShellTreeTransaction(child, bounds); + } + + private constructor(child: SpawnedChild, bounds: TransportBounds = {}) { + this.#child = child; + this.#bounds = { + maxLineBytes: bounds.maxLineBytes ?? TRANSPORT_MAX_LINE_BYTES, + maxBufferedBytes: bounds.maxBufferedBytes ?? TRANSPORT_MAX_BUFFERED_BYTES, + maxBufferedLines: bounds.maxBufferedLines ?? TRANSPORT_MAX_BUFFERED_LINES, + maxEnumRows: bounds.maxEnumRows ?? TRANSPORT_MAX_ENUM_ROWS, + totalSessionMs: bounds.totalSessionMs ?? TRANSPORT_TOTAL_SESSION_MS_DEFAULT, + }; + this.#sessionDeadline = Date.now() + this.#bounds.totalSessionMs; + child.stdout.setEncoding("utf8"); + let buffer = ""; + child.stdout.on("data", (chunk: string) => { + // After a failure the transport stops receiving: no further buffering, + // no further queueing (addendum 13:13 — overflow must also bound the + // receiver, not just report once). + if (this.#failure) { + buffer = ""; + return; + } + buffer += chunk; + let newline = buffer.indexOf("\n"); + while (newline >= 0) { + const line = buffer.slice(0, newline).replace(/\r$/u, ""); + buffer = buffer.slice(newline + 1); + const lineBytes = Buffer.byteLength(line, "utf8"); + if (lineBytes > this.#bounds.maxLineBytes) { + this.#failAndDrain(); + this.#fail(new Error(`process_tree_transport_overflow: line of ${String(lineBytes)} bytes exceeds ${String(this.#bounds.maxLineBytes)}`)); + return; + } + this.#bufferedBytes += lineBytes; + if (this.#bufferedBytes > this.#bounds.maxBufferedBytes || this.#lines.length >= this.#bounds.maxBufferedLines) { + this.#failAndDrain(); + this.#fail(new Error(`process_tree_transport_overflow: buffered stream exceeded its byte or line budget (${String(this.#bufferedBytes)} bytes, ${String(this.#lines.length)} lines)`)); + return; + } + const waiter = this.#waiters.shift(); + if (waiter) { + clearTimeout(waiter.timer); + waiter.resolve(line); + this.#bufferedBytes -= lineBytes; + } else { + this.#lines.push(line); + } + newline = buffer.indexOf("\n"); + } + if (Buffer.byteLength(buffer, "utf8") > this.#bounds.maxLineBytes) { + this.#failAndDrain(); + this.#fail(new Error(`process_tree_transport_overflow: unterminated line exceeds ${String(this.#bounds.maxLineBytes)} bytes`)); + } + }); + child.stderr.setEncoding("utf8"); + child.stderr.on("data", (chunk: string) => { + // The helper sets $ProgressPreference='SilentlyContinue', so the + // progress/CLIXML noise PowerShell otherwise redirects here is + // suppressed at the source. Anything that still arrives on stderr is + // treated as a genuine error record (R4): never surfaced verbatim — + // localized text stays private — but never silently ignored either. + void chunk; + this.#fail(new Error("process_tree_transport_stderr")); + }); + child.on("error", (error) => this.#fail(new Error(`process_tree_transport_error: ${String(error.message)}`))); + child.on("close", (code, signal) => { + this.#closed = true; + this.#exitCode = code; + this.#exitSignal = signal; + if (!this.#closeHandled) this.#fail(new Error("process_tree_transport_closed")); + for (const waiter of this.#exitWaiters.splice(0)) waiter(); + }); + } + + /** Clears buffered receive state; used when a budget trips so a failed + * transport neither accumulates nor serves stale lines afterwards. */ + #failAndDrain(): void { + for (const line of this.#lines) this.#bufferedBytes -= Buffer.byteLength(line, "utf8"); + this.#lines = []; + } + + #fail(error: Error): void { + if (this.#failure) return; + this.#failure = error; + for (const waiter of this.#waiters.splice(0)) { + clearTimeout(waiter.timer); + waiter.reject(error); + } + } + + #nextLine(timeoutMs: number, what: string): Promise { + if (this.#failure) return Promise.reject(this.#failure); + // R4 addendum 13:13: EVERY line-taking path is bounded by the total + // session budget — a buffered line is not served after expiry, and its + // bytes return to the budget when taken. + const remainingSession = this.#sessionDeadline - Date.now(); + if (remainingSession <= 0) { + const error = new Error(`process_tree_transport_session_timeout: ${what}`); + this.#failAndDrain(); + this.#fail(error); + return Promise.reject(error); + } + const buffered = this.#lines.shift(); + if (buffered !== undefined) { + this.#bufferedBytes -= Buffer.byteLength(buffered, "utf8"); + return Promise.resolve(buffered); + } + if (this.#closed) return Promise.reject(new Error("process_tree_transport_closed")); + const boundedTimeout = Math.min(timeoutMs, remainingSession); + return new Promise((resolve, reject) => { + const timer = setTimeout(() => { + const index = this.#waiters.findIndex((w) => w.resolve === resolve); + if (index >= 0) this.#waiters.splice(index, 1); + // Distinguish which budget expired: the total session cap vs the + // individual command wait. + const reason = remainingSession < timeoutMs ? "process_tree_transport_session_timeout" : "process_tree_transport_timeout"; + reject(new Error(`${reason}: ${what}`)); + this.#fail(new Error(`${reason}: ${what}`)); + }, boundedTimeout); + if (typeof timer.unref === "function") timer.unref(); + this.#waiters.push({ resolve, reject, timer }); + }); + } + + static async start(timeoutMs = 20_000): Promise { + const encoded = Buffer.from(windowsTreeTransactionScript(), "utf16le").toString("base64"); + const child = spawn("powershell.exe", ["-NoProfile", "-NonInteractive", "-ExecutionPolicy", "Bypass", "-EncodedCommand", encoded], { + shell: false, + windowsHide: true, + stdio: ["pipe", "pipe", "pipe"], + }) as unknown as SpawnedChild; + const transaction = PowerShellTreeTransaction.attach(child); + try { + const ready = await transaction.#nextLine(timeoutMs, "helper startup"); + if (ready !== "READY") throw new Error("process_tree_transport_protocol: startup marker missing"); + return transaction; + } catch (error) { + // R4 addendum: a failed startup must converge its OWN helper — the + // kill is issued and the exit is awaited within bounds; an unconfirmed + // exit is reported, never assumed. + transaction.#closeHandled = true; + try { child.kill("SIGKILL"); } catch { /* already gone */ } + const exited = await transaction.#waitForExit(3_000); + if (!exited) { + throw new Error(`${error instanceof Error ? error.message : String(error)}; process_tree_close_failed: startup helper did not exit within bounds`); + } + throw error; + } + } + + async enumerate(): Promise { + this.#child.stdin.write("enum\n"); + const begin = await this.#nextLine(20_000, "enum begin"); + if (begin !== "ENUM_BEGIN") throw new Error("process_tree_snapshot_invalid: protocol missing"); + const rows: WindowsProcessRow[] = []; + for (;;) { + const line = await this.#nextLine(20_000, "enum row"); + if (line === "ENUM_END") break; + const match = /^R (\d{1,12}) (\d{1,12}) (\d{1,25}|unknown) (\d{1,25}|unknown)$/u.exec(line); + if (!match) throw new Error("process_tree_snapshot_invalid: malformed identity"); + const pid = Number(match[1]); + const parentPid = Number(match[2]); + // Structurally valid but never signalable rows (e.g. System Idle + // Process pid 0) are skipped; the snapshot itself stays valid. + if (!Number.isSafeInteger(pid) || pid <= 0 || !Number.isSafeInteger(parentPid) || parentPid < 0) continue; + if (rows.length >= this.#bounds.maxEnumRows) { + throw new Error(`process_tree_snapshot_invalid: enumeration exceeded ${String(this.#bounds.maxEnumRows)} rows`); + } + rows.push({ pid, parentPid, creationFileTime: match[3] === "unknown" ? null : match[3]!, startTicks: match[4] === "unknown" ? null : match[4]! }); + } + return rows; + } + + async hold(pids: readonly number[]): Promise { + if (pids.length === 0) return []; + this.#child.stdin.write(`hold ${pids.join(" ")}\n`); + const outcomes: HoldOutcome[] = []; + for (;;) { + const line = await this.#nextLine(20_000, "hold response"); + if (line === "HOLD_END") break; + const match = /^H (\d{1,12}) (ok (\d{1,25}) (\d{1,25})|held|absent (\d{1,6})|denied (\d{1,6})|error (\d{1,6}))$/u.exec(line); + if (!match) throw new Error("process_tree_hold_invalid: malformed response"); + const pid = Number(match[1]); + if (match[2]!.startsWith("ok")) { + outcomes.push({ pid, status: "ok", creationFileTime: match[3], creationTicks: match[4] }); + } else if (match[2] === "held") { + outcomes.push({ pid, status: "error", detail: "already_held" }); + } else if (match[2]!.startsWith("absent")) { + outcomes.push({ pid, status: "absent", error: Number(match[5] ?? 0) }); + } else if (match[2]!.startsWith("denied")) { + outcomes.push({ pid, status: "denied", error: Number(match[6] ?? 0) }); + } else { + outcomes.push({ pid, status: "error", error: Number(match[7] ?? 0) }); + } + } + return outcomes; + } + + async kill(pid: number, expectedTicks: string, waitMs: number): Promise { + this.#child.stdin.write(`kill ${String(pid)} ${expectedTicks} ${String(Math.max(0, Math.floor(waitMs)))}\n`); + const line = await this.#nextLine(waitMs + 25_000, "kill response"); + const end = await this.#nextLine(5_000, "kill terminator"); + if (end !== "KILL_END") throw new Error("process_tree_kill_invalid: terminator missing"); + const match = /^K (\d{1,12}) (signaled|identity_changed|not_held|wait_timeout|already_exited|error) (\d{1,25})$/u.exec(line); + if (!match || Number(match[1]) !== pid) throw new Error("process_tree_kill_invalid: malformed response"); + const status = match[2] as KillOutcome["status"]; + const value = match[3]!; + return { + status, + exitFileTime: status === "signaled" || status === "already_exited" || status === "wait_timeout" ? (value === "0" ? null : value) : null, + detail: status === "identity_changed" ? value : undefined, + }; + } + + async close(): Promise { + // R4: closing is verified, not assumed. The helper must acknowledge with + // a CLOSED receipt where failed==0 (every CloseHandle + // succeeded), and then actually exit within bounds. A forced-kill + // fallback converges the process but stays VISIBLE as a failure — the + // verified path never depends on it. + let failure: Error | null = this.#failure; + if (!failure && !this.#closed) { + try { + this.#child.stdin.write("close\n"); + const ack = await this.#nextLine(10_000, "close acknowledgement"); + const match = /^CLOSED (\d{1,9}) (\d{1,9})$/u.exec(ack); + if (!match) throw new Error(`process_tree_close_invalid: unexpected close receipt ${JSON.stringify(ack.slice(0, 60))}`); + const failedHandles = Number(match[2]); + if (failedHandles > 0) throw new Error(`process_tree_close_failed: ${String(failedHandles)} CloseHandle call(s) failed`); + this.#closeHandled = true; + } catch (error) { + failure = error instanceof Error ? error : new Error(String(error)); + this.#closeHandled = true; // its exit below is expected; report `failure` + } + } + const exited = await this.#waitForExit(5_000); + if (!exited) { + this.#closeHandled = true; + try { this.#child.kill("SIGKILL"); } catch { /* already gone */ } + try { this.#child.stdin.end(); } catch { /* already closed */ } + const exitedAfterKill = await this.#waitForExit(3_000); + if (!exitedAfterKill) { + throw new Error(failure ? `${failure.message}; process_tree_close_failed: helper did not exit within bounds` : "process_tree_close_failed: helper did not exit within bounds"); + } + throw new Error(failure ? `${failure.message}; process_tree_close_forced: helper exited only after a forced kill` : "process_tree_close_forced: helper exited only after a forced kill"); + } + // R4 addendum 13:13: a natural close must also be a CLEAN exit — a + // non-zero exit code or a signal on the helper is an explicit failure. + if (this.#exitCode !== 0 || this.#exitSignal !== null) { + const abnormal = `process_tree_close_abnormal: helper exited with code ${String(this.#exitCode)}${this.#exitSignal ? `, signal ${this.#exitSignal}` : ""}`; + throw new Error(failure ? `${failure.message}; ${abnormal}` : abnormal); + } + if (failure) throw failure; + if (this.#failure) throw this.#failure; // e.g. an error record hit stderr during shutdown + } + + #waitForExit(timeoutMs: number): Promise { + if (this.#closed) return Promise.resolve(true); + return new Promise((resolve) => { + let settled = false; + const finish = (value: boolean): void => { + if (settled) return; + settled = true; + clearTimeout(timer); + const index = this.#exitWaiters.indexOf(waiter); + if (index >= 0) this.#exitWaiters.splice(index, 1); + resolve(value); + }; + const waiter = (): void => finish(true); + const timer = setTimeout(() => finish(false), timeoutMs); + if (typeof timer.unref === "function") timer.unref(); + this.#exitWaiters.push(waiter); + }); + } +} + +/** Opens the real hidden PowerShell transaction helper. */ +export async function openPowerShellTreeTransaction(): Promise { + return PowerShellTreeTransaction.start(); } diff --git a/src/adapters/zcode-adapter.ts b/src/adapters/zcode-adapter.ts index 3e94b38..84cd942 100644 --- a/src/adapters/zcode-adapter.ts +++ b/src/adapters/zcode-adapter.ts @@ -36,6 +36,9 @@ import { parseZcodeEnvelope } from "./envelope.js"; import { parseAgentReport } from "./agent-report.js"; import { appendBounded, + captureSpawnGenerationIdentity, + childStillOnSpawnGeneration, + createPlatformProbe, defaultSpawnFunction, terminateProcessTree, type BoundedText, @@ -85,6 +88,9 @@ export interface ZCodeAdapterOptions { resolver?: RuntimeResolver; spawnImpl?: SpawnFunction; terminateProcessTreeImpl?: TerminateProcessTree; + /** Platform probe for the spawn-generation identity capture (ADR-009); + * defaults to the real platform probe. */ + probe?: import("../runtime/process-probe.js").ProcessProbe; /** Wall-clock budget per attempt; default 60 minutes. */ timeoutMs?: number; /** Capture cap per stream in bytes; default 10 MiB. */ @@ -127,6 +133,16 @@ interface RunEntry { readonly timeoutMs: number; child: SpawnedProcess | null; zcodePid: number | null; + /** ADR-009: spawn-generation identity captured while libuv still holds the + * child's native handle (before its close/exit event); Windows termination + * refuses to signal without it. */ + childIdentity: { pid: number; fingerprint: string } | null; + /** True once the CURRENT generation's close event fired. Reset on every + * retry spawn (review-round-2 C1); exit-state continuity per ADR-009 is + * decided from the current child's exitCode/signalCode, with this flag as + * the close-side bound — never as proof that a native handle is still + * held (that guarantee belongs to the exit state). */ + childClosed: boolean; cancelRequested: boolean; timedOut: boolean; finished: boolean; @@ -142,6 +158,7 @@ export class ZCodeAdapter implements CodingAgentAdapter { readonly #resolver: RuntimeResolver; readonly #spawnImpl: SpawnFunction; readonly #terminateImpl: TerminateProcessTree; + readonly #probe: import("../runtime/process-probe.js").ProcessProbe; readonly #timeoutMs: number | null; readonly #maxOutputBytes: number; readonly #retryBackoffMs: number; @@ -158,6 +175,7 @@ export class ZCodeAdapter implements CodingAgentAdapter { this.#resolver = options.resolver ?? new NodeRuntimeResolver(); this.#spawnImpl = options.spawnImpl ?? defaultSpawnFunction; this.#terminateImpl = options.terminateProcessTreeImpl ?? terminateProcessTree; + this.#probe = options.probe ?? createPlatformProbe(); this.#timeoutMs = options.timeoutMs ?? null; this.#maxOutputBytes = options.maxOutputBytes ?? DEFAULT_MAX_OUTPUT_BYTES; this.#retryBackoffMs = options.retryBackoffMs ?? DEFAULT_RETRY_BACKOFF_MS; @@ -286,6 +304,8 @@ export class ZCodeAdapter implements CodingAgentAdapter { timeoutMs: this.#timeoutMs ?? resolveTaskTimeout(input.task, this.#childEnvBase ?? process.env), child: null, zcodePid: null, + childIdentity: null, + childClosed: false, cancelRequested: false, timedOut: false, finished: false, @@ -369,6 +389,24 @@ export class ZCodeAdapter implements CodingAgentAdapter { } entry.child = child; entry.zcodePid = typeof child.pid === "number" ? child.pid : null; + // Per-generation reset (review-round-2 C1): the RunEntry is REUSED + // across the retry policy's spawn generations — every spawn-local + // field starts fresh so a closed previous generation can never + // short-circuit this generation's termination or leak its identity. + entry.childClosed = false; + entry.childIdentity = null; + // ADR-009: continuity is proven from the child's exit state (Node + // delivers exit before close and libuv releases the native handle at + // exit delivery). The shared capture re-verifies after every await and + // discards fingerprints that raced an exit or lack exact precision. + // A late capture from a PREVIOUS generation (its 250 ms retry window + // can outlive the spawn of the next) must never overwrite the current + // generation's identity, so the assignment is bound to this child. + if (process.platform === "win32" && typeof child.pid === "number") { + void captureSpawnGenerationIdentity(child, this.#probe).then((identity) => { + if (entry.child === child) entry.childIdentity = identity; + }); + } let stdout: BoundedText = { value: "", truncated: false }; let stderr: BoundedText = { value: "", truncated: false }; @@ -386,6 +424,7 @@ export class ZCodeAdapter implements CodingAgentAdapter { if (settled) return; settled = true; clearTimeout(timer); + entry.childClosed = true; entry.child = null; void entry.termination.then( () => resolveSnapshot(snapshot), @@ -410,11 +449,24 @@ export class ZCodeAdapter implements CodingAgentAdapter { if (entry.terminationStarted) return entry.termination; entry.terminationStarted = true; const pid = entry.zcodePid; - if (pid === null) { + const child = entry.child; + // ADR-009: on Windows, exit evidence ends the provable generation even + // before stdio close; POSIX keeps its existing close-based behaviour. + const generationRefused = process.platform === "win32" && child !== null + && !childStillOnSpawnGeneration({ pid: pid ?? undefined, exitCode: child.exitCode, signalCode: child.signalCode, exitSeen: entry.childClosed }); + if (pid === null || entry.childClosed || generationRefused) { + // No recorded pid, or the child's exit was already delivered (its + // native handle is released and the PID may have been recycled): + // nothing may be signalled (ADR-009). Windows refuses; POSIX callers + // only reach here for genuinely closed children. entry.termination = Promise.resolve(); return entry.termination; } - entry.termination = this.#terminateImpl(pid).then( + entry.termination = this.#terminateImpl(pid, { + graceMs: 500, + killWaitMs: 5_000, + expectedRootFingerprint: entry.childIdentity?.pid === pid ? entry.childIdentity.fingerprint : null, + }).then( () => undefined, (error: unknown) => { entry.terminationError = diff --git a/src/adapters/zcode-app-server-adapter.ts b/src/adapters/zcode-app-server-adapter.ts index 61ecb4d..9c00f1d 100644 --- a/src/adapters/zcode-app-server-adapter.ts +++ b/src/adapters/zcode-app-server-adapter.ts @@ -20,7 +20,8 @@ import type { ZCodeRunOutcome } from "./zcode-adapter.js"; import { buildContinuePrompt, buildTaskPrompt } from "../prompts/task-prompt.js"; import { BridgeError } from "../runtime/errors.js"; import { loadPersistedRuntimeEnvironment, NodeRuntimeResolver } from "../runtime/resolver.js"; -import { isProcessRunning, terminateProcessTree } from "./process-spawn.js"; +import { captureSpawnGenerationIdentity, childStillOnSpawnGeneration, isProcessRunning, terminateProcessTree } from "./process-spawn.js"; +import { createPlatformProbe, type ProcessProbe } from "../runtime/process-probe.js"; import { createMinimalOsEnv } from "../runtime/child-env.js"; import { catalogProviderId, buildAccountProviderPayload, runtimeAuthReply, zcodeDataBaseDir, zcodeTasksIndexPath } from "../runtime/account-provider.js"; import { resolveSessionPreferences } from "../runtime/session-preferences.js"; @@ -81,6 +82,12 @@ interface AppServerClient { interface RunEntry { handle: AgentHandle; child: ChildProcessWithoutNullStreams | null; + /** ADR-009: spawn-generation identity of `child`, captured while the + * libuv-held native handle keeps the PID unrecyclable (before 'exit'). */ + childIdentity: { pid: number; fingerprint: string } | null; + /** True once the child's 'exit' fired (native handle released; the PID may + * be recycled from that point and must never be signalled again). */ + childExitSeen: boolean; client: AppServerClient | null; sessionId: string | null; finished: boolean; @@ -171,6 +178,9 @@ export interface ZCodeAppServerAdapterOptions { turnBindingCompatMs?: number; /** Idle time before a replay poll (default 10_000); tests lower it. */ replayIdleMs?: number; + /** Platform probe used for the spawn-generation identity capture (ADR-009); + * defaults to the real platform probe. */ + probe?: ProcessProbe; } const RPC_TIMEOUT_MS = 30_000; @@ -201,6 +211,7 @@ export class ZCodeAppServerAdapter implements CodingAgentAdapter { readonly #turnBindingCompatMs: number; readonly #replayIdleMs: number; readonly #browser: AdapterBrowserOptions | undefined; + readonly #probe: ProcessProbe; readonly #runs = new Map(); readonly #workspaceByTask = new Map(); @@ -212,6 +223,7 @@ export class ZCodeAppServerAdapter implements CodingAgentAdapter { this.#homeDir = options.homeDir ?? homedir(); this.#host = validateHostProfile(options.host ?? codexHostProfile(this.#homeDir)); this.#now = options.now ?? (() => new Date()); + this.#probe = options.probe ?? createPlatformProbe(); this.#resolveInteraction = options.resolveInteraction; this.#onOutcomeCheckpoint = options.onOutcomeCheckpoint; this.#onInteractionState = options.onInteractionState; @@ -308,6 +320,8 @@ export class ZCodeAppServerAdapter implements CodingAgentAdapter { const entry: RunEntry = { handle, child: null, + childIdentity: null, + childExitSeen: false, client: null, sessionId: resumeSessionId, finished: false, @@ -835,7 +849,7 @@ export class ZCodeAppServerAdapter implements CodingAgentAdapter { await this.#closeBrowserHost(entry); try { if (entry.client) await entry.client.close(); - else if (entry.child?.pid) await terminateProcessTree(entry.child.pid); + else if (entry.child?.pid) await this.#terminateOwnedChild(entry); } catch (cleanupError) { const browserSuffix = entry.browserCleanupError ? ` Managed browser cleanup is also unverified: ${entry.browserCleanupError}` : ""; throw new BridgeError("cleanup_failed", `ZCode process termination could not be verified; workspace remains occupied.${browserSuffix}`, { cause: cleanupError }); @@ -881,6 +895,25 @@ export class ZCodeAppServerAdapter implements CodingAgentAdapter { let stdoutBuffer = ""; let rpcId = 0; let closed = false; + // ADR-009 spawn-generation binding: libuv releases the child's native + // handle when it delivers 'exit' (before stdio close), so continuity is + // tracked from the exit state. The shared capture re-verifies the + // generation after every await and discards fingerprints that raced an + // exit — a recycled PID's identity is never stored as this spawn's. + child.once("exit", () => { + entry.childExitSeen = true; + }); + const generationChild = { + get pid() { return child.pid; }, + get exitCode() { return child.exitCode; }, + get signalCode() { return child.signalCode; }, + get exitSeen() { return entry.childExitSeen; }, + }; + if (process.platform === "win32" && child.pid) { + void captureSpawnGenerationIdentity(generationChild, this.#probe).then((identity) => { + entry.childIdentity = identity; + }); + } const pending = new Map(); // ADR-008: a fatal wire-format failure ends every pending RPC and the turn // wait immediately, stops buffering, and reports stable reason + safe size @@ -1033,17 +1066,40 @@ export class ZCodeAppServerAdapter implements CodingAgentAdapter { child.once("close", () => { clearTimeout(timer); resolve(); }); }); // The close event is authoritative for this direct child. If it races - // the grace timer, check the PID before taskkill to avoid turning an - // already-exited app-server into a cleanup failure on Windows. + // the grace timer, check the PID before terminating to avoid turning an + // already-exited app-server into a cleanup failure on Windows. The + // Windows path additionally requires the spawn-captured identity + // (ADR-009): after 'exit' the PID may be recycled and is never + // signalled; without a captured identity it refuses (cleanup_failed). if (closed) { if (process.platform !== "win32" && child.pid) await terminateProcessTree(child.pid); return; } - if (child.pid && isProcessRunning(child.pid)) await terminateProcessTree(child.pid); + if (child.pid && isProcessRunning(child.pid)) { + if (entry.childExitSeen) return; + await terminateProcessTree(child.pid, { + graceMs: 500, + killWaitMs: 5_000, + expectedRootFingerprint: entry.childIdentity?.pid === child.pid ? entry.childIdentity.fingerprint : null, + }); + } }; return { child, request, replayEvents, close, get stderr() { return stderr; } }; } + /** ADR-009: terminate the adapter's own spawned child with the identity + * captured while its native handle was still held. Windows refuses to + * signal without it (a bare current PID is never trusted), and a child + * whose 'exit' already fired is never signalled again. POSIX keeps its + * existing process-group behaviour. */ + async #terminateOwnedChild(entry: RunEntry): Promise { + const pid = entry.child?.pid; + if (!pid) return; + if (entry.childExitSeen) return; + const fingerprint = entry.childIdentity?.pid === pid ? entry.childIdentity.fingerprint : null; + await terminateProcessTree(pid, { graceMs: 500, killWaitMs: 5_000, expectedRootFingerprint: fingerprint }); + } + #handleMessage( message: JsonRecord, entry: RunEntry, diff --git a/src/manager/task-manager.ts b/src/manager/task-manager.ts index 0211705..f2da504 100644 --- a/src/manager/task-manager.ts +++ b/src/manager/task-manager.ts @@ -31,7 +31,7 @@ import { buildTaskResult, type TaskFailure } from "./normalize.js"; import type { ZCodeRunOutcome } from "../adapters/zcode-adapter.js"; import { TaskManagerError } from "./errors.js"; import { defaultSpawnWorker, type SpawnWorker } from "./spawn-worker.js"; -import { isProcessRunning, terminateProcessTree, type TerminateProcessTree } from "../adapters/process-spawn.js"; +import { isProcessRunning, terminateProcessTree, verifyProcessTreeClaimantsClean, type TerminateProcessTree, type TreeClaimantTarget } from "../adapters/process-spawn.js"; import { validateTaskTimeout } from "../runtime/task-timeout.js"; import { tryAcquireProcessLock, withProcessLock, publishSelfIdentity } from "../store/process-lock.js"; import { buildTaskPrompt, buildContinuePrompt } from "../prompts/task-prompt.js"; @@ -51,6 +51,8 @@ export interface TaskManagerOptions { isProcessRunning?: (pid: number) => boolean; /** Defaults to real process-tree termination with verification. */ terminateProcessTree?: TerminateProcessTree; + /** Defaults to the read-only recorded-tree claimant scan (ADR-009). */ + verifyTreeClean?: (targets: readonly TreeClaimantTarget[]) => Promise<{ clean: boolean; livePids: readonly number[] }>; /** Defaults to the platform ProcessProbe (Windows PowerShell batch / Linux * /proc / macOS ps). Recovery and cleanup identity checks use it. */ probe?: ProcessProbe; @@ -77,6 +79,7 @@ export class BridgeTaskManager implements ProgressTaskManager { readonly #spawnWorker: SpawnWorker; readonly #isProcessRunning: (pid: number) => boolean; readonly #terminateProcessTree: TerminateProcessTree; + readonly #verifyTreeClean: (targets: readonly TreeClaimantTarget[]) => Promise<{ clean: boolean; livePids: readonly number[] }>; readonly #probe: ProcessProbe | null; readonly #now: () => Date; readonly #dataRoot: string; @@ -96,6 +99,7 @@ export class BridgeTaskManager implements ProgressTaskManager { this.#spawnWorker = options.spawnWorker ?? defaultSpawnWorker; this.#isProcessRunning = options.isProcessRunning ?? isProcessRunning; this.#terminateProcessTree = options.terminateProcessTree ?? terminateProcessTree; + this.#verifyTreeClean = options.verifyTreeClean ?? verifyProcessTreeClaimantsClean; this.#probe = options.probe ?? null; this.#now = options.now ?? (() => new Date()); this.#dataRoot = options.store.dataRoot; @@ -231,6 +235,21 @@ export class BridgeTaskManager implements ProgressTaskManager { return; } + // ADR-009 / review-round-1 R1: executor-exit evidence never clears a + // latched descendant-cleanup failure. When the failure record carries + // known member pids (or is corrupt), only a cancel's claimant-scan-gated + // release may clear the latch; the probe path stays occupied. + const failureRecordState = this.#cleanupFailureRecordState(taskId, status.attempt); + if (failureRecordState !== "none") { + const occurrence = this.#counters.record("terminal_cleanup_latched", failureRecordState); + if (this.#counters.shouldEmit("terminal_cleanup_latched", failureRecordState)) { + this.#store.appendEvent(taskId, "cleanup_unverified", "A recorded tree-cleanup failure with known members keeps the terminal task occupied; cancel runs the recorded-tree scan", { + record_state: failureRecordState, count: occurrence.count, + }, this.#now().toISOString()); + } + return; + } + await this.#exclusive(async () => { const current = this.#safeStatus(taskId); if (!current || !isTerminalStatus(current.status) || current.cleanup_unverified !== true) return; @@ -349,6 +368,170 @@ export class BridgeTaskManager implements ProgressTaskManager { } } + /** Latches a failed termination (ADR-009): occupancy is retained via + * cleanup_unverified and the member pids observed by the failed + * transaction are persisted so later claimant scans cover them. The + * complete member set is persisted structurally (cleanup-failure.json); + * the bounded [cleanup-members: …] marker in the status text is a durable + * fallback that is only trusted when complete — a truncated marker (or a + * missing/corrupt record) fails later releases closed instead of silently + * narrowing the scan (review-round-1 R3 + addendum). */ + #recordCleanupFailure(taskId: string, attempt: number, error: unknown): void { + const message = boundedErrorMessage(error, 300); + const freshPids = Array.isArray((error as { memberPids?: number[] }).memberPids) + ? ((error as { memberPids: number[] }).memberPids).filter((pid) => Number.isSafeInteger(pid) && pid > 0) + : []; + // Recover the previous set MONOTONICALLY (review-round-1 addendum + // 13:16 + 13:39): the prior latch marker is ALWAYS parsed and merged — + // even when the structural record reads back successfully — and its + // incomplete state (or the record's persisted complete:false flag) + // sticks forever: no later failure may wash a partial set back to + // completeness. + let metaPids: number[] = []; + let metaKnownIncomplete = false; + try { + const previous = this.#store.readAttemptMeta<{ member_pids?: unknown; complete?: unknown }>(taskId, attempt, "cleanup-failure.json"); + if (previous !== null) { + metaPids = this.#validMemberPids(previous.member_pids); + metaKnownIncomplete = previous.complete === false; + } + } catch { + // Corrupt/unreadable record: fall through to the marker below. + } + const priorMarker = this.#parseCleanupMembersMarker(this.#safeStatus(taskId)?.error); + const priorMarkerIncomplete = priorMarker !== null && !priorMarker.complete; + const previousPids = [...new Set([...metaPids, ...(priorMarker?.pids ?? [])])]; + const mergedPids = [...new Set([...previousPids, ...freshPids])]; + const joined = mergedPids.join(","); + const fitsWithoutTruncation = joined.length <= 1_600 && !(metaKnownIncomplete || priorMarkerIncomplete); + const memberMarker = mergedPids.length > 0 + ? (fitsWithoutTruncation ? ` [cleanup-members: ${joined}]` : ` [cleanup-members: ${mergedPids.slice(0, 200).join(",")},…]`) + : (metaKnownIncomplete || priorMarkerIncomplete ? " [cleanup-members: ,…]" : ""); + const recordComplete = fitsWithoutTruncation; + try { + this.#store.writeAttemptMeta(taskId, attempt, "cleanup-failure.json", { + member_pids: mergedPids, + complete: recordComplete, + last_error: message, + }); + } catch { + // The status marker below keeps the member set durable even without + // the meta record; an over-long or already-incomplete set keeps its + // truncation marker and later releases fail closed rather than scan + // a partial set. + } + try { + this.#store.writeStatus(taskId, { + cleanup_unverified: true, + error: `cancellation could not be verified: ${boundedErrorMessage(error, 200)}${memberMarker}`, + }); + this.#store.appendEvent(taskId, "cancel_failed", message, mergedPids.length > 0 ? { member_pids: mergedPids } : undefined, this.#now().toISOString()); + } catch { + // Status writes under the lock should not throw here; if they do the + // CANCEL_FAILED propagation still surfaces the failure. + } + } + + /** Validates a persisted member_pids value (review-round-1 addendum R3): a + * PRESENT record without the member_pids field, a null, or a non-array is + * corrupt and fails closed — "missing" must never read as "no members". */ + #validMemberPids(value: unknown): number[] { + if (value === undefined || value === null) { + throw new TaskManagerError("CANCEL_FAILED", "cleanup failure record is corrupt: member pid list is missing"); + } + if (!Array.isArray(value) || !value.every((pid) => Number.isSafeInteger(pid) && (pid as number) > 0)) { + throw new TaskManagerError("CANCEL_FAILED", "cleanup failure record is corrupt: member pid list is not a valid array of positive integers"); + } + return value as number[]; + } + + /** Parses the durable [cleanup-members: …] marker from a latched status + * error text. A TRUNCATED marker (…) is reported as incomplete: it must + * never be treated as the full member set (addendum R3 case 2). */ + #parseCleanupMembersMarker(errorText: string | null | undefined): { pids: number[]; complete: boolean } | null { + const marker = /\[cleanup-members: ([0-9,]+)(,…)?\]/u.exec(errorText ?? ""); + if (!marker) return null; + const pids = marker[1]!.split(",").map((part) => Number(part)).filter((pid) => Number.isSafeInteger(pid) && pid > 0); + return { pids, complete: marker[2] === undefined }; + } + + /** Classification of the persisted cleanup-failure state for an attempt: + * "none" (no record and no known-member marker — legacy latch), "empty" + * (explicitly member-less record), "members" (known member pids must stay + * covered, including via a complete latch marker), or "corrupt" (fail + * closed). A read error is treated as corrupt. */ + #cleanupFailureRecordState(taskId: string, attempt: number): "none" | "empty" | "members" | "corrupt" { + let meta: { member_pids?: unknown; complete?: unknown } | null = null; + try { + meta = this.#store.readAttemptMeta<{ member_pids?: unknown; complete?: unknown }>(taskId, attempt, "cleanup-failure.json") ?? null; + } catch { + return "corrupt"; + } + const marker = this.#parseCleanupMembersMarker(this.#safeStatus(taskId)?.error); + if (meta === null) { + // No structural record: a latch marker proves known members existed + // (terminal recovery must not treat this as legacy). + if (marker === null) return "none"; + return marker.complete ? "members" : "corrupt"; + } + let memberPids: number[]; + try { + memberPids = this.#validMemberPids(meta.member_pids); + } catch { + return "corrupt"; + } + // A record that declares itself partial (complete:false, written when the + // full set exceeded the durable marker) can never bound a release. + if (meta.complete === false) return "corrupt"; + return memberPids.length > 0 ? "members" : "empty"; + } + + /** Recorded pids (with persisted fingerprints when available) that later + * claimant scans must cover after a termination failure. The durable + * [cleanup-members: …] marker inside the latched status text is ALWAYS + * unioned in (R3): a missing or emptied failure record can never narrow + * the scan below what the latch itself recorded; a corrupt record — or a + * TRUNCATED marker that cannot restore the full set — fails closed before + * any scan runs. */ + #claimantTargets( + taskId: string, + attempt: number, + probeSnapshot: { worker_pid: number | null; zcode_pid: number | null; identity: { worker?: ProcessIdentityLike; runtime?: ProcessIdentityLike | null } | null }, + ): TreeClaimantTarget[] { + const targets: TreeClaimantTarget[] = []; + const addPid = (pid: number, fingerprint: string | null): void => { + if (Number.isSafeInteger(pid) && pid > 0 && !targets.some((t) => t.pid === pid)) targets.push({ pid, fingerprint }); + }; + const workerFingerprint = typeof probeSnapshot.identity?.worker?.fingerprint === "string" ? probeSnapshot.identity.worker.fingerprint : null; + const runtimeFingerprint = typeof probeSnapshot.identity?.runtime?.fingerprint === "string" ? probeSnapshot.identity.runtime.fingerprint : null; + if (probeSnapshot.worker_pid !== null) addPid(probeSnapshot.worker_pid, workerFingerprint); + if (probeSnapshot.zcode_pid !== null) addPid(probeSnapshot.zcode_pid, runtimeFingerprint); + // Durable fallback first: the latch text was written together with the + // failure; only a COMPLETE marker may restore the member set. + const marker = this.#parseCleanupMembersMarker(this.#safeStatus(taskId)?.error); + if (marker !== null && !marker.complete) { + throw new TaskManagerError("CANCEL_FAILED", "cleanup member record is incomplete: the latched member list was truncated and cannot be reconstructed; the task stays occupied"); + } + if (marker !== null) { + for (const pid of marker.pids) addPid(pid, null); + } + let meta: { member_pids?: unknown; complete?: unknown } | null = null; + try { + meta = this.#store.readAttemptMeta<{ member_pids?: unknown; complete?: unknown }>(taskId, attempt, "cleanup-failure.json") ?? null; + } catch { + // Read failure: the marker above still covers the recorded members; + // falling back to root-only would narrow coverage and is not done. + } + if (meta === null) return targets; + // A record that declares itself partial can never bound a release. + if (meta.complete === false) { + throw new TaskManagerError("CANCEL_FAILED", "cleanup member record is incomplete: the structured member list is marked partial; the task stays occupied"); + } + // A present-but-corrupt record must fail the release, not narrow it. + for (const pid of this.#validMemberPids(meta.member_pids)) addPid(pid, null); + return targets; + } + /** Applies confirmed probe verdicts to one unchanged attempt snapshot. */ #applyReconcileVerdicts( taskId: string, @@ -356,6 +539,20 @@ export class BridgeTaskManager implements ProgressTaskManager { snapshot: { attempt: number; worker_pid: number | null; zcode_pid: number | null; started_at: string | null; heartbeat: ReturnType }, verdicts: { worker: ProbeVerdict; runtime: ProbeVerdict }, ): void { + if (status.cleanup_unverified === true) { + // ADR-009 / review-round-1 R1: a latched descendant-cleanup failure is + // never cleared by recovery evidence — neither a persisted worker + // result, nor an outcome checkpoint, nor worker_lost finalization. + // Only a cancel's probe-verified, claimant-scan-gated release may + // clear it; occupancy is retained until then. + const occurrence = this.#counters.record("reconcile_latched_cleanup", "cleanup_unverified"); + if (this.#counters.shouldEmit("reconcile_latched_cleanup", "cleanup_unverified")) { + this.#store.appendEvent(taskId, "cleanup_unverified", "Reconciliation skipped: a recorded tree-cleanup failure keeps the task occupied until a verified cancel releases it", { + count: occurrence.count, + }, this.#now().toISOString()); + } + return; + } const persistedResult = this.#store.readResult(taskId); if (persistedResult && persistedResult.attempt === status.attempt) { // The worker wrote the result but died before updating the status file. @@ -888,6 +1085,8 @@ export class BridgeTaskManager implements ProgressTaskManager { // Identity re-verification always happens fresh here, never from cache // (A2): a reused PID must never be signalled for the old executor. const verdicts = await this.#probeExecutors(probeSnapshot); + const workerFingerprint = typeof probeSnapshot.identity?.worker?.fingerprint === "string" ? probeSnapshot.identity.worker.fingerprint : null; + const runtimeFingerprint = typeof probeSnapshot.identity?.runtime?.fingerprint === "string" ? probeSnapshot.identity.runtime.fingerprint : null; if (status.cleanup_unverified) { if (probeSnapshot.worker_pid === null && probeSnapshot.zcode_pid === null) { throw new TaskManagerError("CANCEL_FAILED", "No worker or ZCode process identity is available to verify cleanup"); @@ -898,41 +1097,42 @@ export class BridgeTaskManager implements ProgressTaskManager { if (probeSnapshot.worker_pid !== null && verdicts.worker.state === "unknown") { throw new TaskManagerError("CANCEL_FAILED", `Worker cleanup could not be verified: worker probe returned ${verdicts.worker.reason_code}`); } + const verifiedRoots = new Set(); try { if (probeSnapshot.zcode_pid !== null && verdicts.runtime.state === "alive") { - await this.#terminateProcessTree(probeSnapshot.zcode_pid, { graceMs: 500, killWaitMs: 5_000 }); + await this.#terminateProcessTree(probeSnapshot.zcode_pid, { graceMs: 500, killWaitMs: 5_000, expectedRootFingerprint: runtimeFingerprint }); const after = (await this.#probeExecutors(probeSnapshot)).runtime; if (after.state === "alive") throw new Error("runtime still alive after termination"); if (after.state === "unknown") throw new Error(`runtime exit could not be verified (${after.reason_code})`); + verifiedRoots.add(probeSnapshot.zcode_pid); } if (probeSnapshot.worker_pid !== null && verdicts.worker.state === "alive") { - await this.#terminateProcessTree(probeSnapshot.worker_pid, { graceMs: 500, killWaitMs: 5_000 }); + await this.#terminateProcessTree(probeSnapshot.worker_pid, { graceMs: 500, killWaitMs: 5_000, expectedRootFingerprint: workerFingerprint }); + verifiedRoots.add(probeSnapshot.worker_pid); } } catch (error) { - // A2-04: between the identity probe and taskkill the recorded - // process can exit naturally, so taskkill fails with "process not - // found" for a pid that is genuinely gone. The taskkill output is - // never trusted on its own: a fresh identity probe must prove the - // recorded runtime AND the recorded worker (when one applies) have - // both exited before this becomes an idempotent success. An unknown - // or alive recheck keeps cleanup_unverified and the directory - // occupied (A2-02/A2-05). - let recheck: { runtime: ProbeVerdict; worker: ProbeVerdict }; - try { - recheck = await this.#probeExecutors(probeSnapshot); - } catch (recheckError) { - throw new TaskManagerError("CANCEL_FAILED", `ZCode cleanup could not be verified: ${String(error)}; re-verification failed: ${String(recheckError)}`); - } - const runtimeRecorded = probeSnapshot.zcode_pid !== null; - const workerRecorded = probeSnapshot.worker_pid !== null; - if ((runtimeRecorded && recheck.runtime.state !== "exited") || (workerRecorded && recheck.worker.state !== "exited")) { - throw new TaskManagerError("CANCEL_FAILED", `ZCode cleanup could not be verified: ${String(error)}; re-verification observed runtime ${recheck.runtime.state} (${recheck.runtime.reason_code})${workerRecorded ? ` and worker ${recheck.worker.state} (${recheck.worker.reason_code})` : ""}`); + // ADR-009: a termination failure means the tree state is unverified. + // It is never cleared by executor-only exit evidence — a known + // descendant/ownership failure keeps the directory occupied, and a + // later cancel re-enters this branch after fresh probes. + this.#recordCleanupFailure(taskId, status.attempt, error); + throw new TaskManagerError("CANCEL_FAILED", `ZCode cleanup could not be verified: ${error instanceof Error ? error.message : String(error)}`); + } + // Recorded pids whose tree was not itself verified-terminated in + // this cancel are gated by the read-only claimant scan: releasing on + // root-executor exit evidence alone could orphan an untracked + // descendant (ADR-009; unobserved intermediate ancestors remain + // outside any proof and are logged, not claimed complete). + const scanTargets = this.#claimantTargets(taskId, status.attempt, probeSnapshot).filter((target) => !verifiedRoots.has(target.pid)); + if (scanTargets.length > 0) { + const scan = await this.#verifyTreeClean(scanTargets).catch((error: unknown) => { + throw new TaskManagerError("CANCEL_FAILED", `ZCode cleanup verification failed: ${error instanceof Error ? error.message : String(error)}`); + }); + if (!scan.clean) { + this.#store.appendEvent(taskId, "cleanup_unverified", "Recorded-tree scan still observes live claimants; the task stays occupied", { live_pids: [...scan.livePids] }, this.#now().toISOString()); + throw new TaskManagerError("CANCEL_FAILED", `ZCode cleanup could not be verified: ${String(scan.livePids.length)} recorded-tree process(es) still alive`); } - this.#store.appendEvent(taskId, "cleanup_race_resolved", "Termination raced a natural exit; a fresh identity probe confirmed the recorded processes exited", { - runtime_reason_code: recheck.runtime.reason_code, - worker_reason_code: recheck.worker.reason_code, - }, this.#now().toISOString()); } if (status.status === "running") { const finishedAt = this.#now().toISOString(); @@ -1084,22 +1284,22 @@ export class BridgeTaskManager implements ProgressTaskManager { try { if (!(await waitForSafeExit())) { if (process.platform !== "win32" && status.zcode_pid && this.#isProcessRunning(status.zcode_pid)) { - await this.#terminateProcessTree(status.zcode_pid, { graceMs: 500, killWaitMs: 5_000 }); + await this.#terminateProcessTree(status.zcode_pid, { graceMs: 500, killWaitMs: 5_000, expectedRootFingerprint: runtimeFingerprint }); } - await this.#terminateProcessTree(pid, { graceMs: 500, killWaitMs: 5_000 }); + await this.#terminateProcessTree(pid, { graceMs: 500, killWaitMs: 5_000, expectedRootFingerprint: workerFingerprint }); } } catch (error) { - if (!(await waitForSafeExit())) { - const message = error instanceof Error ? error.message : String(error); - this.#store.writeStatus(taskId, { - error: `cancellation could not be verified: ${message}`, - }); - this.#store.appendEvent(taskId, "cancel_failed", message, undefined, this.#now().toISOString()); - throw new TaskManagerError( - "CANCEL_FAILED", - `process-tree termination for task ${taskId} (pid ${pid}) could not be verified: ${message}`, - ); - } + // ADR-009: a termination failure is never absolved by the worker's + // own exit or result arriving meanwhile — descendants were not proven + // exited, so verified is not replaced by root exit and the directory + // stays occupied. A later cancel re-enters the cleanup_unverified + // flow with fresh probes and the recorded-tree scan. + this.#recordCleanupFailure(taskId, status.attempt, error); + const message = error instanceof Error ? error.message : String(error); + throw new TaskManagerError( + "CANCEL_FAILED", + `process-tree termination for task ${taskId} (pid ${pid}) could not be verified: ${message}`, + ); } // The worker may have completed while termination was in flight; a // persisted terminal result always wins over our cancellation. diff --git a/src/runtime/model-settings.ts b/src/runtime/model-settings.ts index 9e62f6b..0c29842 100644 --- a/src/runtime/model-settings.ts +++ b/src/runtime/model-settings.ts @@ -10,7 +10,7 @@ import { BridgeError } from "./errors.js"; import { loadPersistedRuntimeEnvironment, NodeRuntimeResolver } from "./resolver.js"; import { ZCODE_SESSION_MODES, type ZCodeSessionMode } from "./session-preferences.js"; import type { ZCodeRuntimeConfig } from "../interfaces.js"; -import { terminateProcessTree } from "../adapters/process-spawn.js"; +import { captureSpawnGenerationIdentity, childStillOnSpawnGeneration, createPlatformProbe, terminateProcessTree } from "../adapters/process-spawn.js"; import { codexHostProfile, validateHostProfile, type BridgeHostProfile } from "../host/profile.js"; import { withProcessLock } from "../store/process-lock.js"; import { atomicRename } from "../store/atomic-rename.js"; @@ -322,6 +322,28 @@ function startAppServer(config: ZCodeRuntimeConfig, cwd: string, env: NodeJS.Pro let stderr = ""; let nextId = 0; let closed = false; + // ADR-009 spawn-generation continuity: libuv releases the child's native + // handle when it delivers 'exit' (before stdio close), so continuity is + // tracked from the exit state, never from a close flag. The identity + // capture re-verifies the generation after every await and discards + // results that raced an exit (a recycled PID's fingerprint must never be + // stored as this spawn's identity). + let childExitSeen = false; + child.once("exit", () => { + childExitSeen = true; + }); + const generationChild = { + get pid() { return child.pid; }, + get exitCode() { return child.exitCode; }, + get signalCode() { return child.signalCode; }, + get exitSeen() { return childExitSeen; }, + }; + let childIdentity: { pid: number; fingerprint: string } | null = null; + if (process.platform === "win32" && child.pid) { + void captureSpawnGenerationIdentity(generationChild, createPlatformProbe()).then((identity) => { + childIdentity = identity; + }); + } child.stdout.setEncoding("utf8"); child.stderr.setEncoding("utf8"); const write = (message: JsonRecord): void => { child.stdin.write(`${JSON.stringify(message)}\n`); }; @@ -394,7 +416,17 @@ function startAppServer(config: ZCodeRuntimeConfig, cwd: string, env: NodeJS.Pro child.once("close", () => { clearTimeout(timer); resolve(); }); }); } - if (child.pid && (!closed || process.platform !== "win32")) await terminateProcessTree(child.pid); + if (child.pid && (!closed || process.platform !== "win32")) { + // Windows: only signal while the child is provably on its original + // spawn generation; after exit the PID may be recycled and a missing + // captured identity is a conservative refusal, never bare-PID trust. + if (process.platform === "win32" && !childStillOnSpawnGeneration(generationChild)) return; + await terminateProcessTree(child.pid, { + graceMs: 500, + killWaitMs: 5_000, + expectedRootFingerprint: childIdentity?.pid === child.pid ? childIdentity.fingerprint : null, + }); + } }; return { child, request, close, get stderr() { return stderr; } }; } diff --git a/src/store/process-lock.ts b/src/store/process-lock.ts index cb2d760..0d81bc1 100644 --- a/src/store/process-lock.ts +++ b/src/store/process-lock.ts @@ -9,6 +9,45 @@ import type { ProcessIdentity } from "../runtime/process-probe.js"; * recycled PID to the previous owner (A2). */ let selfIdentity: ProcessIdentity | null = null; +/** Errno codes that can appear while the lock directory's neighborhood is in + * motion. Directly observed under real cross-process contention: EPERM on + * the withdrawal rename. Plausible but not handle-level verified: a delete- + * pending destination name, a competitor or scanner holding a handle inside + * the directory (libuv maps sharing violations to EBUSY, access denial to + * EPERM/EACCES). Node's own recursive rm documents EBUSY/EMFILE/ENFILE/ + * ENOTEMPTY/EPERM as retryable for the same family. Everything outside this + * set fails immediately and visibly. */ +const TRANSIENT_LOCK_FS_CODES = new Set(["EEXIST", "EPERM", "EACCES", "EBUSY", "ENOTEMPTY"]); +/** rename(2) produces these ONLY when the destination already existed as a + * NON-EMPTY directory: POSIX specifies EEXIST/ENOTEMPTY for renaming onto + * an existing non-empty directory (an existing EMPTY directory is instead + * silently replaced — the independently reproduced issue that motivated the + * existence precheck above). Their appearance therefore proves definite + * namespace contention — a competing publication landed between the + * existence precheck and the rename (and may already have withdrawn again + * before any recheck). Contention is never an escaping error: the + * acquisition hands off to the existing-owner policies, and the caller's + * bounded outer loop (withEventLock/withProcessLock deadlines) retries. + * This classification was added after a real Linux full-suite run let + * ENOTEMPTY escape when rapid publish/withdraw cycles outlived the bounded + * publication retries (2026-10-17 matrix review). */ +const DESTINATION_PRESENT_CONTENTION_CODES = new Set(["EEXIST", "ENOTEMPTY"]); +/** Bounded budgets for the transient retries; exhaustion keeps the original + * error visible instead of spinning. */ +const PUBLICATION_RETRY_ATTEMPTS = 3; +const WITHDRAW_RETRY_MS = 2_000; +const SHARED_GUARD_RETRY_MS = 2_000; +const PRIVATE_CLEANUP_RETRY_MS = 1_000; +const RETRY_PAUSE_MS = 10; + +function isTransientLockFsError(error: unknown): boolean { + return TRANSIENT_LOCK_FS_CODES.has((error as NodeJS.ErrnoException)?.code ?? ""); +} + +function pauseBriefly(): void { + Atomics.wait(new Int32Array(new SharedArrayBuffer(4)), 0, 0, RETRY_PAUSE_MS); +} + export function publishSelfIdentity(identity: ProcessIdentity): void { selfIdentity = identity; } @@ -49,6 +88,15 @@ export function tryAcquireProcessLock(directory: string): (() => void) | null { function tryAcquireProcessLockWithRetry(directory: string, retries: number): (() => void) | null { const token = randomUUID(); + // Publishing must never overwrite an existing lock path, whatever it holds. + // POSIX rename silently REPLACES an existing empty directory (observed in + // the rootless Linux sandbox: an empty unknown-owner lock was consumed, + // 2026-10-17 review). An empty, foreign, or unreadable lock is an unknown + // owner that only the explicit acquireExistingLock policies may resolve — + // never a silent takeover. The staged rename below stays the atomic + // publication point for a genuinely free path; a lock that appears inside + // the check-to-rename window still routes through the rename's error path. + if (existsSync(directory)) return acquireExistingLock(directory); // Publish only a complete owner record. If the process exits during the // write, its private staging directory is harmless and the shared lock // path remains absent. Directory rename is the single publication point. @@ -70,19 +118,28 @@ function tryAcquireProcessLockWithRetry(directory: string, retries: number): (() renameSync(staging, directory); } catch (error) { removeStagedLock(staging); - // Windows may report EEXIST or a directory-specific error when another - // process wins the publication race. Only inspect it if the destination - // actually exists. If a competing owner released between the failed - // rename and this check, retry the publication a few times rather than - // surfacing a transient EPERM/EEXIST as a failed store transaction. - if (!existsSync(directory)) { - const code = (error as NodeJS.ErrnoException).code; - if (retries < 3 && (code === "EEXIST" || code === "EPERM" || code === "EACCES")) { - return tryAcquireProcessLockWithRetry(directory, retries + 1); - } - throw error; + const code = (error as NodeJS.ErrnoException).code; + // Any code outside the transient set (EIO, EINVAL, ...) is a real + // failure and must stay visible even when a live owner exists — + // swallowing it here turned an I/O error into a silent "busy" + // (2026-10-17 review). + if (!isTransientLockFsError(error)) throw error; + // EEXIST/ENOTEMPTY definitively describe an intervening competing + // destination (see DESTINATION_PRESENT_CONTENTION_CODES); a destination + // that still exists does too. Either way this is contention, not an + // acquisition failure: hand off to the existing-owner policies — a lock + // that already withdrew again reads as busy there, and the caller's + // bounded outer loop retries. Only the ambiguous absent-destination + // codes (EPERM/EACCES/EBUSY — possibly persistent access failures) + // consume the bounded publication retries and then throw visibly. + if (DESTINATION_PRESENT_CONTENTION_CODES.has(code ?? "") || existsSync(directory)) { + return acquireExistingLock(directory); + } + if (retries < PUBLICATION_RETRY_ATTEMPTS) { + pauseBriefly(); + return tryAcquireProcessLockWithRetry(directory, retries + 1); } - return acquireExistingLock(directory); + throw error; } } catch (error) { removeStagedLock(staging); @@ -94,21 +151,41 @@ function tryAcquireProcessLockWithRetry(directory: string, retries: number): (() // Withdraw the complete owner atomically before deleting anything. A // crash during private cleanup must not leave an unreadable shared lock. const retired = `${directory}.${token}.retired`; - renameSync(directory, retired); + renameWithTransientRetry(directory, retired); removeStagedLock(retired); }; } +/** Observed (real cross-process contention, 2026-10-17): the withdrawal + * rename of the held lock directory fails transiently with EPERM. The + * mechanism is a working hypothesis, not handle-level verified: a competitor + * inspecting owner.json (acquireExistingLock) or a scanner plausibly holds a + * handle inside the directory for milliseconds, which Windows refuses to + * rename across. Retry only the whitelisted codes within a bounded budget; + * on exhaustion the error stays visible and the shared lock keeps its + * complete owner record, so the next acquisition still sees a live owner + * instead of an unreadable or half-deleted lock. */ +function renameWithTransientRetry(source: string, target: string): void { + const deadline = Date.now() + WITHDRAW_RETRY_MS; + while (true) { + try { renameSync(source, target); return; } + catch (error) { + if (!isTransientLockFsError(error) || Date.now() >= deadline) throw error; + pauseBriefly(); + } + } +} + function acquireExistingLock(directory: string): (() => void) | null { + // Reading the owner is one concern; acting on it is another. Only a failure + // to READ a well-formed owner may fall back to the unreadable-lock policy + // below — errors raised while reclaiming a verified-dead owner (for example + // a shared guard that cannot be removed) must propagate, not be swallowed + // by a catch-all (2026-10-17 review). + let owner: { pid: number }; try { - const owner = JSON.parse(readFileSync(path.join(directory, "owner.json"), "utf8")) as { pid: number }; + owner = JSON.parse(readFileSync(path.join(directory, "owner.json"), "utf8")) as { pid: number }; if (!Number.isSafeInteger(owner.pid) || owner.pid <= 0) throw new Error("invalid lock owner"); - try { process.kill(owner.pid, 0); } - catch (failure) { - if ((failure as NodeJS.ErrnoException).code === "ESRCH") { - if (reclaimDeadOwner(directory)) return tryAcquireProcessLock(directory); - } - } } catch { // Never reclaim an unreadable legacy lock by age. It needs explicit // diagnosis; new acquisitions cannot create this state after staged publish. @@ -117,28 +194,119 @@ function acquireExistingLock(directory: string): (() => void) | null { } catch (failure) { if ((failure as NodeJS.ErrnoException).code !== "ENOENT") throw failure; } + return null; + } + try { process.kill(owner.pid, 0); } + catch (failure) { + if ((failure as NodeJS.ErrnoException).code === "ESRCH") { + if (reclaimDeadOwner(directory)) return tryAcquireProcessLock(directory); + } } return null; } function removeStagedLock(directory: string): void { - try { unlinkSync(path.join(directory, "owner.json")); } catch { /* staging may not have reached the write */ } - try { rmdirSync(directory); } catch { /* preserve the original acquisition error */ } + removePrivateFile(path.join(directory, "owner.json")); + removePrivateDirectory(directory); +} + +/** Best-effort removal of a lock's private owner record. A transiently + * blocked unlink (EPERM observed) must be retried, or the surviving + * owner.json keeps the retired directory non-empty forever — retrying only + * the rmdir cannot fix an undeleted file (2026-10-17 review). Exhaustion + * stays silent and private (ADR-005); it must never mask the original + * acquisition error. */ +function removePrivateFile(file: string): void { + const deadline = Date.now() + PRIVATE_CLEANUP_RETRY_MS; + while (true) { + try { unlinkSync(file); return; } + catch (error) { + const code = (error as NodeJS.ErrnoException)?.code; + if (code === "ENOENT" || !isTransientLockFsError(error) || Date.now() >= deadline) return; + pauseBriefly(); + } + } +} + +/** Best-effort removal of a lock's PRIVATE residue directories (staging and + * retired). Retries the transient set briefly so Windows handle timing does + * not leave garbage behind, but never surfaces or replaces an error: those + * leftovers are private (ADR-005). The reclaim guard is deliberately NOT + * handled here — it is shared state whose removal failure must stay visible + * (removeSharedGuard above). */ +function removePrivateDirectory(directory: string): void { + const deadline = Date.now() + PRIVATE_CLEANUP_RETRY_MS; + while (true) { + try { rmdirSync(directory); return; } + catch (error) { + const code = (error as NodeJS.ErrnoException)?.code; + if (code === "ENOENT" || !isTransientLockFsError(error) || Date.now() >= deadline) return; + pauseBriefly(); + } + } } function reclaimDeadOwner(directory: string): boolean { const guard = `${directory}.reclaim`; try { mkdirSync(guard); } catch (error) { if ((error as NodeJS.ErrnoException).code === "EEXIST") return false; throw error; } + // Every exit after the guard exists funnels through exactly one removal so + // no path strands it: the guard is shared coordination state and a leftover + // blocks all future dead-owner reclaims of this lock path (the owner-turns- + // live recheck deterministically stranded one, 2026-10-17 review). Removal + // failure is visible on every outcome; when the reclaim operation itself + // failed, that original error wins over the cleanup error. + let outcome: { kind: "reclaimed" } | { kind: "busy" } | { kind: "error"; error: unknown }; try { const owner = JSON.parse(readFileSync(path.join(directory, "owner.json"), "utf8")) as { pid: number }; - if (!Number.isSafeInteger(owner.pid) || owner.pid <= 0) return false; - try { process.kill(owner.pid, 0); return false; } - catch (error) { if ((error as NodeJS.ErrnoException).code !== "ESRCH") return false; } - const retired = `${directory}.${randomUUID()}.retired`; - renameSync(directory, retired); - removeStagedLock(retired); - return true; - } catch (error) { if ((error as NodeJS.ErrnoException).code !== "ENOENT") throw error; return false; } - finally { rmdirSync(guard); } + if (!Number.isSafeInteger(owner.pid) || owner.pid <= 0) { + outcome = { kind: "busy" }; // invalid owner record: needs explicit diagnosis, never reclaimed here + } else { + try { + process.kill(owner.pid, 0); + outcome = { kind: "busy" }; // owner is live again: plain contention, not death + } catch (failure) { + if ((failure as NodeJS.ErrnoException).code !== "ESRCH") { + outcome = { kind: "busy" }; // unknown liveness is never treated as death + } else { + const retired = `${directory}.${randomUUID()}.retired`; + renameWithTransientRetry(directory, retired); + removeStagedLock(retired); + outcome = { kind: "reclaimed" }; + } + } + } + } catch (error) { + outcome = (error as NodeJS.ErrnoException).code === "ENOENT" + ? { kind: "busy" } // the lock vanished mid-recheck: a normal race, not an error + : { kind: "error", error }; + } + let guardCleanupError: unknown = null; + try { removeSharedGuard(guard); } + catch (error) { guardCleanupError = error; } + if (outcome.kind === "error") throw outcome.error; + if (guardCleanupError !== null) throw guardCleanupError; + return outcome.kind === "reclaimed"; +} + +/** Removes the reclaim guard. Non-transient failures and budget exhaustion + * throw with the guard path spelled out for manual diagnosis; unlike private + * retired/staging residue this shared directory cannot be silently abandoned. */ +function removeSharedGuard(guard: string): void { + const deadline = Date.now() + SHARED_GUARD_RETRY_MS; + while (true) { + try { rmdirSync(guard); return; } + catch (error) { + const code = (error as NodeJS.ErrnoException)?.code; + if (code === "ENOENT") return; + if (!isTransientLockFsError(error) || Date.now() >= deadline) { + throw new Error( + `could not remove the lock reclaim guard at ${guard} (${code ?? "unknown error"}); ` + + "a leftover guard blocks all future dead-owner reclaims of this lock and needs manual diagnosis", + { cause: error }, + ); + } + pauseBriefly(); + } + } } diff --git a/src/store/task-store.ts b/src/store/task-store.ts index 29cc56d..f1bd142 100644 --- a/src/store/task-store.ts +++ b/src/store/task-store.ts @@ -305,7 +305,21 @@ export class TaskStore { this.assertWorkerAttempt(taskId, attempt); this.#writeJsonAtomic(path.join(this.taskDir(taskId), "result.json"), result); const current = this.readStatus(taskId); - this.#writeJsonAtomic(path.join(this.taskDir(taskId), "status.json"), { ...current, ...patch, task_id: taskId, attempt, updated_at: new Date().toISOString() }); + // ADR-009 / review-round-1 R1 (worker-commit addendum): the worker's + // final commit always records its result, but while a cleanup failure + // is latched it can NEVER dissolve it or overwrite its evidence — no + // matter whether the patch clears, re-asserts, or omits + // cleanup_unverified, and no matter what error text it carries. The + // worker's own failure detail stays available in result.json; only a + // manager-verified, scan-gated release clears the latch. + let effective = patch; + if (current.cleanup_unverified === true) { + effective = { ...patch, cleanup_unverified: true }; + if (typeof current.error === "string" && current.error.includes("[cleanup-members:")) { + effective = { ...effective, error: current.error }; + } + } + this.#writeJsonAtomic(path.join(this.taskDir(taskId), "status.json"), { ...current, ...effective, task_id: taskId, attempt, updated_at: new Date().toISOString() }); }); } diff --git a/src/worker/run-task.ts b/src/worker/run-task.ts index 4b66870..5afed23 100644 --- a/src/worker/run-task.ts +++ b/src/worker/run-task.ts @@ -101,10 +101,36 @@ export async function runWorkerTask(options: RunWorkerTaskOptions): Promise>(); + let identityWritesClosed = false; + const trackIdentityWrite = (write: Promise): void => { + identityWrites.add(write); + void write.then(() => identityWrites.delete(write), () => identityWrites.delete(write)); + }; + const reapIdentityWrites = async (): Promise => { + if (identityWritesClosed) return; + const deadline = Date.now() + IDENTITY_WRITE_REAP_MS; + while (identityWrites.size > 0 && Date.now() < deadline) await sleep(25); + identityWritesClosed = true; + }; + // Persist this executor's OS identity so recovery probes can distinguish a // dead worker from a recycled PID (A2). Best effort and asynchronous. - void probe.selfIdentity().then( + trackIdentityWrite(probe.selfIdentity().then( (identity) => { + if (identityWritesClosed) return; store.writeExecutorIdentity(taskId, attempt, { worker: identity as unknown as Record }); observationRevision += 1; try { @@ -112,452 +138,462 @@ export async function runWorkerTask(options: RunWorkerTaskOptions): Promise undefined, - ); + )); - const heartbeat: WorkerHeartbeat = { - attempt, - worker_pid: process.pid, - started_at: startedAt, - heartbeat_at: now().toISOString(), - heartbeat_seq: 0, - session_id: initialStatus.zcode_session_id ?? null, - turn_id: null, - last_event_seq: 0, - last_event_type: "worker_running", - zcode_event_seq: 0, - }; - const persistHeartbeat = (): void => { - heartbeat.heartbeat_at = now().toISOString(); - heartbeat.heartbeat_seq += 1; - store.writeWorkerHeartbeat(taskId, attempt, heartbeat); - writePhase(); // B3 snapshot rides the heartbeat cadence; heartbeats never touch events - }; - persistHeartbeat(); - const heartbeatTimer = setInterval(() => { - try { persistHeartbeat(); } - catch { /* the attempt is already stale/terminal; its owner must stop */ } - }, 3_000); - heartbeatTimer.unref(); - - const continueSpec = store.readAttemptMeta(taskId, attempt, "continue.json"); - const previousResult = continueSpec - ? store.readArchivedResult(taskId, continueSpec.previous_attempt ?? attempt - 1) - : null; - - // Archive the exact prompt for this attempt (pure rebuild from the same - // inputs the adapter uses). - const promptText = continueSpec - ? buildContinuePrompt({ - task, - feedback: continueSpec.feedback, - additionalRequirements: continueSpec.additional_requirements ?? [], - previousSessionId: continueSpec.previous_session_id, - previousResult, - }) - : buildTaskPrompt(task); - store.writeAttemptFile(taskId, attempt, "prompt.txt", promptText); - - // B2 correlation record: persisted timestamps for RPC accept, turn start, - // and turn completion, written as the events arrive. - const correlation: { rpc_accepted_at: string | null; turn_started_at: string | null; turn_completed_at: string | null; turn_id: string | null; session_id: string | null } = { - rpc_accepted_at: null, - turn_started_at: null, - turn_completed_at: null, - turn_id: null, - session_id: initialStatus.zcode_session_id ?? null, - }; - const persistCorrelation = (): void => { - try { store.writeAttemptMeta(taskId, attempt, "turn-correlation.json", { ...correlation }); } - catch { /* advisory evidence */ } - }; - const persistCorrelationTimer = setInterval(persistCorrelation, 5_000); - persistCorrelationTimer.unref(); - - // B4 runtime-confirmed execution profile (model / reasoning depth). - const modelProfile: ExecutionProfile = { - executor: "zcode", - provider_id: null, - model_id: null, - requested_model: task.model ? `${task.model.provider_id}/${task.model.model_id}` : null, - requested_reasoning_level: task.model?.reasoning_level ?? null, - effective_reasoning_level: null, - effective_reasoning_level_source: "not_reported", - selection_source: null, - effective_at: null, - session_id: initialStatus.zcode_session_id ?? null, - turn_id: null, - }; - - let outcome: ZCodeRunOutcome | null = null; - let failure: TaskFailure | null = null; - let pendingModelOutput = ""; - let lastModelOutputAt = 0; - const flushModelOutput = (): void => { - if (!pendingModelOutput) return; - store.assertWorkerAttempt(taskId, attempt); - for (let offset = 0; offset < pendingModelOutput.length; offset += 2_000) store.appendEvent(taskId, "model_output", pendingModelOutput.slice(offset, offset + 2_000)); - pendingModelOutput = ""; - lastModelOutputAt = Date.now(); - }; + let heartbeatTimer: ReturnType | undefined; + let persistCorrelationTimer: ReturnType | undefined; try { - if (options.resolver) { - // Surface configuration problems before spending an adapter run; the - // real adapter resolves internally as well. - await options.resolver.resolve(); - } - const adapter = options.adapter ?? new ZCodeAppServerAdapter({ - host: options.host, - ...(options.resolver ? { resolver: options.resolver } : {}), - onEvent: (event) => { - store.assertWorkerAttempt(taskId, attempt); - if (event.type === "model_output") { - pendingModelOutput += event.summary; - if (pendingModelOutput.length >= 4_000 || Date.now() - lastModelOutputAt >= 500) flushModelOutput(); - return; - } - flushModelOutput(); - const persistedEvent = store.appendEvent(taskId, event.type, event.summary, event.details); - if (persistedEvent) { - heartbeat.last_event_seq = persistedEvent.seq; - heartbeat.last_event_type = event.type; - } - const sessionId = event.type === "session_ready" ? event.details?.["session_id"] : undefined; - if (typeof sessionId === "string") { - heartbeat.session_id = sessionId; - correlation.session_id = sessionId; - modelProfile.session_id = sessionId; - store.writeStatus(taskId, { zcode_session_id: sessionId }, attempt); - } - if (event.type === "turn_started") { - correlation.rpc_accepted_at = correlation.rpc_accepted_at ?? now().toISOString(); - correlation.turn_started_at = now().toISOString(); - if (typeof event.details?.["turn_id"] === "string") { - heartbeat.turn_id = event.details["turn_id"]; - correlation.turn_id = event.details["turn_id"]; - modelProfile.turn_id = event.details["turn_id"]; - } - activityPhase = "executing"; - writePhase(); - } - if (event.type === "turn_completed") { - correlation.turn_completed_at = now().toISOString(); - activityPhase = "finalizing"; - writePhase(); - } - if (event.type === "model_selected") { - // Runtime-confirmed model evidence; requested values never impersonate it. - if (typeof event.details?.["provider_id"] === "string") modelProfile.provider_id = event.details["provider_id"]; - if (typeof event.details?.["model_id"] === "string") modelProfile.model_id = event.details["model_id"]; - if (typeof event.details?.["selected_model"] === "string" && modelProfile.model_id === null) { - modelProfile.model_id = event.details["selected_model"]; + + const heartbeat: WorkerHeartbeat = { + attempt, + worker_pid: process.pid, + started_at: startedAt, + heartbeat_at: now().toISOString(), + heartbeat_seq: 0, + session_id: initialStatus.zcode_session_id ?? null, + turn_id: null, + last_event_seq: 0, + last_event_type: "worker_running", + zcode_event_seq: 0, + }; + const persistHeartbeat = (): void => { + heartbeat.heartbeat_at = now().toISOString(); + heartbeat.heartbeat_seq += 1; + store.writeWorkerHeartbeat(taskId, attempt, heartbeat); + writePhase(); // B3 snapshot rides the heartbeat cadence; heartbeats never touch events + }; + persistHeartbeat(); + heartbeatTimer = setInterval(() => { + try { persistHeartbeat(); } + catch { /* the attempt is already stale/terminal; its owner must stop */ } + }, 3_000); + heartbeatTimer.unref(); + + const continueSpec = store.readAttemptMeta(taskId, attempt, "continue.json"); + const previousResult = continueSpec + ? store.readArchivedResult(taskId, continueSpec.previous_attempt ?? attempt - 1) + : null; + + // Archive the exact prompt for this attempt (pure rebuild from the same + // inputs the adapter uses). + const promptText = continueSpec + ? buildContinuePrompt({ + task, + feedback: continueSpec.feedback, + additionalRequirements: continueSpec.additional_requirements ?? [], + previousSessionId: continueSpec.previous_session_id, + previousResult, + }) + : buildTaskPrompt(task); + store.writeAttemptFile(taskId, attempt, "prompt.txt", promptText); + + // B2 correlation record: persisted timestamps for RPC accept, turn start, + // and turn completion, written as the events arrive. + const correlation: { rpc_accepted_at: string | null; turn_started_at: string | null; turn_completed_at: string | null; turn_id: string | null; session_id: string | null } = { + rpc_accepted_at: null, + turn_started_at: null, + turn_completed_at: null, + turn_id: null, + session_id: initialStatus.zcode_session_id ?? null, + }; + const persistCorrelation = (): void => { + try { store.writeAttemptMeta(taskId, attempt, "turn-correlation.json", { ...correlation }); } + catch { /* advisory evidence */ } + }; + persistCorrelationTimer = setInterval(persistCorrelation, 5_000); + persistCorrelationTimer.unref(); + + // B4 runtime-confirmed execution profile (model / reasoning depth). + const modelProfile: ExecutionProfile = { + executor: "zcode", + provider_id: null, + model_id: null, + requested_model: task.model ? `${task.model.provider_id}/${task.model.model_id}` : null, + requested_reasoning_level: task.model?.reasoning_level ?? null, + effective_reasoning_level: null, + effective_reasoning_level_source: "not_reported", + selection_source: null, + effective_at: null, + session_id: initialStatus.zcode_session_id ?? null, + turn_id: null, + }; + + let outcome: ZCodeRunOutcome | null = null; + let failure: TaskFailure | null = null; + let pendingModelOutput = ""; + let lastModelOutputAt = 0; + const flushModelOutput = (): void => { + if (!pendingModelOutput) return; + store.assertWorkerAttempt(taskId, attempt); + for (let offset = 0; offset < pendingModelOutput.length; offset += 2_000) store.appendEvent(taskId, "model_output", pendingModelOutput.slice(offset, offset + 2_000)); + pendingModelOutput = ""; + lastModelOutputAt = Date.now(); + }; + try { + if (options.resolver) { + // Surface configuration problems before spending an adapter run; the + // real adapter resolves internally as well. + await options.resolver.resolve(); + } + const adapter = options.adapter ?? new ZCodeAppServerAdapter({ + host: options.host, + ...(options.resolver ? { resolver: options.resolver } : {}), + onEvent: (event) => { + store.assertWorkerAttempt(taskId, attempt); + if (event.type === "model_output") { + pendingModelOutput += event.summary; + if (pendingModelOutput.length >= 4_000 || Date.now() - lastModelOutputAt >= 500) flushModelOutput(); + return; } - if (typeof event.details?.["reasoning_level"] === "string") { - modelProfile.effective_reasoning_level = event.details["reasoning_level"]; - modelProfile.effective_reasoning_level_source = "runtime"; + flushModelOutput(); + const persistedEvent = store.appendEvent(taskId, event.type, event.summary, event.details); + if (persistedEvent) { + heartbeat.last_event_seq = persistedEvent.seq; + heartbeat.last_event_type = event.type; } - if (typeof event.details?.["model_source"] === "string") modelProfile.selection_source = event.details["model_source"]; - modelProfile.effective_at = now().toISOString(); - } - if (typeof event.details?.["event_seq"] === "number") heartbeat.zcode_event_seq = event.details["event_seq"]; - if (event.type === "app_server_started" && typeof event.details?.["pid"] === "number") { - store.writeStatus(taskId, { zcode_pid: event.details["pid"] }, attempt); - void probe.identityOf(event.details["pid"]).then( - (identity) => { - // Advisory evidence; a store that vanished after the attempt - // ended must not surface as an unhandled rejection. - try { - store.writeExecutorIdentity(taskId, attempt, { runtime: identity as unknown as Record }); - observationRevision += 1; - store.writeWorkerObservation(taskId, attempt, process.pid, { runtime_identity: identity as unknown as Record }, observationRevision, now().toISOString()); - } catch { /* advisory */ } - }, - () => undefined, - ); - } - }, - onInteractionState: (state) => { - activityPhase = state === "waiting" ? "waiting_for_permission" : "executing"; - writePhase(); - }, - onOutcomeCheckpoint: (checkpoint) => { - store.writeAttemptMeta(taskId, attempt, "outcome-checkpoint.json", { - recorded_at: now().toISOString(), - exit_code: checkpoint.exitCode, - signal: checkpoint.signal, - session_id: checkpoint.sessionId, - response: checkpoint.response, - usage: checkpoint.usage, - error_code: checkpoint.errorCode, - report_error: checkpoint.reportError, - agent_report: checkpoint.agentReport, - report_candidate: checkpoint.reportCandidate, - cleanup_error: checkpoint.cleanupError ?? null, - cleanup_verified: checkpoint.cleanupVerified === true, - usage_normalized: normalizeUsage(checkpoint.usage, { source: "zcode_runtime_turn", scope: checkpoint.sessionId ? `session:${checkpoint.sessionId}` : null, observedAt: now().toISOString() }), - model_profile: modelProfile, - }); - }, - resolveInteraction: async (request, signal) => { - store.assertWorkerAttempt(taskId, attempt); - const safeRequest = sanitizeInteractionRequest({ ...request, request_id: `${attempt}:${request.request_id}` }); - // ADR-008: persistence failures get an explicit recoverable reason and - // never masquerade as a generic "reply unavailable" denial. Nothing is - // executed and no approval is left pending. - let record: ReturnType["record"]; - let created = false; - try { - const written = store.writeInteractionRequest(taskId, safeRequest, now().toISOString()); - record = written.record; - created = written.created; - } catch (error) { - if (error instanceof InteractionRequestTooLargeError) { - store.appendEvent(taskId, "interaction_request_oversized", `ZCode requested approval of a tool input too large to deliver: ${error.actualBytes} UTF-8 bytes, over the ${error.limitBytes}-byte limit; the request was declined and nothing was executed`, { - error_code: "interaction_params_over_limit", - request_id: safeRequest.request_id, - method: safeRequest.method, - actual_bytes: error.actualBytes, - limit_bytes: error.limitBytes, - }, now().toISOString()); - return interactionDecline(request.method, - `Bridge cannot deliver this approval request: the tool input is ${error.actualBytes} UTF-8 bytes, over the ${error.limitBytes}-byte limit. Split the input into smaller parts (for example write the file in multiple Write or append steps) and retry. Nothing was executed.`); + const sessionId = event.type === "session_ready" ? event.details?.["session_id"] : undefined; + if (typeof sessionId === "string") { + heartbeat.session_id = sessionId; + correlation.session_id = sessionId; + modelProfile.session_id = sessionId; + store.writeStatus(taskId, { zcode_session_id: sessionId }, attempt); } - store.appendEvent(taskId, "interaction_request_undeliverable", "ZCode requested approval but the Bridge could not persist the request for the calling host; the request was declined and nothing was executed", { - error_code: "interaction_persist_failed", - request_id: safeRequest.request_id, - method: safeRequest.method, - }, now().toISOString()); - return interactionDecline(request.method, - `Bridge could not persist this approval request for the calling host (${safeErrorSummary(error)}); the request was declined and nothing was executed. Split the input into smaller parts and retry.`); - } - activityPhase = request.method === "interaction/requestPermission" ? "waiting_for_permission" : "waiting_for_user"; - writePhase(); - if (created) { - const published = publishInteractionRequest(store, taskId, safeRequest, record!); - if (!published.event) { - store.appendEvent(taskId, "interaction_request_undeliverable", `ZCode requested approval but the Bridge could not publish it to the calling host (${published.declineReason}); the request was declined and nothing was executed`, { - error_code: "interaction_publish_failed", - request_id: safeRequest.request_id, - method: safeRequest.method, - }, now().toISOString()); - const fallback = interactionDecline(request.method, published.declineReason); - store.answerInteractionRequest(taskId, safeRequest.request_id, fallback, now().toISOString()); + if (event.type === "turn_started") { + correlation.rpc_accepted_at = correlation.rpc_accepted_at ?? now().toISOString(); + correlation.turn_started_at = now().toISOString(); + if (typeof event.details?.["turn_id"] === "string") { + heartbeat.turn_id = event.details["turn_id"]; + correlation.turn_id = event.details["turn_id"]; + modelProfile.turn_id = event.details["turn_id"]; + } activityPhase = "executing"; writePhase(); - return fallback; } - } - let answer: Record | null = null; - while (!signal.aborted) { + if (event.type === "turn_completed") { + correlation.turn_completed_at = now().toISOString(); + activityPhase = "finalizing"; + writePhase(); + } + if (event.type === "model_selected") { + // Runtime-confirmed model evidence; requested values never impersonate it. + if (typeof event.details?.["provider_id"] === "string") modelProfile.provider_id = event.details["provider_id"]; + if (typeof event.details?.["model_id"] === "string") modelProfile.model_id = event.details["model_id"]; + if (typeof event.details?.["selected_model"] === "string" && modelProfile.model_id === null) { + modelProfile.model_id = event.details["selected_model"]; + } + if (typeof event.details?.["reasoning_level"] === "string") { + modelProfile.effective_reasoning_level = event.details["reasoning_level"]; + modelProfile.effective_reasoning_level_source = "runtime"; + } + if (typeof event.details?.["model_source"] === "string") modelProfile.selection_source = event.details["model_source"]; + modelProfile.effective_at = now().toISOString(); + } + if (typeof event.details?.["event_seq"] === "number") heartbeat.zcode_event_seq = event.details["event_seq"]; + if (event.type === "app_server_started" && typeof event.details?.["pid"] === "number") { + store.writeStatus(taskId, { zcode_pid: event.details["pid"] }, attempt); + trackIdentityWrite(probe.identityOf(event.details["pid"]).then( + (identity) => { + // Advisory evidence; a store that vanished after the attempt + // ended must not surface as an unhandled rejection, and a late + // resolve after the reap budget must never resurrect it. + if (identityWritesClosed) return; + try { + store.writeExecutorIdentity(taskId, attempt, { runtime: identity as unknown as Record }); + observationRevision += 1; + store.writeWorkerObservation(taskId, attempt, process.pid, { runtime_identity: identity as unknown as Record }, observationRevision, now().toISOString()); + } catch { /* advisory */ } + }, + () => undefined, + )); + } + }, + onInteractionState: (state) => { + activityPhase = state === "waiting" ? "waiting_for_permission" : "executing"; + writePhase(); + }, + onOutcomeCheckpoint: (checkpoint) => { + store.writeAttemptMeta(taskId, attempt, "outcome-checkpoint.json", { + recorded_at: now().toISOString(), + exit_code: checkpoint.exitCode, + signal: checkpoint.signal, + session_id: checkpoint.sessionId, + response: checkpoint.response, + usage: checkpoint.usage, + error_code: checkpoint.errorCode, + report_error: checkpoint.reportError, + agent_report: checkpoint.agentReport, + report_candidate: checkpoint.reportCandidate, + cleanup_error: checkpoint.cleanupError ?? null, + cleanup_verified: checkpoint.cleanupVerified === true, + usage_normalized: normalizeUsage(checkpoint.usage, { source: "zcode_runtime_turn", scope: checkpoint.sessionId ? `session:${checkpoint.sessionId}` : null, observedAt: now().toISOString() }), + model_profile: modelProfile, + }); + }, + resolveInteraction: async (request, signal) => { store.assertWorkerAttempt(taskId, attempt); - let current: ReturnType; + const safeRequest = sanitizeInteractionRequest({ ...request, request_id: `${attempt}:${request.request_id}` }); + // ADR-008: persistence failures get an explicit recoverable reason and + // never masquerade as a generic "reply unavailable" denial. Nothing is + // executed and no approval is left pending. + let record: ReturnType["record"]; + let created = false; try { - current = store.readInteractionRequest(taskId, safeRequest.request_id); + const written = store.writeInteractionRequest(taskId, safeRequest, now().toISOString()); + record = written.record; + created = written.created; } catch (error) { - // The full record became unreadable mid-wait; a corrupted request - // must not hang or count as authorization (ADR-008). - activityPhase = "executing"; - writePhase(); - store.appendEvent(taskId, "interaction_request_undeliverable", "The persisted ZCode interaction request became unreadable while waiting for the calling host; the request was declined and nothing was executed", { - error_code: "interaction_record_unreadable", + if (error instanceof InteractionRequestTooLargeError) { + store.appendEvent(taskId, "interaction_request_oversized", `ZCode requested approval of a tool input too large to deliver: ${error.actualBytes} UTF-8 bytes, over the ${error.limitBytes}-byte limit; the request was declined and nothing was executed`, { + error_code: "interaction_params_over_limit", + request_id: safeRequest.request_id, + method: safeRequest.method, + actual_bytes: error.actualBytes, + limit_bytes: error.limitBytes, + }, now().toISOString()); + return interactionDecline(request.method, + `Bridge cannot deliver this approval request: the tool input is ${error.actualBytes} UTF-8 bytes, over the ${error.limitBytes}-byte limit. Split the input into smaller parts (for example write the file in multiple Write or append steps) and retry. Nothing was executed.`); + } + store.appendEvent(taskId, "interaction_request_undeliverable", "ZCode requested approval but the Bridge could not persist the request for the calling host; the request was declined and nothing was executed", { + error_code: "interaction_persist_failed", request_id: safeRequest.request_id, method: safeRequest.method, }, now().toISOString()); return interactionDecline(request.method, - `Bridge can no longer read the full content of this approval request (${safeErrorSummary(error)}); the request was declined and nothing was executed.`); + `Bridge could not persist this approval request for the calling host (${safeErrorSummary(error)}); the request was declined and nothing was executed. Split the input into smaller parts and retry.`); } - if (!current) { - // The full record vanished mid-wait; waiting can never resolve, so - // decline explicitly instead of polling forever (ADR-008). - activityPhase = "executing"; - writePhase(); - store.appendEvent(taskId, "interaction_request_undeliverable", "The persisted ZCode interaction request disappeared while waiting for the calling host; the request was declined and nothing was executed", { - error_code: "interaction_record_missing", - request_id: safeRequest.request_id, - method: safeRequest.method, - }, now().toISOString()); - return interactionDecline(request.method, - "Bridge can no longer read the full content of this approval request (the persisted record disappeared); the request was declined and nothing was executed."); + activityPhase = request.method === "interaction/requestPermission" ? "waiting_for_permission" : "waiting_for_user"; + writePhase(); + if (created) { + const published = publishInteractionRequest(store, taskId, safeRequest, record!); + if (!published.event) { + store.appendEvent(taskId, "interaction_request_undeliverable", `ZCode requested approval but the Bridge could not publish it to the calling host (${published.declineReason}); the request was declined and nothing was executed`, { + error_code: "interaction_publish_failed", + request_id: safeRequest.request_id, + method: safeRequest.method, + }, now().toISOString()); + const fallback = interactionDecline(request.method, published.declineReason); + store.answerInteractionRequest(taskId, safeRequest.request_id, fallback, now().toISOString()); + activityPhase = "executing"; + writePhase(); + return fallback; + } } - if (current.state === "answered" && current.answer) { - // An answer may only be consumed while the persisted request is - // still the one this worker received from ZCode; an edited record - // never counts as authorization, even with self-consistent - // metadata (ADR-008). - if (current.method !== safeRequest.method || stableJson(current.params) !== stableJson(safeRequest.params)) { + let answer: Record | null = null; + while (!signal.aborted) { + store.assertWorkerAttempt(taskId, attempt); + let current: ReturnType; + try { + current = store.readInteractionRequest(taskId, safeRequest.request_id); + } catch (error) { + // The full record became unreadable mid-wait; a corrupted request + // must not hang or count as authorization (ADR-008). activityPhase = "executing"; writePhase(); - store.appendEvent(taskId, "interaction_request_undeliverable", "The persisted ZCode interaction request no longer matches the request ZCode sent; the request was declined and nothing was executed", { - error_code: "interaction_record_mismatch", + store.appendEvent(taskId, "interaction_request_undeliverable", "The persisted ZCode interaction request became unreadable while waiting for the calling host; the request was declined and nothing was executed", { + error_code: "interaction_record_unreadable", request_id: safeRequest.request_id, method: safeRequest.method, }, now().toISOString()); return interactionDecline(request.method, - "Bridge can no longer read the full content of this approval request (the persisted record no longer matches the request ZCode sent); the request was declined and nothing was executed."); + `Bridge can no longer read the full content of this approval request (${safeErrorSummary(error)}); the request was declined and nothing was executed.`); } - answer = current.answer; - break; + if (!current) { + // The full record vanished mid-wait; waiting can never resolve, so + // decline explicitly instead of polling forever (ADR-008). + activityPhase = "executing"; + writePhase(); + store.appendEvent(taskId, "interaction_request_undeliverable", "The persisted ZCode interaction request disappeared while waiting for the calling host; the request was declined and nothing was executed", { + error_code: "interaction_record_missing", + request_id: safeRequest.request_id, + method: safeRequest.method, + }, now().toISOString()); + return interactionDecline(request.method, + "Bridge can no longer read the full content of this approval request (the persisted record disappeared); the request was declined and nothing was executed."); + } + if (current.state === "answered" && current.answer) { + // An answer may only be consumed while the persisted request is + // still the one this worker received from ZCode; an edited record + // never counts as authorization, even with self-consistent + // metadata (ADR-008). + if (current.method !== safeRequest.method || stableJson(current.params) !== stableJson(safeRequest.params)) { + activityPhase = "executing"; + writePhase(); + store.appendEvent(taskId, "interaction_request_undeliverable", "The persisted ZCode interaction request no longer matches the request ZCode sent; the request was declined and nothing was executed", { + error_code: "interaction_record_mismatch", + request_id: safeRequest.request_id, + method: safeRequest.method, + }, now().toISOString()); + return interactionDecline(request.method, + "Bridge can no longer read the full content of this approval request (the persisted record no longer matches the request ZCode sent); the request was declined and nothing was executed."); + } + answer = current.answer; + break; + } + await sleep(250); } - await sleep(250); - } - activityPhase = "executing"; - writePhase(); - if (answer) return answer; - return interactionDecline(request.method, "The task attempt ended before the calling host replied; nothing was executed"); - }, - }); - const workspaceRef: WorkspaceRef = store.readWorkspaceRef(taskId) ?? { - requestedPath: task.workspace, - canonicalPath: task.worktree_path ?? task.workspace, - mode: task.worktree_path ? "worktree" : "direct", - ...(task.worktree_path ? { sourcePath: task.workspace } : {}), + activityPhase = "executing"; + writePhase(); + if (answer) return answer; + return interactionDecline(request.method, "The task attempt ended before the calling host replied; nothing was executed"); + }, + }); + const workspaceRef: WorkspaceRef = store.readWorkspaceRef(taskId) ?? { + requestedPath: task.workspace, + canonicalPath: task.worktree_path ?? task.workspace, + mode: task.worktree_path ? "worktree" : "direct", + ...(task.worktree_path ? { sourcePath: task.workspace } : {}), + }; + const handle = continueSpec + ? await adapter.continueTask({ + task, + workspace: workspaceRef, + attempt, + feedback: continueSpec.feedback, + additionalRequirements: [...(continueSpec.additional_requirements ?? [])], + previousSessionId: continueSpec.previous_session_id, + previousResult, + }) + : await adapter.startTask({ task, workspace: workspaceRef, attempt }); + // The manager persists cancellation intent before its bounded fallback + // kill. Observe it while the runtime is active so the adapter can abort + // pending interactions and classify the exit as cancellation first. + let cancelTimer: ReturnType | undefined; + const cancellation = new Promise((_resolve, reject) => { + cancelTimer = setInterval(() => { + try { + const current = store.readStatus(taskId); + if (current.attempt !== attempt || !current.cancel_requested) return; + clearInterval(cancelTimer); + void adapter.cancelTask(handle).catch(reject); + } catch (error) { + clearInterval(cancelTimer); + reject(error); + } + }, 50); + }); + try { outcome = await Promise.race([adapter.getResult(handle), cancellation]); } + finally { clearInterval(cancelTimer); } + if (outcome.cleanupError) failure = { code: "cleanup_failed", message: `ZCode returned a turn result, but process cleanup could not be verified: ${outcome.cleanupError}` }; + flushModelOutput(); + } catch (error) { + flushModelOutput(); + failure = { + code: error instanceof BridgeError ? error.code : "worker_error", + message: error instanceof Error ? error.message : String(error), + }; + } + await reapIdentityWrites(); // land identity evidence before terminal artifacts; bounded, then closed + activityPhase = "finalizing"; + writePhase(); + + const finishedAt = now().toISOString(); + store.assertWorkerAttempt(taskId, attempt); + const stdoutLog = store.appendLog(taskId, "stdout", outcome?.stdout ? `${outcome.stdout}\n` : ""); + const stderrLog = store.appendLog( + taskId, + "stderr", + outcome?.stderr + ? `${outcome.stderr}\n` + : failure + ? `${failure.code}: ${failure.message}\n` + : "", + ); + + // The event-driven profile is authoritative; when the runtime reported the + // selection through the outcome only (no model_selected event reached us), + // adopt that evidence instead of leaving nulls (B4-12). + if (modelProfile.model_id === null && outcome?.modelProfile?.model_id) { + Object.assign(modelProfile, outcome.modelProfile); + } + // Turn boundaries likewise fall back to the outcome's correlation record. + correlation.rpc_accepted_at = correlation.rpc_accepted_at ?? outcome?.phaseTimestamps?.rpc_accepted_at ?? null; + correlation.turn_started_at = correlation.turn_started_at ?? outcome?.phaseTimestamps?.turn_started_at ?? null; + correlation.turn_completed_at = correlation.turn_completed_at ?? outcome?.phaseTimestamps?.turn_completed_at ?? null; + if (modelProfile.turn_id === null && correlation.turn_id) modelProfile.turn_id = correlation.turn_id; + // B4 timing with explicit boundaries; a regressed/missing clock yields null + // notes instead of negative or fabricated values. + const timingNotes: string[] = []; + const timing = { + queued_ms: phaseDuration(initialStatus.created_at, startedAt, timingNotes), + execution_ms: phaseDuration(startedAt, finishedAt, timingNotes), + turn_ms: phaseDuration(correlation.turn_started_at, correlation.turn_completed_at, timingNotes), + finalize_ms: phaseDuration(correlation.turn_completed_at, finishedAt, timingNotes), + wall_ms: phaseDuration(startedAt, finishedAt, timingNotes), + derived: false, + notes: timingNotes, }; - const handle = continueSpec - ? await adapter.continueTask({ - task, - workspace: workspaceRef, - attempt, - feedback: continueSpec.feedback, - additionalRequirements: [...(continueSpec.additional_requirements ?? [])], - previousSessionId: continueSpec.previous_session_id, - previousResult, - }) - : await adapter.startTask({ task, workspace: workspaceRef, attempt }); - // The manager persists cancellation intent before its bounded fallback - // kill. Observe it while the runtime is active so the adapter can abort - // pending interactions and classify the exit as cancellation first. - let cancelTimer: ReturnType | undefined; - const cancellation = new Promise((_resolve, reject) => { - cancelTimer = setInterval(() => { - try { - const current = store.readStatus(taskId); - if (current.attempt !== attempt || !current.cancel_requested) return; - clearInterval(cancelTimer); - void adapter.cancelTask(handle).catch(reject); - } catch (error) { - clearInterval(cancelTimer); - reject(error); - } - }, 50); + const usage: NormalizedUsage | null = normalizeUsage(outcome?.usage ?? null, { + source: "zcode_runtime_turn", + scope: outcome?.sessionId ? `session:${outcome.sessionId}` : null, + observedAt: finishedAt, + }); + persistCorrelation(); + + store.writeAttemptMeta(taskId, attempt, "outcome.json", { + started_at: startedAt, + finished_at: finishedAt, + failure, + outcome: outcome + ? { + exitCode: outcome.exitCode, + signal: outcome.signal, + sessionId: outcome.sessionId, + response: outcome.response, + usage: outcome.usage, + timedOut: outcome.timedOut, + cancelled: outcome.cancelled, + attempts: outcome.attempts, + errorCode: outcome.errorCode, + reportError: outcome.reportError, + stdoutTruncated: outcome.stdoutTruncated, + stderrTruncated: outcome.stderrTruncated, + agentReport: outcome.agentReport, + reportCandidate: outcome.reportCandidate, + cleanupError: outcome.cleanupError ?? null, + cleanupVerified: outcome.cleanupVerified === true, + } + : null, + usage_normalized: usage, + model_profile: modelProfile, + timing, + correlation: { ...correlation }, + logs: { stdout_truncated: stdoutLog.truncated, stderr_truncated: stderrLog.truncated }, }); - try { outcome = await Promise.race([adapter.getResult(handle), cancellation]); } - finally { clearInterval(cancelTimer); } - if (outcome.cleanupError) failure = { code: "cleanup_failed", message: `ZCode returned a turn result, but process cleanup could not be verified: ${outcome.cleanupError}` }; - flushModelOutput(); - } catch (error) { - flushModelOutput(); - failure = { - code: error instanceof BridgeError ? error.code : "worker_error", - message: error instanceof Error ? error.message : String(error), - }; - } - clearInterval(heartbeatTimer); - clearInterval(persistCorrelationTimer); - activityPhase = "finalizing"; - writePhase(); - const finishedAt = now().toISOString(); - store.assertWorkerAttempt(taskId, attempt); - const stdoutLog = store.appendLog(taskId, "stdout", outcome?.stdout ? `${outcome.stdout}\n` : ""); - const stderrLog = store.appendLog( - taskId, - "stderr", - outcome?.stderr - ? `${outcome.stderr}\n` - : failure - ? `${failure.code}: ${failure.message}\n` - : "", - ); - - // The event-driven profile is authoritative; when the runtime reported the - // selection through the outcome only (no model_selected event reached us), - // adopt that evidence instead of leaving nulls (B4-12). - if (modelProfile.model_id === null && outcome?.modelProfile?.model_id) { - Object.assign(modelProfile, outcome.modelProfile); + const result = buildTaskResult({ + task, + attempt, + startedAt, + finishedAt, + outcome, + failure, + cancelled: outcome?.cancelled === true || failure?.code === "cancelled", + sessionId: store.readStatus(taskId).zcode_session_id ?? continueSpec?.previous_session_id, + usage, + model: modelProfile, + timing, + }); + store.appendEvent(taskId, "task_finished", `Task reached terminal status: ${result.status}`, { + status: result.status, + needs_master_decision: result.needs_master_decision, + }, finishedAt); + store.commitWorkerResult(taskId, attempt, result, { + status: result.status, + finished_at: finishedAt, + exit_code: result.exit_code, + zcode_session_id: result.session_id, + error_code: result.error_code ?? null, + error: result.status === "failed" ? result.summary : null, + worker_pid: null, + cleanup_unverified: failure?.code === "cleanup_failed", + zcode_pid: failure?.code === "cleanup_failed" ? store.readStatus(taskId).zcode_pid : null, + }); + return { status: result.status, result }; + } finally { + if (heartbeatTimer) clearInterval(heartbeatTimer); + if (persistCorrelationTimer) clearInterval(persistCorrelationTimer); + await reapIdentityWrites(); } - // Turn boundaries likewise fall back to the outcome's correlation record. - correlation.rpc_accepted_at = correlation.rpc_accepted_at ?? outcome?.phaseTimestamps?.rpc_accepted_at ?? null; - correlation.turn_started_at = correlation.turn_started_at ?? outcome?.phaseTimestamps?.turn_started_at ?? null; - correlation.turn_completed_at = correlation.turn_completed_at ?? outcome?.phaseTimestamps?.turn_completed_at ?? null; - if (modelProfile.turn_id === null && correlation.turn_id) modelProfile.turn_id = correlation.turn_id; - // B4 timing with explicit boundaries; a regressed/missing clock yields null - // notes instead of negative or fabricated values. - const timingNotes: string[] = []; - const timing = { - queued_ms: phaseDuration(initialStatus.created_at, startedAt, timingNotes), - execution_ms: phaseDuration(startedAt, finishedAt, timingNotes), - turn_ms: phaseDuration(correlation.turn_started_at, correlation.turn_completed_at, timingNotes), - finalize_ms: phaseDuration(correlation.turn_completed_at, finishedAt, timingNotes), - wall_ms: phaseDuration(startedAt, finishedAt, timingNotes), - derived: false, - notes: timingNotes, - }; - const usage: NormalizedUsage | null = normalizeUsage(outcome?.usage ?? null, { - source: "zcode_runtime_turn", - scope: outcome?.sessionId ? `session:${outcome.sessionId}` : null, - observedAt: finishedAt, - }); - persistCorrelation(); - - store.writeAttemptMeta(taskId, attempt, "outcome.json", { - started_at: startedAt, - finished_at: finishedAt, - failure, - outcome: outcome - ? { - exitCode: outcome.exitCode, - signal: outcome.signal, - sessionId: outcome.sessionId, - response: outcome.response, - usage: outcome.usage, - timedOut: outcome.timedOut, - cancelled: outcome.cancelled, - attempts: outcome.attempts, - errorCode: outcome.errorCode, - reportError: outcome.reportError, - stdoutTruncated: outcome.stdoutTruncated, - stderrTruncated: outcome.stderrTruncated, - agentReport: outcome.agentReport, - reportCandidate: outcome.reportCandidate, - cleanupError: outcome.cleanupError ?? null, - cleanupVerified: outcome.cleanupVerified === true, - } - : null, - usage_normalized: usage, - model_profile: modelProfile, - timing, - correlation: { ...correlation }, - logs: { stdout_truncated: stdoutLog.truncated, stderr_truncated: stderrLog.truncated }, - }); - - const result = buildTaskResult({ - task, - attempt, - startedAt, - finishedAt, - outcome, - failure, - cancelled: outcome?.cancelled === true || failure?.code === "cancelled", - sessionId: store.readStatus(taskId).zcode_session_id ?? continueSpec?.previous_session_id, - usage, - model: modelProfile, - timing, - }); - store.appendEvent(taskId, "task_finished", `Task reached terminal status: ${result.status}`, { - status: result.status, - needs_master_decision: result.needs_master_decision, - }, finishedAt); - store.commitWorkerResult(taskId, attempt, result, { - status: result.status, - finished_at: finishedAt, - exit_code: result.exit_code, - zcode_session_id: result.session_id, - error_code: result.error_code ?? null, - error: result.status === "failed" ? result.summary : null, - worker_pid: null, - cleanup_unverified: failure?.code === "cleanup_failed", - zcode_pid: failure?.code === "cleanup_failed" ? store.readStatus(taskId).zcode_pid : null, - }); - return { status: result.status, result }; } function interactionSummary(request: ZCodeInteractionRequest): string { diff --git a/test/audit-remediation.test.ts b/test/audit-remediation.test.ts index b22b61a..aedbf80 100644 --- a/test/audit-remediation.test.ts +++ b/test/audit-remediation.test.ts @@ -28,7 +28,18 @@ test("cooperative cancellation waits for the verified worker result without a se await fx.manager.createTask(fx.makeTask()); const pid = fx.spawned[0]!.pid; const finish = async () => { - await delay(40); + // Coordinate on the manager's persisted intent instead of a fixed + // delay. What was observed in the Linux24 full-suite failure is only + // the stale/terminal rejection at 104ms TAP duration; a delayed + // delivery is one possible mechanism (demonstrated by the 600ms + // diagnostic probe), not the proven original cause. Coordinating on + // the cancel_requested record keeps the worker commit causally after + // the manager's intent within this test's bounded timeout; the 5ms + // poll is evented coordination, not a timing guarantee about the + // manager's 500ms real-time window. + const deadline = Date.now() + 5_000; + while (fx.store.readStatus("task_1").cancel_requested !== true && Date.now() < deadline) await delay(5); + assert.equal(fx.store.readStatus("task_1").cancel_requested, true, "the manager must record its cancellation intent before the worker result can be waited for"); const result = buildTaskResult({ task: fx.makeTask(), attempt: 1, startedAt: null, finishedAt: new Date().toISOString(), outcome: null, cancelled: true, sessionId: "cooperative-session" }); fx.store.commitWorkerResult("task_1", 1, result, { status: "cancelled", worker_pid: null, zcode_pid: null }); fx.pidsAlive.delete(pid); @@ -41,6 +52,32 @@ test("cooperative cancellation waits for the verified worker result without a se } finally { await fx.cleanup(); } }); +test("a worker result arriving after the manager's verified cancellation is stale and cannot replace it", async () => { + // Boundary pin for the same fixture family. A fixed timer cannot place the + // worker commit after the manager's terminal result (Codex counterexample: + // scheduling 600ms before cancelTask still landed inside the 500ms window + // when cancellation started 188ms later). Instead the manager's verified + // cancellation is awaited to completion first, and only then is the worker- + // side commit attempted — deterministically stale by contract: rejected, + // exactly one termination, no second one, and the manager's result stands. + const fx = await makeManagerFixture(); + try { + await fx.manager.createTask(fx.makeTask()); + const pid = fx.spawned[0]!.pid; + const workerPidKeptAlive = () => fx.pidsAlive.has(pid); + assert.equal(workerPidKeptAlive(), true, "the fake worker starts alive so the manager must go through its cooperative-then-terminate path"); + assert.equal((await fx.manager.cancelTask("task_1")).status, "cancelled"); + assert.deepEqual(fx.terminateCalls, [pid], "the worker outlived the cooperative window fixture-side, so exactly one verified termination happened"); + assert.equal(fx.pidsAlive.has(pid), false, "the terminated worker is no longer considered alive"); + // The worker-side commit happens strictly AFTER the manager's terminal + // result; it must be rejected as stale and never replace that result. + const result = buildTaskResult({ task: fx.makeTask(), attempt: 1, startedAt: null, finishedAt: new Date().toISOString(), outcome: null, cancelled: true, sessionId: "late-session" }); + assert.throws(() => fx.store.commitWorkerResult("task_1", 1, result, { status: "cancelled", worker_pid: null, zcode_pid: null }), /stale or terminal worker attempt rejected/); + assert.equal((await fx.manager.getResult("task_1")).session_id, null, "the manager's verified cancellation result stands over the late worker commit"); + assert.deepEqual(fx.terminateCalls, [pid], "no second termination was issued"); + } finally { await fx.cleanup(); } +}); + test("a taskkill exit race requires a verified terminal result and dead recorded processes", async () => { for (const cleanupFailed of [false, true]) { const fx = await makeManagerFixture(); @@ -61,10 +98,12 @@ test("a taskkill exit race requires a verified terminal result and dead recorded })(); throw new Error("taskkill raced process exit"); } }); - if (cleanupFailed) { - await assert.rejects(manager.cancelTask("task_1"), /termination.*could not be verified/); - assert.equal(fx.store.readStatus("task_1").cleanup_unverified, true); - } else assert.equal((await manager.cancelTask("task_1")).status, "cancelled"); + // ADR-009: a termination failure propagates regardless of the worker + // result arriving meanwhile — root exit never replaces verified tree + // cleanup, so both variants keep occupancy (the RED-C swallow shape). + await assert.rejects(manager.cancelTask("task_1"), /could not be verified/); + assert.equal(fx.store.readStatus("task_1").cleanup_unverified, true); + assert.equal(fx.store.readStatus("task_1").status, "running"); await finishing; } finally { manager?.dispose(); await fx.cleanup(); } } @@ -117,6 +156,9 @@ interface CleanupRaceFixture { runtimePid: number; workspaceDir: string; setTerminate: (impl: TerminateProcessTree) => void; + /** Recorded-tree claimant scans requested by the manager (ADR-009). */ + scanCalls: Array>; + setScanResult: (result: { clean: boolean; livePids: readonly number[] } | Error) => void; dispose: () => Promise; } @@ -130,11 +172,18 @@ async function makeCleanupRaceFixture(options: { persistIdentity?: boolean } = { const probe = new ScriptedProbe(); const pidsAlive = new Set(); const terminateCalls: number[] = []; + const scanCalls: Array> = []; const workerPid = 22696; const runtimePid = 26188; + let scanResult: { clean: boolean; livePids: readonly number[] } | Error = { clean: true, livePids: [] }; let terminateImpl: TerminateProcessTree = async () => { throw new Error("the test must install a terminate script before cancelling"); }; + const verifyTreeClean = async (targets: readonly { pid: number; fingerprint: string | null }[]) => { + scanCalls.push([...targets]); + if (scanResult instanceof Error) throw scanResult; + return scanResult; + }; const manager = new BridgeTaskManager({ store, workspaceProvider: new DirectWorkspaceProvider(), @@ -147,6 +196,7 @@ async function makeCleanupRaceFixture(options: { persistIdentity?: boolean } = { terminateCalls.push(pid); return terminateImpl(pid, options); }, + verifyTreeClean, probe: probe as unknown as ProcessProbe, pollIntervalMs: 0, }); @@ -176,6 +226,8 @@ async function makeCleanupRaceFixture(options: { persistIdentity?: boolean } = { runtimePid, workspaceDir, setTerminate: (impl) => { terminateImpl = impl; }, + scanCalls, + setScanResult: (result) => { scanResult = result; }, dispose: async () => { manager.dispose(); await removeTempDir(dataRoot); @@ -184,11 +236,12 @@ async function makeCleanupRaceFixture(options: { persistIdentity?: boolean } = { }; } -test("a natural exit between the cancel probe and taskkill resolves cleanup as an idempotent success", async () => { +test("a termination failure stays occupied; a later cancel releases only after probes AND the recorded-tree scan", async () => { const fx = await makeCleanupRaceFixture(); try { - // Both recorded processes exit naturally while taskkill is in flight; - // taskkill then reports the runtime pid as not found and exits non-zero. + // Both recorded processes exit naturally while the terminator is in + // flight and it reports failure. ADR-009: the failure is never cleared + // by executor-only exit evidence — the first cancel keeps occupancy. fx.setTerminate(async (pid) => { fx.probe.set(pid, "exited", "pid_absent"); fx.probe.set(fx.workerPid, "exited", "pid_absent"); @@ -196,17 +249,26 @@ test("a natural exit between the cancel probe and taskkill resolves cleanup as a fx.pidsAlive.delete(fx.workerPid); throw new Error(`taskkill: 没有找到进程 ${String(pid)}。`); }); + await assert.rejects(fx.manager.cancelTask("task_1"), /could not be verified/); + let persisted = fx.store.readStatus("task_1"); + assert.equal(persisted.status, "running", "the tree failure keeps the task occupied"); + assert.equal(persisted.cleanup_unverified, true); + // The retry sees both executors conclusively exited; nothing is + // signalled again, and the release is gated by the read-only claimant + // scan over the recorded pids. const status = await fx.manager.cancelTask("task_1"); assert.equal(status.status, "cancelled"); - const persisted = fx.store.readStatus("task_1"); - assert.equal(persisted.cleanup_unverified ?? false, false, "the probe-verified race clears the flag"); + persisted = fx.store.readStatus("task_1"); + assert.equal(persisted.cleanup_unverified ?? false, false, "the scan-gated release clears the flag"); assert.equal(persisted.zcode_pid, null); - assert.deepEqual(fx.terminateCalls, [fx.runtimePid], "the exited worker pid must never be signalled"); + assert.deepEqual(fx.terminateCalls, [fx.runtimePid], "only the alive-at-probe runtime was ever signalled; the exited retry probes and scans instead"); + assert.equal(fx.scanCalls.length, 1, "the retry release consulted the recorded-tree scan once"); + assert.ok(fx.scanCalls[0]!.some((t) => t.pid === fx.runtimePid), "the recorded runtime pid is scanned"); assert.equal((await fx.manager.getResult("task_1")).status, "cancelled"); } finally { await fx.dispose(); } }); -test("legacy cleanup without executor identities uses the probe to resolve an exited-PID race", async () => { +test("legacy cleanup without executor identities releases through the same scan gate", async () => { const fx = await makeCleanupRaceFixture({ persistIdentity: false }); try { assert.equal(fx.store.readExecutorIdentity("task_1", 1), null); @@ -217,10 +279,36 @@ test("legacy cleanup without executor identities uses the probe to resolve an ex fx.pidsAlive.delete(fx.workerPid); throw new Error(`taskkill: 没有找到进程 ${String(pid)}。`); }); + await assert.rejects(fx.manager.cancelTask("task_1"), /could not be verified/); + assert.equal(fx.store.readStatus("task_1").cleanup_unverified, true); const status = await fx.manager.cancelTask("task_1"); assert.equal(status.status, "cancelled"); assert.equal(fx.store.readStatus("task_1").cleanup_unverified ?? false, false); assert.equal(fx.store.readStatus("task_1").zcode_pid, null); + assert.equal(fx.scanCalls.length, 1); + } finally { await fx.dispose(); } +}); + +test("a scan that still observes live claimants keeps the retry occupied", async () => { + const fx = await makeCleanupRaceFixture(); + try { + fx.setTerminate(async (pid) => { + fx.probe.set(pid, "exited", "pid_absent"); + fx.probe.set(fx.workerPid, "exited", "pid_absent"); + fx.pidsAlive.delete(pid); + fx.pidsAlive.delete(fx.workerPid); + throw new Error("terminator failed"); + }); + await assert.rejects(fx.manager.cancelTask("task_1"), /could not be verified/); + fx.setTerminate(async () => { throw new Error("unreachable: nothing is alive to terminate"); }); + fx.setScanResult({ clean: false, livePids: [31_337] }); + await assert.rejects(fx.manager.cancelTask("task_1"), /still alive/); + const persisted = fx.store.readStatus("task_1"); + assert.equal(persisted.status, "running"); + assert.equal(persisted.cleanup_unverified, true); + fx.setScanResult({ clean: true, livePids: [] }); + const status = await fx.manager.cancelTask("task_1"); + assert.equal(status.status, "cancelled"); } finally { await fx.dispose(); } }); @@ -285,7 +373,9 @@ test("a cleanup race that proves only the runtime exited keeps the occupation wh fx.probe.set(fx.workerPid, "alive", "pid_and_fingerprint_match"); throw new Error(`taskkill: 没有找到进程 ${String(pid)}。`); }); - await assert.rejects(fx.manager.cancelTask("task_1"), /worker alive/); + // ADR-009: the terminate failure itself propagates; the worker-alive + // evidence only keeps occupancy (it never clears a known failure). + await assert.rejects(fx.manager.cancelTask("task_1"), /could not be verified/); const persisted = fx.store.readStatus("task_1"); assert.equal(persisted.status, "running"); assert.equal(persisted.cleanup_unverified, true); diff --git a/test/helpers.ts b/test/helpers.ts index 94f94b9..d989cbe 100644 --- a/test/helpers.ts +++ b/test/helpers.ts @@ -1,9 +1,10 @@ // Shared test fixtures: fake spawn infrastructure, disposable directories, and // sample payloads. Unit tests never invoke the real ZCode CLI or any model. import { EventEmitter } from "node:events"; -import { mkdtemp, rm } from "node:fs/promises"; +import { mkdtemp, rm, writeFile } from "node:fs/promises"; import { tmpdir } from "node:os"; import path from "node:path"; +import { isProcessRunning } from "../src/adapters/process-spawn.js"; import type { AgentReport, TaskPackage, @@ -34,6 +35,10 @@ export class FakeChild implements SpawnedProcess { readonly stdout: FakeStream = new FakeStream(); readonly stderr: FakeStream = new FakeStream(); readonly pid: number; + /** Mirrors a real ChildProcess: null until the exit event is delivered + * (which happens before stdio close). ADR-009 continuity reads these. */ + exitCode: number | null = null; + signalCode: string | null = null; killCallCount = 0; readonly #events = new EventEmitter(); @@ -59,9 +64,22 @@ export class FakeChild implements SpawnedProcess { this.stderr.emitData(text); } - /** Deferred so the adapter's listeners (attached synchronously) are ready. */ + /** Marks the child exited: exit state first, then the event — mirroring + * Node's exit-before-close ordering (ADR-009). */ + emitExit(code: number | null, signal: string | null): void { + if (code !== null) this.exitCode = code; + if (signal !== null) this.signalCode = signal; + setImmediate(() => this.#events.emit("exit", code, signal)); + } + + /** Deferred so the adapter's listeners (attached synchronously) are ready. + * The exit state is set before the close event fires, like a real child. */ emitClose(code: number | null, signal: string | null): void { - setImmediate(() => this.#events.emit("close", code, signal)); + setImmediate(() => { + if (code !== null) this.exitCode = code; + if (signal !== null) this.signalCode = signal; + this.#events.emit("close", code, signal); + }); } emitError(error: Error): void { @@ -172,8 +190,94 @@ export async function makeTempDir(label: string): Promise { return mkdtemp(path.join(tmpdir(), `zcode-bridge-test-${label}-`)); } +/** + * Windows teardown tolerance (2026-10-17): recursive removals of trees that + * hosted child processes and lock directories can fail transiently with + * ENOTEMPTY/EPERM/EBUSY while directory entries propagate. These are exactly + * the codes Node documents as retryable for recursive rm; the retry budget is + * bounded and exhaustion still rejects, so genuine cleanup failures stay + * visible instead of being masked. + */ +export const TEMP_TREE_REMOVE_OPTIONS = { maxRetries: 5, retryDelay: 100 } as const; + export async function removeTempDir(dir: string): Promise { - await rm(dir, { recursive: true, force: true }); + await rm(dir, { recursive: true, force: true, ...TEMP_TREE_REMOVE_OPTIONS }); +} + +export interface OwnedProcessFixtureCleanup { + /** Pids of ONLY the processes this test spawned itself — never broad + * PID-tree signalling, and never a pid whose provenance is not this test. */ + readonly pids: readonly number[]; + /** File the fixture children watch for their cooperative release. */ + readonly releasePath: string; + /** Fixture root removed only after every owned child confirmed exit. */ + readonly dir: string; + /** Whether the test's primary path already failed; cleanup problems then + * never mask the original failure but are still reported loudly. */ + readonly primaryFailed: boolean; + readonly timeoutMs?: number; + /** When a descendant's identity could not be established (for example a + * failed pid receipt), the tree must not be deleted around a possibly live + * unknown child: release the known pids but preserve everything. */ + readonly preserveTree?: boolean; +} + +/** + * Fixture-only cooperative teardown for REAL child processes (2026-10-17 + * matrix2 review): a probe that has written its receipt is not necessarily + * exited — a detached worker's cwd is the fixture root, so removing the tree + * before the exit is confirmed failed with EBUSY in the wild. The fixture + * scripts watch their release file; this helper writes it and CONFIRMS each + * owned pid reports exited within a bounded window. Mechanical gates: any + * release-signaling or exit-confirmation failure — or an explicit + * preserveTree request for unknown descendants — keeps the fixture tree for + * diagnosis instead of deleting around live or unaccounted children; a blind + * sleep or a plain rmdir retry cannot substitute for the exit confirmation. + * Errors stay visible unless the primary path already failed, in which case + * they are reported without masking it. Callers whose primary action already + * ran a successful gated cleanup must not call this again — track that + * success at the call site instead of relying on this helper to detect it. + */ +export async function cleanupOwnedProcessFixture(cleanup: OwnedProcessFixtureCleanup): Promise { + const issues: string[] = []; + let treeRemovalAllowed = !cleanup.preserveTree; + try { + await writeFile(cleanup.releasePath, String(Date.now()), "utf8"); + } catch (error) { + issues.push(`writing the release signal failed: ${error instanceof Error ? error.message : String(error)}`); + treeRemovalAllowed = false; + } + const timeoutMs = cleanup.timeoutMs ?? 15_000; + for (const pid of cleanup.pids) { + const deadline = Date.now() + timeoutMs; + while (isProcessRunning(pid)) { + if (Date.now() >= deadline) { + issues.push(`owned test process pid ${String(pid)} did not report exit within ${String(timeoutMs)} ms of its release signal`); + treeRemovalAllowed = false; + break; + } + await new Promise((resolve) => setTimeout(resolve, 25)); + } + } + if (cleanup.preserveTree) { + issues.push(`tree removal skipped by request: a descendant's identity is unknown; fixture tree preserved at ${cleanup.dir} for diagnosis`); + } + if (treeRemovalAllowed) { + try { + await removeTempDir(cleanup.dir); + } catch (error) { + issues.push(`removing the fixture tree failed: ${error instanceof Error ? error.message : String(error)}`); + treeRemovalAllowed = false; + } + } + if (issues.length > 0) { + const preserved = treeRemovalAllowed ? "" : `; fixture tree preserved at ${cleanup.dir} for diagnosis`; + const message = `fixture teardown failed: ${issues.join("; ")}${preserved}`; + if (!cleanup.primaryFailed) throw new Error(message); + // The primary failure is already propagating; surface this loudly without + // replacing it. + console.error(message); + } } export const TRANSIENT_STDERR = "Error: Bundled 与 Active ZCode Built-in Release 均不可用"; diff --git a/test/late-identity-lifecycle.test.ts b/test/late-identity-lifecycle.test.ts new file mode 100644 index 0000000..63dec87 --- /dev/null +++ b/test/late-identity-lifecycle.test.ts @@ -0,0 +1,175 @@ +// Late-identity lifecycle regressions (2026-10-17 reviews): the worker's +// advisory identity probes must never outlive the attempt. The attempt-1 +// review proved an injected 250ms probe resurrected a deleted fixture tree; +// the attempt-2 review then showed these tests only used timers and asserted +// before the promises settled (a mutant dropping both callback fences still +// passed and left three executor-identity.json files behind). Every deferred +// probe here is settled EXPLICITLY after the fixture is removed, and the +// assertions wait until the worker's callbacks have provably run +// (whenSelfSettled/whenRuntimeSettled resolve after the worker's own .then, +// which attached first, plus one macrotask). Removing either fence makes the +// corresponding test fail with a resurrected tree. +import assert from "node:assert/strict"; +import test from "node:test"; +import process from "node:process"; +import { existsSync, rmSync } from "node:fs"; +import path from "node:path"; +import type { ProcessIdentity, ProcessProbe, ProbeRequest, ProbeVerdict } from "../src/runtime/process-probe.js"; +import { runWorkerTask } from "../src/worker/run-task.js"; +import { TaskStore } from "../src/store/task-store.js"; +import { FakeAdapter, makeManagerFixture, type ManagerFixture } from "./manager-helpers.js"; +import { runtimeFixture } from "./runtime-fixture.js"; +import { TEMP_TREE_REMOVE_OPTIONS } from "./helpers.js"; + +const removeTree = (dir: string): void => { + rmSync(dir, { recursive: true, force: true, ...TEMP_TREE_REMOVE_OPTIONS }); +}; + +const stubIdentity = (pid: number): ProcessIdentity => ({ + pid, + fingerprint: `fingerprint-${pid}`, + fingerprint_precision: "exact", + identity_version: 1, + platform: process.platform, + captured_at: new Date().toISOString(), +}); + +interface DeferredIdentityProbe extends ProcessProbe { + /** Resolves (or rejects, with an error) the pending selfIdentity probe. */ + settleSelf: (error?: Error) => void; + /** Resolves (or rejects, with an error) the pending runtime identity probe. */ + settleRuntime: (error?: Error) => void; + /** Resolves after the worker's selfIdentity callback has run (or errored). */ + whenSelfSettled: () => Promise; + /** Resolves after the worker's runtime-identity callback has run (or errored). */ + whenRuntimeSettled: () => Promise; +} + +function deferredProbe(options: { autoSelfMs?: number } = {}): DeferredIdentityProbe { + let settleSelf: (error?: Error) => void = () => undefined; + let settleRuntime: (error?: Error) => void = () => undefined; + const self = new Promise((resolve, reject) => { + settleSelf = (error?: Error) => { if (error) reject(error); else resolve(stubIdentity(process.pid)); }; + if (options.autoSelfMs !== undefined) setTimeout(() => settleSelf(), options.autoSelfMs); + }); + const runtime = new Promise((resolve, reject) => { + settleRuntime = (error?: Error) => { if (error) reject(error); else resolve(stubIdentity(41_000)); }; + }); + // The worker attaches its own .then to `self`/`runtime` during the run, + // BEFORE these derived promises attach, so awaiting one of these plus a + // macrotask guarantees the worker's callback has already executed. + const drain = async (promise: Promise): Promise => { + await promise.then(() => undefined, () => undefined); + await new Promise((resolve) => setImmediate(resolve)); + }; + return { + platform: process.platform, + selfIdentity: () => self, + identityOf: (_pid: number) => runtime, + probe: (_requests: readonly ProbeRequest[]): Promise => Promise.resolve([]), + settleSelf, + settleRuntime, + whenSelfSettled: () => drain(self), + whenRuntimeSettled: () => drain(runtime), + }; +} + +async function workerFixture(): Promise<{ fixture: ManagerFixture; store: TaskStore }> { + const fixture = await makeManagerFixture(); + const store = new TaskStore(fixture.dataRoot); + store.createTask(fixture.makeTask(), "2026-09-27T00:00:00.000Z"); + return { fixture, store }; +} + +const tasksRootOf = (dataRoot: string): string => path.join(dataRoot, ".tasks"); + +test("identity resolving within the reap budget is persisted before the worker returns", { timeout: 15_000 }, async () => { + const { fixture, store } = await workerFixture(); + const probe = deferredProbe({ autoSelfMs: 250 }); + try { + const run = await runWorkerTask({ dataRoot: fixture.dataRoot, taskId: "task_1", attempt: 1, adapter: new FakeAdapter(), probe }); + assert.equal(run.status, "completed"); + await probe.whenSelfSettled(); + const identity = store.readExecutorIdentity("task_1", 1); + assert.equal(identity?.worker?.fingerprint, `fingerprint-${process.pid}`, "the identity write must land before the terminal result"); + } finally { + removeTree(fixture.dataRoot); + await fixture.cleanup(); + } +}); + +test("identity settled explicitly after removal is dropped and never resurrects the store (kills the self-identity fence mutant)", { timeout: 20_000 }, async () => { + const { fixture } = await workerFixture(); + // No timer: the probe stays pending past the worker's 2s reap budget. + const probe = deferredProbe(); + try { + const startedAt = Date.now(); + const run = await runWorkerTask({ dataRoot: fixture.dataRoot, taskId: "task_1", attempt: 1, adapter: new FakeAdapter(), probe }); + assert.equal(run.status, "completed"); + assert.ok(Date.now() - startedAt >= 1_900, "the worker must wait out the bounded reap budget before returning"); + removeTree(fixture.dataRoot); + assert.equal(existsSync(tasksRootOf(fixture.dataRoot)), false, "fixture removed before the late settle"); + probe.settleSelf(); + await probe.whenSelfSettled(); + assert.equal(existsSync(tasksRootOf(fixture.dataRoot)), false, "a late identity write must not recreate the store tree"); + } finally { + removeTree(fixture.dataRoot); + await fixture.cleanup(); + } +}); + +test("a genuinely rejecting identity probe on the failure path has no unhandled rejection and no resurrection (kills the fence mutant too)", { timeout: 20_000 }, async () => { + const { fixture } = await workerFixture(); + const probe = deferredProbe(); + const unhandled: unknown[] = []; + const onUnhandled = (reason: unknown): void => { unhandled.push(reason); }; + process.on("unhandledRejection", onUnhandled); + try { + const adapter = new FakeAdapter(); + adapter.behavior = "adapterFailed"; + const run = await runWorkerTask({ dataRoot: fixture.dataRoot, taskId: "task_1", attempt: 1, adapter, probe }); + assert.equal(run.status, "failed"); + removeTree(fixture.dataRoot); + // A REAL rejection, delivered only after the fixture is gone. + probe.settleSelf(new Error("late identity probe failure")); + await probe.whenSelfSettled(); + await new Promise((resolve) => setImmediate(resolve)); + assert.equal(existsSync(tasksRootOf(fixture.dataRoot)), false, "no resurrection from the rejecting failure-path probe"); + assert.deepEqual(unhandled, [], "a rejected advisory identity probe must stay handled"); + } finally { + process.off("unhandledRejection", onUnhandled); + removeTree(fixture.dataRoot); + await fixture.cleanup(); + } +}); + +test("the real adapter's late runtime identity settled after removal is reaped identically (kills the runtime fence mutant)", { timeout: 30_000 }, async () => { + const fixture = await runtimeFixture(); + const probe = deferredProbe({ autoSelfMs: 0 }); + const store = new TaskStore(fixture.root); + store.createTask({ + task_id: "task_1", + workspace: fixture.root, + objective: "probe runtime identity lifecycle", + requirements: [], + allowed_paths: [], + forbidden_paths: [], + acceptance_criteria: [], + test_commands: [], + }, new Date().toISOString()); + const envBefore = { ...process.env }; + Object.assign(process.env, fixture.env); + try { + const run = await runWorkerTask({ dataRoot: fixture.root, taskId: "task_1", attempt: 1, host: fixture.host, probe }); + assert.equal(run.status, "completed"); + removeTree(fixture.root); + probe.settleRuntime(); + await probe.whenRuntimeSettled(); + assert.equal(existsSync(tasksRootOf(fixture.root)), false, "a late runtime identity write must not recreate the store tree"); + } finally { + for (const key of Object.keys(process.env)) if (!(key in envBefore)) delete process.env[key]; + Object.assign(process.env, envBefore); + removeTree(fixture.root); + await fixture.cleanup(); + } +}); diff --git a/test/legacy-retry-generation.test.ts b/test/legacy-retry-generation.test.ts new file mode 100644 index 0000000..9d51d22 --- /dev/null +++ b/test/legacy-retry-generation.test.ts @@ -0,0 +1,141 @@ +// Review-round-2 C1 regressions: the legacy adapter REUSES its RunEntry +// across retry generations. Every spawn-local field must reset per +// generation so a closed first generation can neither short-circuit the +// second generation's termination nor leak its identity. Fake spawns and an +// injected terminator only — zero OS signals. +import assert from "node:assert/strict"; +import test from "node:test"; +import { ZCodeAdapter } from "../src/adapters/zcode-adapter.js"; +import type { ProcessProbe } from "../src/adapters/process-spawn.js"; +import { FakeSpawn, TRANSIENT_STDERR, makeTask, makeWorkspace, makeTempDir, removeTempDir, stubResolver } from "./helpers.js"; + +/** Full-shape fake probe: exact per-pid fingerprints, identity_version and + * captured_at included, so the ProcessProbe contract is honestly satisfied. */ +function generationProbe(behavior: (pid: number) => Promise<{ fingerprint: string | null }>): ProcessProbe { + const identityOf = async (pid: number) => { + const answer = await behavior(pid); + return { + pid, + fingerprint: answer.fingerprint, + fingerprint_precision: "exact" as const, + identity_version: 1 as const, + platform: process.platform, + captured_at: new Date().toISOString(), + }; + }; + return { + platform: process.platform, + identityOf, + selfIdentity: () => identityOf(process.pid), + probe: async () => [], + }; +} + +test("C1: after a transient first generation, cancelling the live second generation really terminates it with its own identity", { timeout: 20_000 }, async () => { + const fake = new FakeSpawn(); + const terminations: Array<{ pid: number; expectedRootFingerprint: string | null }> = []; + const probe = generationProbe(async (pid) => ({ fingerprint: `exact-${String(pid)}` })); + const promptDir = await makeTempDir("legacy-retry-cancel"); + const adapter = new ZCodeAdapter({ + resolver: stubResolver(), + spawnImpl: fake.spawnFn, + terminateProcessTreeImpl: async (pid, options) => { + terminations.push({ pid, expectedRootFingerprint: options?.expectedRootFingerprint ?? null }); + return { pid, signal: "SIGKILL", verified: true }; + }, + probe, + promptTmpDir: promptDir, + childEnvBase: { PATH: "C:\\Windows" }, + retryBackoffMs: 1, + timeoutMs: 60_000, + }); + try { + // Generation 1 dies with the known transient error; the retry policy + // spawns generation 2, which stays alive. + fake.script.push((child) => { + child.emitStderr(TRANSIENT_STDERR); + child.emitClose(1, null); + }); + fake.script.push(() => { + // Generation 2 keeps running until cancelled. + }); + const handle = await adapter.startTask({ task: makeTask(), workspace: makeWorkspace(), attempt: 1 }); + for (let index = 0; index < 400 && fake.records.length < 2; index += 1) { + await new Promise((resolve) => setTimeout(resolve, 5)); + } + assert.equal(fake.records.length, 2, "the retry policy spawned a second generation"); + await new Promise((resolve) => setTimeout(resolve, 30)); // let captures settle + await adapter.cancelTask(handle); + const second = fake.records[1]!.child; + assert.equal(terminations.length, 1, "exactly one termination — against the live generation"); + assert.equal(terminations[0]!.pid, second.pid, "the SECOND generation is terminated"); + if (process.platform === "win32") { + assert.equal(terminations[0]!.expectedRootFingerprint, `exact-${String(second.pid)}`, "the termination carries the second generation's exact identity"); + } + fake.records[1]!.child.emitStdout(JSON.stringify({})); + fake.records[1]!.child.emitClose(0, null); + await adapter.getResult(handle).catch(() => undefined); + } finally { + for (const record of fake.records) record.child.emitClose(0, null); + await removeTempDir(promptDir); + } +}); + +test("C1: a late identity capture from generation 1 never overwrites generation 2's identity", { timeout: 20_000 }, async () => { + const fake = new FakeSpawn(); + const terminations: Array<{ pid: number; expectedRootFingerprint: string | null }> = []; + let identityOfCalls = 0; + // Generation 1's first read fails (null) so its capture enters the 250 ms + // retry window; the delayed second read resolves AFTER generation 2 has + // spawned and captured its own identity. + const probe = generationProbe(async (pid) => { + identityOfCalls += 1; + if (pid === fake.records[0]?.child.pid && identityOfCalls === 1) { + await new Promise((resolve) => setTimeout(resolve, 80)); + return { fingerprint: null }; + } + return { fingerprint: `exact-${String(pid)}` }; + }); + const promptDir = await makeTempDir("legacy-retry-stale"); + const adapter = new ZCodeAdapter({ + resolver: stubResolver(), + spawnImpl: fake.spawnFn, + terminateProcessTreeImpl: async (pid, options) => { + terminations.push({ pid, expectedRootFingerprint: options?.expectedRootFingerprint ?? null }); + return { pid, signal: "SIGKILL", verified: true }; + }, + probe, + promptTmpDir: promptDir, + childEnvBase: { PATH: "C:\\Windows" }, + retryBackoffMs: 1, + timeoutMs: 60_000, + }); + try { + fake.script.push((child) => { + child.emitStderr(TRANSIENT_STDERR); + child.emitClose(1, null); + }); + fake.script.push(() => { + // Generation 2 stays alive. + }); + const handle = await adapter.startTask({ task: makeTask(), workspace: makeWorkspace(), attempt: 1 }); + for (let index = 0; index < 400 && fake.records.length < 2; index += 1) { + await new Promise((resolve) => setTimeout(resolve, 5)); + } + assert.equal(fake.records.length, 2, "generation 2 spawned"); + // Wait long enough that generation 1's delayed capture has resolved. + await new Promise((resolve) => setTimeout(resolve, 450)); + await adapter.cancelTask(handle); + const second = fake.records[1]!.child; + assert.equal(terminations.length, 1); + assert.equal(terminations[0]!.pid, second.pid); + if (process.platform === "win32") { + assert.equal(terminations[0]!.expectedRootFingerprint, `exact-${String(second.pid)}`, "generation 2's identity survived generation 1's late capture"); + } + fake.records[1]!.child.emitClose(0, null); + await adapter.getResult(handle).catch(() => undefined); + } finally { + for (const record of fake.records) record.child.emitClose(0, null); + await removeTempDir(promptDir); + } +}); diff --git a/test/manager-helpers.ts b/test/manager-helpers.ts index b531419..613eaa5 100644 --- a/test/manager-helpers.ts +++ b/test/manager-helpers.ts @@ -148,7 +148,12 @@ export interface ManagerFixture { spawned: Array<{ dataRoot: string; taskId: string; pid: number }>; pidsAlive: Set; terminateCalls: number[]; + /** expectedRootFingerprint received per termination call (null = none). */ + terminateExpected: Array; setTerminateError: (error: Error | null) => void; + /** Recorded-tree claimant scans requested by the manager (ADR-009). */ + scanCalls: Array>; + setScanResult: (result: { clean: boolean; livePids: readonly number[] } | Error) => void; /** Moves the fake clock forward; the manager's injected now() reads it. */ advanceMs: (ms: number) => void; makeTask: (overrides?: Partial) => TaskPackage; @@ -167,17 +172,27 @@ export async function makeManagerFixture(options: { const spawned: Array<{ dataRoot: string; taskId: string; pid: number }> = []; const pidsAlive = new Set(); const terminateCalls: number[] = []; + const terminateExpected: Array = []; + const scanCalls: Array> = []; let terminateError: Error | null = null; + let scanResult: { clean: boolean; livePids: readonly number[] } | Error = { clean: true, livePids: [] }; const clockBaseMs = Date.UTC(2026, 8, 27, 0, 0, 0); let clockMs = 1_000; - const terminate: TerminateProcessTree = async (pid) => { + const terminate: TerminateProcessTree = async (pid, terminateOptions) => { terminateCalls.push(pid); + terminateExpected.push(terminateOptions?.expectedRootFingerprint ?? null); if (terminateError) throw terminateError; pidsAlive.delete(pid); return { pid, signal: "SIGKILL", verified: true }; }; + const verifyTreeClean = async (targets: readonly { pid: number; fingerprint: string | null }[]) => { + scanCalls.push([...targets]); + if (scanResult instanceof Error) throw scanResult; + return scanResult; + }; + const manager = new BridgeTaskManager({ store, workspaceProvider: new DirectWorkspaceProvider(), @@ -189,6 +204,7 @@ export async function makeManagerFixture(options: { }, isProcessRunning: (pid) => pidsAlive.has(pid), terminateProcessTree: terminate, + verifyTreeClean, pollIntervalMs: 0, maxConcurrentWorkers: options.maxConcurrentWorkers, workerStartGraceMs: options.workerStartGraceMs ?? 0, @@ -203,9 +219,14 @@ export async function makeManagerFixture(options: { spawned, pidsAlive, terminateCalls, + terminateExpected, setTerminateError: (error) => { terminateError = error; }, + scanCalls, + setScanResult: (result) => { + scanResult = result; + }, advanceMs: (ms) => { clockMs += ms; }, diff --git a/test/manager-recovery-latch.test.ts b/test/manager-recovery-latch.test.ts new file mode 100644 index 0000000..3056d6f --- /dev/null +++ b/test/manager-recovery-latch.test.ts @@ -0,0 +1,201 @@ +// Deterministic consumer-level regressions for evaluator/review-round-1 R1/R3: +// a latched descendant-cleanup failure must survive recovery (persisted +// worker results, outcome checkpoints, worker_lost finalization, terminal +// probe recovery) and a corrupt failure record must fail closed instead of +// narrowing the recorded-tree scan. No real processes; injected terminator. +import assert from "node:assert/strict"; +import test from "node:test"; +import { WindowsTreeCleanupError } from "../src/adapters/windows-process-cleanup.js"; +import { buildTaskResult } from "../src/manager/normalize.js"; +import { FakeAdapter, makeManagerFixture } from "./manager-helpers.js"; + +test("R1: recoverTasks cannot release a task whose latched cleanup failure carries known members", async () => { + const fx = await makeManagerFixture(); + try { + await fx.manager.createTask(fx.makeTask()); + const pid = fx.spawned[0]!.pid; + // A failed manager termination observed a surviving descendant (334455). + fx.setTerminateError(new WindowsTreeCleanupError("process_tree_cleanup_unverified", "known descendant still alive", [pid, 334455])); + await assert.rejects(fx.manager.cancelTask("task_1"), /could not be verified/); + assert.equal(fx.store.readStatus("task_1").cleanup_unverified, true); + assert.deepEqual(fx.store.readAttemptMeta("task_1", 1, "cleanup-failure.json")?.member_pids, [pid, 334455]); + + // The worker now commits a terminal result and exits before its own + // status update — the exact R1 timeline. Recovery must NOT adopt it. + const result = buildTaskResult({ task: fx.makeTask(), attempt: 1, startedAt: null, finishedAt: new Date().toISOString(), outcome: null, cancelled: true, sessionId: "session-x" }); + fx.store.writeResult("task_1", result); + fx.pidsAlive.delete(pid); + await fx.manager.recoverTasks(); + const after = fx.store.readStatus("task_1"); + assert.equal(after.status, "running", "the latched task stays occupied"); + assert.equal(after.cleanup_unverified, true, "the latch survives recovery"); + assert.equal(fx.scanCalls.length, 0, "no scan-gated release happened outside cancel"); + + // The eventual release still works: fresh probes see the executors gone + // and the claimant scan covers the recorded members (334455 included). + fx.setTerminateError(null); + const status = await fx.manager.cancelTask("task_1"); + assert.equal(status.status, "cancelled"); + assert.equal(fx.scanCalls.length, 1); + assert.ok(fx.scanCalls[0]!.some((t) => t.pid === 334455), "the recorded member pid is scanned"); + assert.equal(fx.store.readStatus("task_1").cleanup_unverified ?? false, false); + } finally { + await fx.cleanup(); + } +}); + +test("R1/R3: a corrupt failure record fails the later cancel closed instead of narrowing the scan", async () => { + const fx = await makeManagerFixture(); + try { + await fx.manager.createTask(fx.makeTask()); + const pid = fx.spawned[0]!.pid; + fx.setTerminateError(new WindowsTreeCleanupError("process_tree_cleanup_unverified", "known descendant still alive", [pid, 334455])); + await assert.rejects(fx.manager.cancelTask("task_1"), /could not be verified/); + fx.pidsAlive.delete(pid); // the worker later exits naturally + + // The record is damaged: the member list is no longer a valid pid array. + fx.store.writeAttemptMeta("task_1", 1, "cleanup-failure.json", { member_pids: "corrupt-known-member-list", last_error: "known descendant still alive" }); + await assert.rejects(fx.manager.cancelTask("task_1"), /corrupt/); + const after = fx.store.readStatus("task_1"); + assert.equal(after.status, "running"); + assert.equal(after.cleanup_unverified, true, "the latch stays"); + assert.equal(fx.scanCalls.length, 0, "a corrupt record never reaches a release scan"); + } finally { + await fx.cleanup(); + } +}); + +test("R1 boundary: a latched terminal task WITHOUT a member-bearing failure record still recovers via probes", async () => { + const fx = await makeManagerFixture(); + try { + await fx.manager.createTask(fx.makeTask()); + await fx.runWorker("task_1", new FakeAdapter()); // completes normally + // Legacy shape: cleanup_unverified latched by an older code path with no + // cleanup-failure.json at all. Probe-verified recovery keeps working. + fx.store.writeStatus("task_1", { cleanup_unverified: true }); + await fx.manager.recoverTasks(); + const after = fx.store.readStatus("task_1"); + assert.equal(after.cleanup_unverified ?? false, false, "no recorded members: the probe path still releases"); + assert.equal(fx.scanCalls.length, 0); + } finally { + await fx.cleanup(); + } +}); + +test("R3: the durable status marker recovers the member set when the failure record is missing", async () => { + const fx = await makeManagerFixture(); + try { + await fx.manager.createTask(fx.makeTask()); + const pid = fx.spawned[0]!.pid; + fx.setTerminateError(new WindowsTreeCleanupError("process_tree_cleanup_unverified", "known descendant still alive", [pid, 334455])); + await assert.rejects(fx.manager.cancelTask("task_1"), /could not be verified/); + assert.match(String(fx.store.readStatus("task_1").error), /\[cleanup-members: .*\b334455\b/, "the latch text carries the member set durably"); + fx.pidsAlive.delete(pid); + // The record file disappears (loss/corruption-by-absence after the + // latch): the marker in the latched status still expands the scan. + fx.store.writeAttemptMeta("task_1", 1, "cleanup-failure.json", { member_pids: [], last_error: "known descendant still alive" }); + const status = await fx.manager.cancelTask("task_1"); + assert.equal(status.status, "cancelled"); + assert.equal(fx.scanCalls.length, 1); + assert.ok(fx.scanCalls[0]!.some((t) => t.pid === 334455), "the marker-recording member pid is scanned"); + } finally { + await fx.cleanup(); + } +}); + +test("R3 addendum: a failure record with a missing member_pids field fails closed", async () => { + const fx = await makeManagerFixture(); + try { + await fx.manager.createTask(fx.makeTask()); + const pid = fx.spawned[0]!.pid; + fx.setTerminateError(new WindowsTreeCleanupError("process_tree_cleanup_unverified", "known descendant still alive", [pid, 334455])); + await assert.rejects(fx.manager.cancelTask("task_1"), /could not be verified/); + fx.pidsAlive.delete(pid); + // last_error survives but member_pids is gone: missing must never read + // as an empty member set. + fx.store.writeAttemptMeta("task_1", 1, "cleanup-failure.json", { last_error: "known descendant still alive" }); + await assert.rejects(fx.manager.cancelTask("task_1"), /corrupt/); + const after = fx.store.readStatus("task_1"); + assert.equal(after.status, "running"); + assert.equal(after.cleanup_unverified, true); + assert.equal(fx.scanCalls.length, 0); + } finally { + await fx.cleanup(); + } +}); + +test("R3 addendum: a truncated member marker cannot restore the full set and fails closed", async () => { + const fx = await makeManagerFixture(); + try { + await fx.manager.createTask(fx.makeTask()); + const pid = fx.spawned[0]!.pid; + // Enough members that the durable marker must truncate (>1600 chars). + const manyMembers = [pid, ...Array.from({ length: 260 }, (_, index) => 330_000 + index), 334_455]; + fx.setTerminateError(new WindowsTreeCleanupError("process_tree_cleanup_unverified", "known descendant still alive", manyMembers)); + await assert.rejects(fx.manager.cancelTask("task_1"), /could not be verified/); + fx.pidsAlive.delete(pid); + assert.match(String(fx.store.readStatus("task_1").error), /cleanup-members: .*,…]/, "the marker is truncated for an over-long set"); + // The structural record is then lost (the evaluator probe writes a + // literal JSON null): only the truncated marker remains. + fx.store.writeAttemptMeta("task_1", 1, "cleanup-failure.json", null as unknown as Parameters[3]); + await assert.rejects(fx.manager.cancelTask("task_1"), /incomplete/); + const after = fx.store.readStatus("task_1"); + assert.equal(after.status, "running"); + assert.equal(after.cleanup_unverified, true); + assert.equal(fx.scanCalls.length, 0); + } finally { + await fx.cleanup(); + } +}); + +test("R3 addendum 13:39: three consecutive terminator failures never wash an incomplete member set back to complete", async () => { + const fx = await makeManagerFixture(); + try { + await fx.manager.createTask(fx.makeTask()); + const pid = fx.spawned[0]!.pid; + // The worker stays ALIVE through all three terminator failures so every + // cancel really invokes the terminator and #recordCleanupFailure; it + // only exits before the fourth cancel (the evaluator probe's shape). + const manyMembers = [pid, ...Array.from({ length: 260 }, (_, index) => 330_000 + index), 334_455]; + + // Failure 1: the full set truncates the durable marker; the structural + // record is written with complete:false. + fx.setTerminateError(new WindowsTreeCleanupError("process_tree_cleanup_unverified", "first known descendant failure", manyMembers)); + await assert.rejects(fx.manager.cancelTask("task_1"), /could not be verified/); + assert.match(String(fx.store.readStatus("task_1").error), /cleanup-members: .*,…]/); + assert.equal(fx.store.readAttemptMeta("task_1", 1, "cleanup-failure.json")?.complete, false); + + // The structural record is then lost; only the truncated marker remains. + fx.store.writeAttemptMeta("task_1", 1, "cleanup-failure.json", null as unknown as Parameters[3]); + + // Failure 2 (worker still alive, terminator really called): a + // member-less failure recovers the partial marker and must keep both + // the marker and the rewritten record explicitly incomplete. + fx.setTerminateError(new WindowsTreeCleanupError("process_tree_cleanup_unverified", "second failure", [])); + await assert.rejects(fx.manager.cancelTask("task_1"), /could not be verified/); + assert.match(String(fx.store.readStatus("task_1").error), /cleanup-members: .*,…]/); + assert.equal(fx.store.readAttemptMeta("task_1", 1, "cleanup-failure.json")?.complete, false, "the recovered record stays explicitly partial"); + + // Failure 3 (worker STILL alive, terminator really called again): the + // partial structural record now reads back VALID — exactly the case the + // previous code washed clean. The incomplete state must survive this + // successful read: no fresh summary may claim completeness. + fx.setTerminateError(new WindowsTreeCleanupError("process_tree_cleanup_unverified", "third failure", [])); + await assert.rejects(fx.manager.cancelTask("task_1"), /could not be verified/); + assert.match(String(fx.store.readStatus("task_1").error), /cleanup-members: .*,…]/, "the marker stays truncated after a third failure"); + assert.equal(fx.store.readAttemptMeta("task_1", 1, "cleanup-failure.json")?.complete, false, "the record stays explicitly partial after a third failure"); + + // The worker now exits naturally; the fourth cancel cannot invoke the + // terminator any more. The partial state must still refuse the release: + // member 334455 (beyond the marker's first 200 entries) must never be + // scanned away by a narrowed set. + fx.pidsAlive.delete(pid); + await assert.rejects(fx.manager.cancelTask("task_1"), /incomplete/); + const after = fx.store.readStatus("task_1"); + assert.equal(after.status, "running"); + assert.equal(after.cleanup_unverified, true); + assert.equal(fx.scanCalls.length, 0, "no release scan runs on a partial member record"); + } finally { + await fx.cleanup(); + } +}); diff --git a/test/owned-fixture-cleanup.test.ts b/test/owned-fixture-cleanup.test.ts new file mode 100644 index 0000000..3e3de0c --- /dev/null +++ b/test/owned-fixture-cleanup.test.ts @@ -0,0 +1,152 @@ +// Gated fixture-teardown regressions for cleanupOwnedProcessFixture +// (2026-10-17 matrix2 review): the helper must confirm OWNED child exit +// before removing the fixture tree, and any release/exit-confirmation +// failure must preserve the tree and fail visibly instead of deleting +// around a live child. The refusing-child scenarios are controlled — the +// child ignores the release file on purpose so the gate's timeout branch is +// deterministic; afterwards the test stops its OWN child handle (kill +// first, then a bounded close confirmation) and never deletes the tree when +// the stop cannot be confirmed. +import assert from "node:assert/strict"; +import test from "node:test"; +import { spawn, type ChildProcess } from "node:child_process"; +import { existsSync, mkdtempSync, rmSync, writeFileSync } from "node:fs"; +import { readFile } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import path from "node:path"; +import { cleanupOwnedProcessFixture, TEMP_TREE_REMOVE_OPTIONS } from "./helpers.js"; +import { isProcessRunning } from "../src/adapters/process-spawn.js"; + +/** A real child of THIS test: writes a ready receipt from its fixture cwd, + * then either watches the release file (cooperative) or keeps running + * (refusing) so the gate's timeout branch is deterministic. */ +function spawnFixtureChild(dir: string, options: { cooperative: boolean }): { + child: ChildProcess; + release: string; + ready: string; + closed: Promise<{ code: number | null; signal: NodeJS.Signals | null }>; +} { + const script = path.join(dir, options.cooperative ? "cooperative.cjs" : "refusing.cjs"); + const release = path.join(dir, "release"); + const ready = path.join(dir, "ready.json"); + writeFileSync(script, [ + '"use strict";', + "const fs = require('node:fs');", + `const release = ${JSON.stringify(release)};`, + `const ready = ${JSON.stringify(ready)};`, + "fs.writeFileSync(ready, JSON.stringify({ pid: process.pid, cwd: process.cwd() }));", + options.cooperative + ? "function watch() { if (fs.existsSync(release)) process.exit(0); setTimeout(watch, 25); } watch();" + : "setInterval(() => undefined, 1_000);", + ].join("\n"), "utf8"); + const child = spawn(process.execPath, [script], { cwd: dir, stdio: "ignore", windowsHide: true }); + const closed = new Promise<{ code: number | null; signal: NodeJS.Signals | null }>((resolve) => { + child.once("close", (code, signal) => resolve({ code, signal })); + }); + child.once("error", () => { /* surfaced through the bounded close wait */ }); + return { child, release, ready, closed }; +} + +const waitForReady = async (ready: string, timeoutMs: number): Promise<{ pid: number; cwd: string }> => { + const deadline = Date.now() + timeoutMs; + for (;;) { + try { + return JSON.parse(await readFile(ready, "utf8")) as { pid: number; cwd: string }; + } catch { + if (Date.now() >= deadline) throw new Error(`fixture child did not write its ready receipt within ${String(timeoutMs)} ms`); + await new Promise((resolve) => setTimeout(resolve, 25)); + } + } +}; + +/** Stop this test's OWN child handle: deliver the kill signal first, then + * CONFIRM the close event within the bound. A kill delivery failure or a + * close timeout throws — the test fails and the caller keeps the tree. */ +const stopFixtureChild = async (child: ChildProcess, closed: Promise, timeoutMs = 5_000): Promise => { + if (child.exitCode !== null || child.signalCode !== null) { await closed; return; } + const signalled = child.kill(); + if (!signalled) throw new Error(`the kill signal could not be delivered to fixture child pid ${String(child.pid)}`); + const winner = await Promise.race([ + closed.then(() => "closed" as const), + new Promise<"timeout">((resolve) => { + const timer = setTimeout(() => resolve("timeout"), timeoutMs); + if (typeof timer.unref === "function") timer.unref(); + }), + ]); + if (winner !== "closed") throw new Error(`fixture child pid ${String(child.pid)} did not close within ${String(timeoutMs)} ms of the kill signal`); +}; + +/** Test-side final gate: only removes the tree after the owned child's close + * is confirmed; a stop failure fails the test and preserves the tree. */ +const finalizeTestTree = async (child: ChildProcess, closed: Promise, dir: string): Promise => { + await stopFixtureChild(child, closed); + rmSync(dir, { recursive: true, force: true, ...TEMP_TREE_REMOVE_OPTIONS }); +}; + +test("a cooperative owned child is released, confirmed exited with code 0, and the tree removed", { timeout: 30_000 }, async () => { + const dir = mkdtempSync(path.join(tmpdir(), "bridge-owned-cleanup-ok-")); + const { child, release, ready, closed } = spawnFixtureChild(dir, { cooperative: true }); + try { + const handshake = await waitForReady(ready, 10_000); + assert.equal(handshake.cwd, dir, "the child runs with the fixture tree as its cwd (the EBUSY shape)"); + assert.ok(typeof child.pid === "number" && isProcessRunning(child.pid), "the fixture child is alive after its ready receipt — receipt is not exit"); + await cleanupOwnedProcessFixture({ pids: [child.pid!], releasePath: release, dir, primaryFailed: false, timeoutMs: 10_000 }); + const outcome = await closed; + assert.equal(outcome.code, 0, "the cooperative child must exit cleanly via the release path (not a startup failure)"); + assert.equal(existsSync(dir), false, "the fixture tree is removed only after the exit confirmation"); + } finally { + await finalizeTestTree(child, closed, dir); + } +}); + +test("a child that ignores the release signal preserves the tree and fails visibly", { timeout: 30_000 }, async () => { + const dir = mkdtempSync(path.join(tmpdir(), "bridge-owned-cleanup-refuses-")); + const { child, release, ready, closed } = spawnFixtureChild(dir, { cooperative: false }); + try { + await waitForReady(ready, 10_000); + assert.ok(typeof child.pid === "number" && isProcessRunning(child.pid)); + await assert.rejects( + cleanupOwnedProcessFixture({ pids: [child.pid!], releasePath: release, dir, primaryFailed: false, timeoutMs: 300 }), + /did not report exit within 300 ms.*fixture tree preserved/u, + "the gate must report the unconfirmed exit and preserve the tree", + ); + assert.equal(existsSync(dir), true, "no deletion may happen around a live owned child"); + assert.equal(isProcessRunning(child.pid!), true, "the refusing child is still alive"); + } finally { + await finalizeTestTree(child, closed, dir); + } +}); + +test("with a primary failure propagating, the gate reports diagnostics without masking it and still preserves the tree", { timeout: 30_000 }, async () => { + const dir = mkdtempSync(path.join(tmpdir(), "bridge-owned-cleanup-primary-")); + const { child, release, ready, closed } = spawnFixtureChild(dir, { cooperative: false }); + const originalError = console.error; + const diagnostics: string[] = []; + const primaryError = new Error("primary assertion failed (injected)"); + let caught: unknown = null; + let treePreservedWhenGateFinished = false; + try { + await waitForReady(ready, 10_000); + console.error = (...args: unknown[]) => { diagnostics.push(args.map(String).join(" ")); }; + try { + throw primaryError; + } finally { + try { + await cleanupOwnedProcessFixture({ pids: [child.pid!], releasePath: release, dir, primaryFailed: true, timeoutMs: 300 }); + treePreservedWhenGateFinished = existsSync(dir); + } finally { + console.error = originalError; + } + } + assert.fail("the primary error must have propagated"); + } catch (error) { + caught = error; + } finally { + console.error = originalError; + await finalizeTestTree(child, closed, dir); + } + assert.equal(caught, primaryError, "the primary failure propagates with its identity intact"); + assert.ok(diagnostics.some((line) => line.includes("fixture teardown failed") && line.includes("fixture tree preserved")), + `the cleanup diagnostics stay visible alongside the primary failure: ${JSON.stringify(diagnostics)}`); + assert.equal(treePreservedWhenGateFinished, true, "the tree stays for diagnosis while the primary failure propagates (before this test's own finalizer runs)"); +}); diff --git a/test/process-lock.test.ts b/test/process-lock.test.ts index ebf5aeb..6504170 100644 --- a/test/process-lock.test.ts +++ b/test/process-lock.test.ts @@ -1,10 +1,657 @@ import assert from "node:assert/strict"; -import { spawnSync } from "node:child_process"; -import { existsSync, mkdtempSync, readdirSync, rmSync, writeFileSync } from "node:fs"; +import { spawn, spawnSync } from "node:child_process"; +import { existsSync, mkdirSync, mkdtempSync, readdirSync, readFileSync, rmSync, writeFileSync } from "node:fs"; import { tmpdir } from "node:os"; +import { syncBuiltinESMExports } from "node:module"; import path from "node:path"; import test from "node:test"; import { tryAcquireProcessLock } from "../src/store/process-lock.js"; +import { captureSpawnGenerationIdentity, createPlatformProbe, terminateProcessTree } from "../src/adapters/process-spawn.js"; + +// Windows filesystem timing (2026-10-17 investigation): under real +// cross-process contention the lock's directory renames can fail transiently — +// publication onto a delete-pending name (EEXIST/EPERM/ENOTEMPTY) and +// withdrawal-stage EPERM (the competitor-handle explanation is a working +// hypothesis, not handle-level verified). These regressions pin the contract: +// whitelisted codes are retried within bounded budgets, non-transient errors +// and budget exhaustion stay visible, and a failed withdrawal keeps a complete +// readable lock that is never evicted while its owner lives. +import fs from "node:fs"; + +/** Patches one fs builtin for the duration of `body` so the compiled + * process-lock module (which imports the same builtin) observes the wrapper; + * syncBuiltinESMExports keeps the ESM view aligned. The wrapper closes over + * the real implementation it receives. */ +function withPatchedFs( + fnName: K, + makeWrapper: (real: typeof fs[K]) => typeof fs[K], + body: () => void, +): void { + const real = fs[fnName] as typeof fs[K]; + fs[fnName] = makeWrapper(real); + syncBuiltinESMExports(); + try { body(); } + finally { fs[fnName] = real; syncBuiltinESMExports(); } +} + +function withPatchedRenameSync(fail: (source: string, target: string) => NodeJS.ErrnoException | null, body: () => void): void { + withPatchedFs("renameSync", (real) => ((source: fs.PathLike, target: fs.PathLike) => { + const injected = fail(String(source), String(target)); + if (injected) throw injected; + return real(source, target); + }) as typeof fs.renameSync, body); +} + +function withPatchedUnlinkSync(fail: (target: string) => NodeJS.ErrnoException | null, body: () => void): void { + withPatchedFs("unlinkSync", (real) => ((target: fs.PathLike) => { + const injected = fail(String(target)); + if (injected) throw injected; + return real(target); + }) as typeof fs.unlinkSync, body); +} + +function withPatchedRmdirSync(fail: (target: string) => NodeJS.ErrnoException | null, body: () => void): void { + withPatchedFs("rmdirSync", (real) => ((target: fs.PathLike, options?: fs.RmDirOptions) => { + const injected = fail(String(target)); + if (injected) throw injected; + return real(target, options); + }) as typeof fs.rmdirSync, body); +} + +/** Sequence-controlled owner.json reads: the Nth read of a path can return a + * different buffer or fail, reproducing records that change or vanish between + * acquireExistingLock's read and the guarded recheck inside reclaim. */ +function withSequencedOwnerReads(path: string, outcomes: Array, body: () => void): void { + let reads = 0; + withPatchedFs("readFileSync", (real) => ((target: fs.PathOrFileDescriptor, options?: unknown) => { + if (String(target) === path && reads < outcomes.length) { + const outcome = outcomes[reads]!; + reads += 1; + if (outcome instanceof Error) throw outcome; + return outcome; + } + return real(target, options as undefined); + }) as typeof fs.readFileSync, body); +} + +/** Sequence-controlled liveness probes: the Nth kill(pid, 0) can be forced to + * ESRCH or success, reproducing an owner that is dead at first sight and live + * again at the guarded recheck. */ +function withSequencedKill(pid: number, outcomes: Array<"esrch" | "alive">, body: () => void): void { + let probes = 0; + const real = process.kill.bind(process); + process.kill = ((target: number, signal?: NodeJS.Signals | number) => { + if (target === pid && probes < outcomes.length) { + const outcome = outcomes[probes]!; + probes += 1; + if (outcome === "esrch") { + const error = new Error("injected ESRCH") as NodeJS.ErrnoException; + error.code = "ESRCH"; + throw error; + } + return true; + } + return real(target, signal); + }) as typeof process.kill; + try { body(); } + finally { process.kill = real as typeof process.kill; } +} + +const errnoError = (code: string): NodeJS.ErrnoException => { + const error = new Error(`injected ${code} (deterministic Windows-race regression)`) as NodeJS.ErrnoException; + error.code = code; + return error; +}; + +// Dead owners use the suite's fixed high-pid fixture (the same convention as +// observation-lifecycle A2-07). The fixture's premise — process.kill(pid, 0) +// reporting ESRCH — is asserted before each use instead of assumed, and no +// claim is made about why the pid is unused on any platform. +const IMPOSSIBLE_DEAD_PID = 987654; +const assertDeadPidFixture = (): void => { + assert.throws(() => process.kill(IMPOSSIBLE_DEAD_PID, 0), (error: NodeJS.ErrnoException) => error.code === "ESRCH", + "the fixed dead-owner pid fixture requires ESRCH on this platform; pick another value if this platform ever allocates it"); +}; +const seedDeadOwner = (lock: string): void => { + mkdirSync(lock, { recursive: true }); + writeFileSync(path.join(lock, "owner.json"), JSON.stringify({ pid: IMPOSSIBLE_DEAD_PID, token: "dead-owner", started_at: new Date().toISOString() })); +}; + +test("ambiguous absent-destination publication failures (EPERM/EACCES/EBUSY) are retried and publish exactly one owner", () => { + const root = mkdtempSync(path.join(tmpdir(), "bridge-lock-publish-")); + try { + const lock = path.join(root, "state.lock"); + let injected: string[] = []; + withPatchedRenameSync((_source, target) => { + if (target !== lock || injected.length >= 2) return null; + // Ambiguous transient codes only: with no destination these may be + // delete-pending races or scanner handles, which is what the bounded + // publication retry exists for. EEXIST/ENOTEMPTY are NOT injected here + // — they are definite destination-present contention (see the next + // test) and must never consume the retry budget. + const code = injected.length === 0 ? "EPERM" : "EBUSY"; + injected.push(code); + return errnoError(code); + }, () => { + const release = tryAcquireProcessLock(lock); + assert.ok(release, "publication must succeed after bounded transient retries"); + assert.equal(existsSync(lock), true, "lock must be published"); + const owner = JSON.parse(readFileSync(path.join(lock, "owner.json"), "utf8")); + assert.equal(owner.pid, process.pid, "the single owner must be this process"); + release(); + }); + assert.deepEqual(injected, ["EPERM", "EBUSY"], "both injected failures must have occurred"); + assert.equal(existsSync(lock), false, "release must withdraw the published lock"); + assert.deepEqual(readdirSync(root), [], "no staging or retired garbage may remain"); + } finally { rmSync(root, { recursive: true, force: true, maxRetries: 10, retryDelay: 50 }); } +}); + +test("definite destination-present contention (EEXIST/ENOTEMPTY) is never an escaping error, even when the destination already withdrew", { timeout: 15_000 }, () => { + // Deterministic reproduction of the Linux22 full-suite escape: a competing + // owner publishes between the existence precheck and the rename, and + // withdraws again before any recheck — so the destination is ABSENT when + // the failed rename is inspected. rename(2) only produces EEXIST/ENOTEMPTY + // for a destination that existed at rename time, so the errno itself is + // the contention evidence. The acquisition must report busy (null) without + // burning the publication retry budget; the previous implementation burned + // all four attempts and let the errno escape (worker/probe-red-enotempty- + // contention-node22.json). + const root = mkdtempSync(path.join(tmpdir(), "bridge-lock-contention-errno-")); + try { + for (const code of ["ENOTEMPTY", "EEXIST"]) { + const lock = path.join(root, `state-${code}.lock`); + let renameAttempts = 0; + withPatchedRenameSync((_source, target) => { + if (target !== lock) return null; + renameAttempts += 1; + return errnoError(code); + }, () => { + assert.equal(tryAcquireProcessLock(lock), null, `${code} from a publication rename is contention, reported as busy`); + }); + assert.equal(renameAttempts, 1, `${code} must hand off on the first attempt instead of consuming publication retries`); + } + // And when the competing destination is still present, the existence + // precheck routes to the existing-owner policies before any rename is + // attempted at all — the live owner stays put and nothing is published. + const held = path.join(root, "state-held.lock"); + const owner = tryAcquireProcessLock(held); + assert.ok(owner); + let heldAttempts = 0; + withPatchedRenameSync((_source, target) => { + if (target !== held) return null; + heldAttempts += 1; + return errnoError("ENOTEMPTY"); + }, () => { + assert.equal(tryAcquireProcessLock(held), null, "a held destination reports busy"); + }); + assert.equal(heldAttempts, 0, "the precheck must keep a held destination away from the publication rename entirely"); + owner(); + } finally { rmSync(root, { recursive: true, force: true, maxRetries: 10, retryDelay: 50 }); } +}); + +test("publication budget exhaustion fails visibly with the original code and leaves no artifacts", () => { + const root = mkdtempSync(path.join(tmpdir(), "bridge-lock-budget-")); + try { + const lock = path.join(root, "state.lock"); + let attempts = 0; + assert.throws(() => withPatchedRenameSync((_source, target) => { + if (target !== lock) return null; + attempts += 1; + return errnoError("EPERM"); + }, () => tryAcquireProcessLock(lock)), (error: NodeJS.ErrnoException) => error.code === "EPERM"); + assert.ok(attempts >= 2 && attempts <= 4, `retry must be bounded (attempts: ${attempts})`); + assert.equal(existsSync(lock), false, "nothing may be published on exhaustion"); + assert.deepEqual(readdirSync(root), [], "staging directories must be cleaned between attempts"); + } finally { rmSync(root, { recursive: true, force: true, maxRetries: 10, retryDelay: 50 }); } +}); + +test("non-transient publication errors are not retried", () => { + const root = mkdtempSync(path.join(tmpdir(), "bridge-lock-nontransient-")); + try { + const lock = path.join(root, "state.lock"); + let attempts = 0; + assert.throws(() => withPatchedRenameSync((_source, target) => { + if (target !== lock) return null; + attempts += 1; + return errnoError("EINVAL"); + }, () => tryAcquireProcessLock(lock)), (error: NodeJS.ErrnoException) => error.code === "EINVAL"); + assert.equal(attempts, 1, "a non-transient code must fail on the first attempt"); + } finally { rmSync(root, { recursive: true, force: true, maxRetries: 10, retryDelay: 50 }); } +}); + +test("a transient EPERM during withdrawal is retried and the lock is released", () => { + const root = mkdtempSync(path.join(tmpdir(), "bridge-lock-withdraw-")); + try { + const lock = path.join(root, "state.lock"); + const release = tryAcquireProcessLock(lock); + assert.ok(release); + let failedOnce = false; + withPatchedRenameSync((source, target) => { + if (source === lock && target.endsWith(".retired") && !failedOnce) { + failedOnce = true; + return errnoError("EPERM"); + } + return null; + }, () => { + release(); + assert.equal(failedOnce, true, "the injected withdrawal failure must have occurred"); + }); + assert.equal(existsSync(lock), false, "the lock must be withdrawn after the retry"); + assert.deepEqual(readdirSync(root), [], "retired directory must be cleaned"); + } finally { rmSync(root, { recursive: true, force: true, maxRetries: 10, retryDelay: 50 }); } +}); + +test("exhausted withdrawal retries keep a complete lock and never evict the live owner", { timeout: 15_000 }, () => { + const root = mkdtempSync(path.join(tmpdir(), "bridge-lock-withdraw-budget-")); + try { + const lock = path.join(root, "state.lock"); + const release = tryAcquireProcessLock(lock); + assert.ok(release); + assert.throws(() => withPatchedRenameSync((_source, target) => (target.endsWith(".retired") ? errnoError("EPERM") : null), () => { + release(); + }), (error: NodeJS.ErrnoException) => error.code === "EPERM"); + assert.equal(existsSync(lock), true, "the shared lock must survive a failed withdrawal"); + const owner = JSON.parse(readFileSync(path.join(lock, "owner.json"), "utf8")); + assert.equal(owner.pid, process.pid, "the lock record must stay complete and readable"); + assert.equal(tryAcquireProcessLock(lock), null, "a live owner must not be evicted"); + } finally { rmSync(root, { recursive: true, force: true, maxRetries: 10, retryDelay: 50 }); } +}); + +/** A tracked child for the contention scenarios: listeners attach at spawn + * time so no close/error event can be lost, and every child can be reaped + * (terminated and awaited) even when an assertion fails mid-test. */ +interface TrackedChild { + readonly child: ReturnType; + readonly closed: Promise; + stdout: () => string; +} + +function spawnTracked(script: string, argv: string[] = []): TrackedChild { + const child = spawn(process.execPath, [script, ...argv], { windowsHide: true, stdio: ["ignore", "pipe", "inherit"] }); + let stdout = ""; + child.stdout?.on("data", (chunk: Buffer) => { stdout += String(chunk); }); + child.once("error", () => { /* recorded via the close promise */ }); + return { child, closed: new Promise((resolve) => child.once("close", resolve)), stdout: () => stdout }; +} + +/** Terminates every still-running child of this test and CONFIRMS the close + * event for each before cleanup may run — a leftover prober holding the lock + * tree is exactly the teardown race this task fixes. A termination failure or + * a close that never arrives within the bound is reported by throwing, never + * by silently proceeding (2026-10-17 review: swallowed errors and timeout + * races were false successes). "Closed" means the close event fired; on POSIX + * a signal exit legitimately leaves exitCode null while signalCode is set. + * The terminator is injectable so the failure branches are testable. */ +async function reapTrackedChildren( + tracked: readonly TrackedChild[], + terminate: (child: import("node:child_process").ChildProcess) => Promise = async (child) => { + // ADR-009: hold the ORIGINAL ChildProcess. The shared capture + // re-verifies the spawn generation after every await (exit state, never + // the close flag), so a fingerprint read after an exit-and-recycle race + // is discarded and the Windows terminator refuses to signal. + if (child.pid === undefined) throw new Error("a running child has no pid to terminate"); + const identity = await captureSpawnGenerationIdentity(child, createPlatformProbe()); + return terminateProcessTree(child.pid, { expectedRootFingerprint: identity?.fingerprint ?? null, killWaitMs: 5_000 }); + }, +): Promise { + const failures: string[] = []; + await Promise.all(tracked.map(async ({ child, closed }) => { + const running = child.exitCode === null && child.signalCode === null; + if (running) { + if (child.pid === undefined) { + failures.push("a running child has no pid to terminate"); + } else { + try { await terminate(child); } + catch (error) { failures.push(`terminating pid ${child.pid} failed: ${error instanceof Error ? error.message : String(error)}`); } + } + } + let closedInTime = false; + const deadline = new Promise<"deadline">((resolve) => { + const timer = setTimeout(() => resolve("deadline"), 5_000); + if (typeof timer.unref === "function") timer.unref(); + }); + const winner = await Promise.race([closed.then(() => "closed" as const), deadline]); + if (winner === "closed") closedInTime = true; + if (!closedInTime) failures.push(`pid ${child.pid ?? "?"} did not report close within 5s of reaping`); + else await closed; // drain the promise so a rejection can never surface later + })); + if (failures.length > 0) throw new Error(`child reap failed: ${failures.join("; ")}`); +} + +test("real cross-process contention never lets a lock error escape and the lock stays exclusive", { timeout: 90_000 }, async () => { + const root = mkdtempSync(path.join(tmpdir(), "bridge-lock-contention-")); + const tracked: TrackedChild[] = []; + try { + const lock = path.join(root, "state.lock"); + const ledger = path.join(root, "holders.log"); + const script = path.join(root, "contend.mjs"); + writeFileSync(script, [ + "const fs = await import(\"node:fs\");", + `const { tryAcquireProcessLock } = await import(${JSON.stringify(new URL("../src/store/process-lock.js", import.meta.url).href)});`, + `const lock = ${JSON.stringify(lock)};`, + `const ledger = ${JSON.stringify(ledger)};`, + "let escaped = 0; let acquired = 0; const sample = [];", + "for (let i = 0; i < 800; i++) {", + " try {", + " const release = tryAcquireProcessLock(lock);", + " if (release) {", + " acquired += 1;", + // The ledger append happens INSIDE the critical section: if the lock + // ever admits two owners, a \"+\" row appears before the previous + // holder's \"-\" and the prefix count exceeds one. + " fs.appendFileSync(ledger, `+${process.pid}\\n`);", + " fs.appendFileSync(ledger, `-${process.pid}\\n`);", + " release();", + " }", + " } catch (error) { escaped += 1; if (sample.length < 3) sample.push({ code: error?.code, message: String(error?.message) }); }", + "}", + "process.stdout.write(JSON.stringify({ pid: process.pid, escaped, acquired, sample }));", + ].join("\n")); + tracked.push(...[1, 2, 3].map(() => spawnTracked(script))); + const summaries = []; + for (const tracker of tracked) { + assert.equal(await tracker.closed, 0, `a contention prober crashed: ${tracker.stdout().slice(0, 400)}`); + summaries.push(JSON.parse(tracker.stdout())); + } + for (const summary of summaries) { + assert.equal(summary.escaped, 0, `lock errors must never escape contention: ${JSON.stringify(summary.sample)}`); + } + // Fairness is not a contract of tryAcquireProcessLock: no prober is + // guaranteed to win within a fixed iteration count. What the contract + // does promise is exclusivity whenever wins happen, verified by the + // ledger below plus meaningful total activity. + const totalAcquired = summaries.reduce((sum: number, summary: { acquired?: number }) => sum + (summary.acquired ?? 0), 0); + assert.ok(totalAcquired >= 3, `contention must exercise the critical section (total wins: ${totalAcquired})`); + let concurrentHolders = 0; + let maxConcurrentHolders = 0; + let ledgerRows = 0; + for (const row of readFileSync(ledger, "utf8").split("\n")) { + if (!row) continue; + ledgerRows += 1; + concurrentHolders += row.startsWith("+") ? 1 : -1; + maxConcurrentHolders = Math.max(maxConcurrentHolders, concurrentHolders); + } + assert.equal(ledgerRows, totalAcquired * 2, "every win must be ledgered exactly once (+ and -)"); + assert.equal(maxConcurrentHolders, 1, "the lock must admit at most one owner at a time"); + assert.equal(existsSync(lock), false, "every winner released; the shared lock must be withdrawn at the end"); + } finally { + await reapTrackedChildren(tracked); + rmSync(root, { recursive: true, force: true, maxRetries: 10, retryDelay: 50 }); + } +}); + +test("a crashing or hung prober is reaped before cleanup leaves no residue", { timeout: 30_000 }, async () => { + const root = mkdtempSync(path.join(tmpdir(), "bridge-lock-reap-")); + const tracked: TrackedChild[] = []; + try { + const crashScript = path.join(root, "crash.mjs"); + const hangScript = path.join(root, "hang.mjs"); + writeFileSync(crashScript, "process.exit(7);\n"); + writeFileSync(hangScript, "setInterval(() => undefined, 1_000);\n"); + tracked.push(spawnTracked(crashScript)); + const crash = tracked[0]!; + tracked.push(spawnTracked(hangScript)); + const hang = tracked[1]!; + assert.equal(await crash.closed, 7, "the sacrificial crasher must fail first"); + const hungStillRunning = hang.child.exitCode === null && hang.child.signalCode === null; + assert.equal(hungStillRunning, true, "the hanger must still be alive to exercise the reap path"); + await reapTrackedChildren(tracked); + // Closed means the close event fired; per the Node 22.18 docs a POSIX + // signal exit legitimately leaves exitCode null while signalCode is set, + // so either combination proves termination and closing. + const hangClosed = hang.child.exitCode !== null || hang.child.signalCode !== null; + assert.equal(hangClosed, true, "the hung child must have been terminated and closed (exitCode xor signalCode set)"); + } finally { + await reapTrackedChildren(tracked); + rmSync(root, { recursive: true, force: true, maxRetries: 10, retryDelay: 50 }); + assert.equal(existsSync(root), false, "cleanup only after every child of this test has closed"); + } +}); + +test("a failing terminator or a child that never closes makes the reap fail visibly before any cleanup", { timeout: 30_000 }, async () => { + const root = mkdtempSync(path.join(tmpdir(), "bridge-lock-reap-fail-")); + const tracked: TrackedChild[] = []; + let hang: TrackedChild | null = null; + try { + const hangScript = path.join(root, "hang.mjs"); + writeFileSync(hangScript, "setInterval(() => undefined, 1_000);\n"); + hang = spawnTracked(hangScript); + tracked.push(hang); + // Branch 1: the terminator itself fails — the reap must throw, and the + // caller's cleanup must not have deleted the tree under the live child. + await assert.rejects( + reapTrackedChildren(tracked, async () => { throw new Error("terminate refused (injected)"); }), + /terminating pid .* failed: terminate refused/, + ); + assert.equal(hang.child.exitCode === null && hang.child.signalCode === null, true, "the child is still alive after the failed reap"); + assert.equal(existsSync(root), true, "no cleanup may run while this test's child is still alive"); + // Branch 2: the terminator "succeeds" without killing anything — the + // missing close event must surface as a bounded, visible reap failure. + await assert.rejects( + reapTrackedChildren(tracked, async () => undefined), + /did not report close within 5s/, + ); + assert.equal(existsSync(root), true, "still no cleanup under a live child"); + } finally { + // Real reaping so this test never leaks its own child. + await reapTrackedChildren(tracked); + rmSync(root, { recursive: true, force: true, maxRetries: 10, retryDelay: 50 }); + } +}); + +test("an existing lock path is never renamed onto, so a live owner sees plain contention without publication attempts", { timeout: 15_000 }, () => { + const root = mkdtempSync(path.join(tmpdir(), "bridge-lock-liveowner-")); + try { + const lock = path.join(root, "state.lock"); + const owner = tryAcquireProcessLock(lock); + assert.ok(owner); + let renameAttempts = 0; + withPatchedRenameSync((_source, target) => { + if (target === lock) renameAttempts += 1; + return null; + }, () => { + assert.equal(tryAcquireProcessLock(lock), null, "contention with a live owner reports busy"); + }); + assert.equal(renameAttempts, 0, "publication must not even attempt a rename onto an existing lock path"); + const ownerRecord = JSON.parse(readFileSync(path.join(lock, "owner.json"), "utf8")); + assert.equal(ownerRecord.pid, process.pid, "the live owner's lock stays complete"); + owner(); + assert.equal(existsSync(lock), false); + // With the path genuinely free again, a non-transient publication error + // still throws instead of being reported as busy. + assert.throws( + () => withPatchedRenameSync((_source, target) => (target === lock ? errnoError("EIO") : null), () => { + tryAcquireProcessLock(lock); + }), + (error: NodeJS.ErrnoException) => error.code === "EIO", + "an I/O error during publication onto a free path must stay visible", + ); + } finally { rmSync(root, { recursive: true, force: true, maxRetries: 10, retryDelay: 50 }); } +}); + +test("an existing empty, foreign, or non-directory lock path is never silently taken over (Linux overwrite guard)", { timeout: 15_000 }, () => { + // Rootless Linux reproduced (2026-10-17 review): POSIX rename replaces an + // existing EMPTY directory, so the old publication path consumed an unknown + // owner's empty lock. The pre-publication existence check must route every + // existing shape — empty dir, foreign content, plain file — through the + // acquireExistingLock policies and leave it byte-for-byte untouched. + const root = mkdtempSync(path.join(tmpdir(), "bridge-lock-unknown-")); + try { + const lock = path.join(root, "state.lock"); + mkdirSync(lock, { recursive: true }); + assert.equal(tryAcquireProcessLock(lock), null, "an empty unknown lock must block, not be replaced"); + assert.deepEqual(readdirSync(lock), [], "the empty lock directory stays untouched"); + writeFileSync(path.join(lock, "foreign.txt"), "unknown owner marker"); + assert.equal(tryAcquireProcessLock(lock), null, "a foreign unknown lock must block"); + assert.deepEqual(readdirSync(lock), ["foreign.txt"], "foreign content stays and no owner record is added"); + } finally { rmSync(root, { recursive: true, force: true, maxRetries: 10, retryDelay: 50 }); } + const fileRoot = mkdtempSync(path.join(tmpdir(), "bridge-lock-unknown-file-")); + try { + const fileLock = path.join(fileRoot, "state.lock"); + writeFileSync(fileLock, "not a directory"); + assert.equal(tryAcquireProcessLock(fileLock), null, "a plain file at the lock path must block"); + assert.equal(readFileSync(fileLock, "utf8"), "not a directory", "the file stays untouched"); + } finally { rmSync(fileRoot, { recursive: true, force: true, maxRetries: 10, retryDelay: 50 }); } +}); + +test("an owner turning live during the guarded recheck leaves no reclaim guard behind", { timeout: 15_000 }, () => { + const root = mkdtempSync(path.join(tmpdir(), "bridge-lock-relive-")); + try { + const lock = path.join(root, "state.lock"); + mkdirSync(lock, { recursive: true }); + // This record names THIS process, but the first liveness probe is forced + // to ESRCH so the recheck path runs; the guarded second probe then sees + // the owner live again — the deterministic reproduction of the stranded + // guard from the attempt-2 review. + writeFileSync(path.join(lock, "owner.json"), JSON.stringify({ pid: process.pid, token: "reliving", started_at: new Date().toISOString() })); + withSequencedKill(process.pid, ["esrch", "alive"], () => { + assert.equal(tryAcquireProcessLock(lock), null, "a re-living owner means plain contention"); + }); + assert.equal(existsSync(`${lock}.reclaim`), false, "the recheck exit must remove the shared guard"); + const ownerRecord = JSON.parse(readFileSync(path.join(lock, "owner.json"), "utf8")); + assert.equal(ownerRecord.pid, process.pid, "the re-living owner's lock stays complete"); + } finally { rmSync(root, { recursive: true, force: true, maxRetries: 10, retryDelay: 50 }); } +}); + +test("an owner record that turns invalid inside the guarded recheck blocks without stranding the guard", { timeout: 15_000 }, () => { + const root = mkdtempSync(path.join(tmpdir(), "bridge-lock-reinvalid-")); + try { + assertDeadPidFixture(); + const lock = path.join(root, "state.lock"); + seedDeadOwner(lock); + assertDeadPidFixture(); + const ownerPath = path.join(lock, "owner.json"); + withSequencedOwnerReads(ownerPath, [Buffer.from(JSON.stringify({ pid: IMPOSSIBLE_DEAD_PID, token: "t" })), Buffer.from(JSON.stringify({ pid: 0 }))], () => { + assert.equal(tryAcquireProcessLock(lock), null, "an invalid record at recheck means busy, never a reclaim"); + }); + assert.equal(existsSync(`${lock}.reclaim`), false, "the invalid-record exit must remove the shared guard"); + assert.equal(existsSync(lock), true, "the unknown lock is preserved for explicit diagnosis"); + } finally { rmSync(root, { recursive: true, force: true, maxRetries: 10, retryDelay: 50 }); } +}); + +test("a lock vanishing at recheck returns to contention, and a guard cleanup failure there stays visible", { timeout: 15_000 }, () => { + const root = mkdtempSync(path.join(tmpdir(), "bridge-lock-renotempty-")); + try { + assertDeadPidFixture(); + const lock = path.join(root, "state.lock"); + const vanished = path.join(root, "vanished.lock"); + seedDeadOwner(vanished); + const ownerPath = path.join(vanished, "owner.json"); + // Read #1 (acquireExistingLock) sees the dead owner; read #2 (guarded + // recheck) hits ENOENT because the lock vanished — a normal race. + withSequencedOwnerReads(ownerPath, [readFileSync(ownerPath), errnoError("ENOENT")], () => { + assert.equal(tryAcquireProcessLock(vanished), null, "a vanished lock is a lost race, not an error"); + }); + assert.equal(existsSync(`${vanished}.reclaim`), false, "the ENOENT exit removes its guard"); + + // Same race, but the shared guard's cleanup fails: the failure must not + // be silently swallowed just to return the busy result. + seedDeadOwner(lock); + assertDeadPidFixture(); + const secondOwnerPath = path.join(lock, "owner.json"); + withPatchedRmdirSync((target) => (target.endsWith(".reclaim") ? errnoError("EIO") : null), () => { + withSequencedOwnerReads(secondOwnerPath, [readFileSync(secondOwnerPath), errnoError("ENOENT")], () => { + assert.throws(() => tryAcquireProcessLock(lock), /reclaim guard/u, + "a guard cleanup failure must surface even when the reclaim itself raced to nothing"); + }); + }); + assert.equal(existsSync(`${lock}.reclaim`), true, "the stuck guard stays diagnosable"); + } finally { rmSync(root, { recursive: true, force: true, maxRetries: 10, retryDelay: 50 }); } +}); + +test("dead-owner reclaim with a non-removable shared guard fails visibly and diagnosably", { timeout: 15_000 }, async () => { + const root = mkdtempSync(path.join(tmpdir(), "bridge-lock-guard-eio-")); + try { + const lock = path.join(root, "state.lock"); + seedDeadOwner(lock); + assertDeadPidFixture(); + let guardAttempts = 0; + assert.throws( + () => withPatchedRmdirSync((target) => { + if (target.endsWith(".reclaim")) { guardAttempts += 1; return errnoError("EIO"); } + return null; + }, () => { + tryAcquireProcessLock(lock); + }), + (error: Error) => { + assert.match(error.message, /reclaim guard/u, "the failure must name the shared guard"); + assert.match(error.message, /state\.lock\.reclaim/u, "the failure must spell out the diagnosable path"); + return true; + }, + "a reclaim reported successful while the guard survives must not stand", + ); + assert.ok(guardAttempts >= 1, "the injected guard failure must have occurred"); + assert.equal(existsSync(lock), false, "the dead owner's lock itself was still retired"); + assert.equal(existsSync(`${lock}.reclaim`), true, "the stuck guard remains visible for manual diagnosis"); + // Recovery: once the guard is manually removed, acquisition works again. + rmSync(`${lock}.reclaim`, { recursive: true, force: true, maxRetries: 10, retryDelay: 50 }); + const release = tryAcquireProcessLock(lock); + assert.ok(release, "acquisition recovers after manual guard removal"); + release(); + } finally { rmSync(root, { recursive: true, force: true, maxRetries: 10, retryDelay: 50 }); } +}); + +test("a transient guard cleanup failure is retried and the reclaim completes cleanly", { timeout: 15_000 }, async () => { + const root = mkdtempSync(path.join(tmpdir(), "bridge-lock-guard-transient-")); + try { + const lock = path.join(root, "state.lock"); + seedDeadOwner(lock); + assertDeadPidFixture(); + let guardFailuresLeft = 2; + const acquired: { release: (() => void) | null } = { release: null }; + withPatchedRmdirSync((target) => { + if (target.endsWith(".reclaim") && guardFailuresLeft > 0) { guardFailuresLeft -= 1; return errnoError("EBUSY"); } + return null; + }, () => { + acquired.release = tryAcquireProcessLock(lock); + }); + assert.ok(acquired.release, "the reclaim must succeed after transient guard retries"); + assert.equal(existsSync(`${lock}.reclaim`), false, "no guard residue"); + acquired.release(); + assert.deepEqual(readdirSync(root), [], "no lock residue of any kind after release"); + } finally { rmSync(root, { recursive: true, force: true, maxRetries: 10, retryDelay: 50 }); } +}); + +test("guard cleanup budget exhaustion stays visible instead of spinning", { timeout: 15_000 }, async () => { + const root = mkdtempSync(path.join(tmpdir(), "bridge-lock-guard-budget-")); + try { + const lock = path.join(root, "state.lock"); + seedDeadOwner(lock); + assertDeadPidFixture(); + const startedAt = Date.now(); + assert.throws( + () => withPatchedRmdirSync((target) => (target.endsWith(".reclaim") ? errnoError("EBUSY") : null), () => { + tryAcquireProcessLock(lock); + }), + /reclaim guard/u, + "budget exhaustion must surface the guard failure", + ); + assert.ok(Date.now() - startedAt < 5_000, "the retry budget must be bounded"); + assert.equal(existsSync(`${lock}.reclaim`), true, "the stuck guard stays diagnosable"); + } finally { rmSync(root, { recursive: true, force: true, maxRetries: 10, retryDelay: 50 }); } +}); + +test("a transiently blocked owner.json unlink inside retirement leaves no residue", { timeout: 15_000 }, () => { + const root = mkdtempSync(path.join(tmpdir(), "bridge-lock-unlink-")); + try { + const lock = path.join(root, "state.lock"); + const release = tryAcquireProcessLock(lock); + assert.ok(release); + let unlinkFailedOnce = false; + withPatchedUnlinkSync((target) => { + if (target.endsWith("owner.json") && target.includes(".retired") && !unlinkFailedOnce) { + unlinkFailedOnce = true; + return errnoError("EPERM"); + } + return null; + }, () => { + release(); + }); + assert.equal(unlinkFailedOnce, true, "the injected unlink failure must have occurred"); + assert.equal(existsSync(lock), false, "the lock itself must be withdrawn"); + assert.equal(readdirSync(root).filter((name) => name.includes("state.lock")).length, 0, "no retired or staging residue may survive release"); + } finally { rmSync(root, { recursive: true, force: true, maxRetries: 10, retryDelay: 50 }); } +}); test("crashing during lock retirement cannot strand an ownerless shared lock", { timeout: 15_000 }, () => { const root = mkdtempSync(path.join(tmpdir(), "bridge-lock-retire-")); diff --git a/test/process-spawn.test.ts b/test/process-spawn.test.ts index 982d61a..801a9b6 100644 --- a/test/process-spawn.test.ts +++ b/test/process-spawn.test.ts @@ -1,56 +1,293 @@ // Real (local, model-free) process-tree termination test for Windows. Spawns a // Node parent that spawns its own sleeper child, then verifies that // terminateProcessTree kills the entire verified tree. Skipped on non-Windows. +// Fixture lifecycle (2026-10-17 matrix2 review): the production call may +// refuse (it did, with a malformed-identity snapshot rejection) — the test +// must never orphan its own children in that case. BOTH fixture scripts +// independently watch the same private release file (triggered only in the +// finally, after the primary assertions or their failure — never before the +// production call, so it cannot mask production behavior); the parent also +// stops its OWN child handle, and only a real close event on that handle +// counts as confirmation. The finally gate confirms every owned pid exited +// before removing the tree, and an unreported grandchild preserves the tree. +// Before the production termination call, a read-only ownership precondition +// verifies the selected tree belongs to this fixture (the matrix2 incident +// selected stale system rows that merely shared a ParentProcessId); any +// unrelated, older, or unknown entry fails visibly BEFORE any production +// signal is sent. import assert from "node:assert/strict"; -import { spawn } from "node:child_process"; -import { mkdtempSync, rmSync, writeFileSync } from "node:fs"; +import { spawn, spawnSync, type ChildProcess } from "node:child_process"; +import { existsSync, mkdtempSync, writeFileSync } from "node:fs"; import { tmpdir } from "node:os"; import path from "node:path"; import test from "node:test"; -import { isProcessRunning, terminateProcessTree } from "../src/adapters/process-spawn.js"; +import { captureSpawnGenerationIdentity, createPlatformProbe, isProcessRunning, terminateProcessTree } from "../src/adapters/process-spawn.js"; +import { cleanupOwnedProcessFixture } from "./helpers.js"; + +function writeFixtureScripts(dir: string): { parentScript: string; childScript: string; releasePath: string } { + const parentScript = path.join(dir, "parent.cjs"); + const childScript = path.join(dir, "sleeper.cjs"); + const releasePath = path.join(dir, "release"); + writeFileSync(childScript, [ + '"use strict";', + "const fs = require('node:fs');", + `const release = ${JSON.stringify(releasePath)};`, + "// Independent cooperative release: this sleeper exits on the fixture's", + "// private release file even when its parent is already gone.", + "function watch() { if (fs.existsSync(release)) process.exit(0); setTimeout(watch, 50); }", + "watch();", + "setInterval(() => {}, 1000);", + ].join("\n"), "utf8"); + writeFileSync(parentScript, [ + '"use strict";', + "const fs = require('node:fs');", + "const { spawn } = require('node:child_process');", + `const childScript = ${JSON.stringify(childScript)};`, + `const release = ${JSON.stringify(releasePath)};`, + "const child = spawn(process.execPath, [childScript], { stdio: 'ignore' });", + "let childClosed = false;", + "let childError = null;", + "child.once('close', () => { childClosed = true; });", + "child.once('error', (error) => { childError = error; });", + "console.log(`${child.pid}:${child.pid > 0 ? 'ok' : 'bad'}`);", + "// Cooperative release for the parent: stop OUR OWN child handle and", + "// exit only after its real close event. A spawn error is reported and", + "// exits nonzero — it never counts as a confirmed close.", + "function watchRelease() {", + " if (fs.existsSync(release)) {", + " if (childError !== null) { process.exit(1); return; }", + " if (!childClosed) {", + " try { child.kill(); } catch { /* the bounded close wait decides */ }", + " const deadline = Date.now() + 5000;", + " (function awaitClose() {", + " if (childClosed) { process.exit(0); return; }", + " if (childError !== null || Date.now() > deadline) { process.exit(1); return; }", + " setTimeout(awaitClose, 20);", + " })();", + " return;", + " }", + " process.exit(0);", + " return;", + " }", + " setTimeout(watchRelease, 50);", + "}", + "watchRelease();", + "setInterval(() => {}, 1000);", + ].join("\n"), "utf8"); + return { parentScript, childScript, releasePath }; +} + +/** Spawns the fixture parent and returns the handle and pid IMMEDIATELY; the + * pid receipt is a separate promise with error/close registered at spawn, so + * a receipt failure still leaves the caller holding the owned parent pid. */ +function spawnFixtureParent(parentScript: string): { parent: ChildProcess; parentPid: number; receipt: Promise } { + const parent = spawn(process.execPath, [parentScript], { + shell: false, + stdio: ["ignore", "pipe", "ignore"], + windowsHide: true, + }); + const parentPid = parent.pid ?? 0; + const receipt = new Promise((resolve, reject) => { + let buffer = ""; + const timer = setTimeout(() => reject(new Error("parent did not report a complete child pid receipt")), 10_000); + if (typeof timer.unref === "function") timer.unref(); + parent.stdout?.setEncoding("utf8"); + parent.stdout?.on("data", (chunk: string) => { + buffer += chunk; + const newline = buffer.indexOf("\n"); + if (newline < 0) return; // wait for the complete receipt line + const [pidText, tag] = buffer.slice(0, newline).trim().split(":"); + buffer = buffer.slice(newline + 1); + const pid = Number.parseInt(pidText ?? "", 10); + if (tag === "ok" && Number.isSafeInteger(pid) && pid > 0) { + clearTimeout(timer); + resolve(pid); + } else { + clearTimeout(timer); + reject(new Error(`parent reported an unusable child pid receipt: ${JSON.stringify(pidText ?? "")}`)); + } + }); + parent.once("error", (error) => { clearTimeout(timer); reject(error); }); + parent.once("close", () => { clearTimeout(timer); reject(new Error("parent exited before reporting its child pid")); }); + }); + return { parent, parentPid, receipt }; +} + +interface TreeRow { + pid: number; + parentPid: number; + name: string; + creationFileTime: bigint; + commandLine: string; +} + +/** Read-only fresh snapshot of the PID closure under rootPid: no signalling, + * purely CIM enumeration. Base64 keeps the pipe protocol unambiguous; the + * FILETIME is compared as an exact BigInt, with no clock skew allowance. */ +function snapshotTreeClosure(rootPid: number): TreeRow[] { + const script = [ + "$ErrorActionPreference = 'Stop'", + "$all = @(Get-CimInstance Win32_Process -ErrorAction Stop)", + `$selected = [System.Collections.Generic.HashSet[int]]::new(); [void]$selected.Add(${rootPid})`, + "do { $changed = $false; foreach ($item in $all) { if ($selected.Contains([int]$item.ParentProcessId)) { if ($selected.Add([int]$item.ProcessId)) { $changed = $true } } } } while ($changed)", + "'bridge_tree_owned_v1'", + "foreach ($item in $all) { if ($selected.Contains([int]$item.ProcessId)) { $cmd = [Convert]::ToBase64String([Text.Encoding]::UTF8.GetBytes([string]$item.CommandLine)); '{0}|{1}|{2}|{3}|{4}' -f $item.ProcessId, $item.ParentProcessId, $item.Name, $item.CreationDate.ToFileTimeUtc(), $cmd } }", + ].join("; "); + const run = spawnSync("powershell", ["-NoProfile", "-Command", script], { encoding: "utf8", timeout: 20_000, windowsHide: true }); + if (run.status !== 0) throw new Error(`ownership snapshot failed (exit ${String(run.status)}): ${String(run.stderr).slice(0, 300)}`); + const lines = run.stdout.trim().split(/\r?\n/u); + if (lines.shift() !== "bridge_tree_owned_v1") throw new Error("ownership snapshot failed: protocol marker missing"); + return lines.filter((line) => line.trim()).map((line) => { + const [pidText, parentText, name, fileTimeText, commandB64] = line.split("|"); + const pid = Number(pidText); + const parentPid = Number(parentText); + if (!Number.isSafeInteger(pid) || pid <= 0 || !Number.isSafeInteger(parentPid) || !name || !/^\d+$/u.test(fileTimeText ?? "")) { + throw new Error(`ownership snapshot failed: malformed row ${JSON.stringify(line.slice(0, 120))}`); + } + return { pid, parentPid, name, creationFileTime: BigInt(fileTimeText!), commandLine: Buffer.from(commandB64 ?? "", "base64").toString("utf8") }; + }); +} + +/** + * Fixture ownership precondition (2026-10-17 matrix2 Safety addendum): every + * process selected under the test's root must belong to THIS fixture. Node + * rows must be exactly the recorded parent/grandchild pids whose command + * lines reference this run's unique fixture scripts; the sleeper's parent is + * the root and its creation cannot predate the root's; console hosts hang + * directly off those verified pids and cannot predate their own parent. All + * creation-time comparisons are exact BigInt checks with no skew allowance, + * so a stale system row that merely shares a ParentProcessId fails HERE, + * before the production terminator is ever invoked. This is a fixture guard, + * not a production policy; it never skips, fabricates, or re-picks pids. + */ +function assertTreeBelongsToFixture(rootPid: number, grandchildPid: number, parentScript: string, childScript: string): void { + const rows = snapshotTreeClosure(rootPid); + const byPid = new Map(rows.map((row) => [row.pid, row])); + const root = byPid.get(rootPid); + if (!root) throw new Error(`ownership precondition failed: root pid ${String(rootPid)} missing from its own snapshot`); + const sleeper = byPid.get(grandchildPid); + if (!sleeper) throw new Error(`ownership precondition failed: expected fixture sleeper pid ${String(grandchildPid)} not selected`); + if (sleeper.parentPid !== rootPid) throw new Error(`ownership precondition failed: sleeper parent pid ${String(sleeper.parentPid)} is not the fixture root`); + if (sleeper.creationFileTime < root.creationFileTime) { + throw new Error(`ownership precondition failed: sleeper pid ${String(grandchildPid)} predates the fixture root; refusing to signal an unrelated process`); + } + if (root.name.toLowerCase() !== "node.exe" || !root.commandLine.includes(parentScript)) { + throw new Error("ownership precondition failed: the fixture root is not this test's node parent"); + } + if (sleeper.name.toLowerCase() !== "node.exe" || !sleeper.commandLine.includes(childScript)) { + throw new Error("ownership precondition failed: the recorded sleeper is not this test's node child"); + } + const allowedImages = new Set(["node.exe", "conhost.exe", "openconsole.exe"]); + for (const row of rows) { + if (!allowedImages.has(row.name.toLowerCase())) { + throw new Error(`ownership precondition failed: pid ${String(row.pid)} has unexpected image ${row.name}; refusing to signal an unknown process`); + } + if (row.name.toLowerCase() === "node.exe" && row.pid !== rootPid && row.pid !== grandchildPid) { + throw new Error(`ownership precondition failed: unexpected node pid ${String(row.pid)} inside the fixture tree`); + } + if (row.name.toLowerCase() !== "node.exe") { + const hostParent = row.parentPid === rootPid ? root : row.parentPid === grandchildPid ? sleeper : null; + if (!hostParent) throw new Error(`ownership precondition failed: console host pid ${String(row.pid)} does not hang off the verified fixture pids`); + if (row.creationFileTime < hostParent.creationFileTime) { + throw new Error(`ownership precondition failed: console host pid ${String(row.pid)} predates its parent; refusing to signal an unrelated process`); + } + } + } +} test("terminateProcessTree kills a real Windows process tree and verifies it", { timeout: 30_000 }, async () => { if (process.platform !== "win32") { return test.skip("Windows-only process tree semantics"); } const dir = mkdtempSync(path.join(tmpdir(), "zcode-bridge-tree-")); + const { parentScript, childScript, releasePath } = writeFixtureScripts(dir); + const spawned = spawnFixtureParent(parentScript); + const parentPid = spawned.parentPid; + let grandchildPid = 0; + let primaryFailed = false; try { - const parentScript = path.join(dir, "parent.cjs"); - writeFileSync( - parentScript, - [ - '"use strict";', - "const { spawn } = require('node:child_process');", - 'const child = spawn(process.execPath, ["-e", "setInterval(() => {}, 1000)"], { stdio: "ignore" });', - "console.log(String(child.pid));", - "setInterval(() => {}, 1000);", - ].join("\n"), - "utf8", - ); - const parent = spawn(process.execPath, [parentScript], { - shell: false, - stdio: ["ignore", "pipe", "ignore"], - windowsHide: true, - }); - const grandchildPid = await new Promise((resolve, reject) => { - parent.stdout?.setEncoding("utf8"); - parent.stdout?.on("data", (chunk: string) => { - const pid = Number.parseInt(chunk.trim(), 10); - if (Number.isInteger(pid) && pid > 0) resolve(pid); - }); - parent.on("error", reject); - setTimeout(() => reject(new Error("parent did not report its child pid")), 10_000); - }); - assert.ok(parent.pid); - assert.ok(isProcessRunning(parent.pid), "parent must be alive before termination"); + grandchildPid = await spawned.receipt; + assert.ok(isProcessRunning(parentPid), "parent must be alive before termination"); assert.ok(isProcessRunning(grandchildPid), "grandchild must be alive before termination"); - const result = await terminateProcessTree(parent.pid, { graceMs: 500, killWaitMs: 5_000 }); + // Read-only ownership guard BEFORE the production terminator runs. + assertTreeBelongsToFixture(parentPid, grandchildPid, parentScript, childScript); + + // ADR-009 identity handoff through the ORIGINAL ChildProcess object: the + // shared capture re-verifies the spawn generation after every await + // (Node delivers exit before close and libuv releases the native handle + // at exit delivery — a close flag would prove nothing). A capture that + // races the parent's exit is discarded and the terminator then refuses + // instead of trusting the bare PID. + const identity = await captureSpawnGenerationIdentity(spawned.parent, createPlatformProbe()); + assert.equal(identity !== null, true, "the fixture parent identity must be readable on its original generation"); + const result = await terminateProcessTree(parentPid, { graceMs: 500, killWaitMs: 8_000, expectedRootFingerprint: identity!.fingerprint }); assert.equal(result.verified, true); - assert.equal(isProcessRunning(parent.pid), false, "parent must be dead after termination"); + // The receipt preserves every verified member's creation time/identity + // (AC4) and the signaled set equals the verified member set (AC2). + assert.ok(result.windows, "the Windows receipt is present"); + assert.ok(result.windows!.members.some((m) => m.pid === parentPid), "the root is in the verified member set"); + assert.ok(result.windows!.members.some((m) => m.pid === grandchildPid), "the grandchild is in the verified member set"); + assert.ok(result.windows!.signaled.length > 0, "the receipt records which members were signalled"); + for (const pid of result.windows!.signaled) { + assert.ok(result.windows!.members.some((m) => m.pid === pid), "signals only reached verified members"); + } + assert.equal(isProcessRunning(parentPid), false, "parent must be dead after termination"); assert.equal(isProcessRunning(grandchildPid), false, "grandchild must be dead after termination"); + } catch (error) { + primaryFailed = true; + throw error; + } finally { + if (parentPid > 0) { + const pids = grandchildPid > 0 ? [parentPid, grandchildPid] : [parentPid]; + await cleanupOwnedProcessFixture({ pids, releasePath, dir, primaryFailed, preserveTree: grandchildPid === 0 }); + } + } +}); + +test("terminateProcessTree with a wrong expected root fingerprint refuses and signals nothing", { timeout: 30_000 }, async () => { + // ADR-009 rejection path on a real, registered, independently-bounded + // fixture: a mismatched expectation means the PID may be owned by another + // generation, so the terminator must refuse BEFORE any signal and the tree + // must stay alive. The bounded fixture release then cleans up without any + // tree signalling. + if (process.platform !== "win32") { + return test.skip("Windows-only process tree semantics"); + } + const dir = mkdtempSync(path.join(tmpdir(), "zcode-bridge-tree-refuse-")); + const { parentScript, childScript, releasePath } = writeFixtureScripts(dir); + const spawned = spawnFixtureParent(parentScript); + const parentPid = spawned.parentPid; + let grandchildPid = 0; + let primaryFailed = false; + try { + grandchildPid = await spawned.receipt; + assert.ok(isProcessRunning(parentPid), "fixture parent starts alive"); + assert.ok(isProcessRunning(grandchildPid), "fixture sleeper starts alive"); + assertTreeBelongsToFixture(parentPid, grandchildPid, parentScript, childScript); + const identity = await captureSpawnGenerationIdentity(spawned.parent, createPlatformProbe()); + assert.equal(identity !== null, true, "the fixture parent identity must be readable on its original generation"); + const wrongFingerprint = identity!.fingerprint === "1" ? "2" : "1"; + await assert.rejects( + terminateProcessTree(parentPid, { graceMs: 500, killWaitMs: 2_000, expectedRootFingerprint: wrongFingerprint }), + /process_tree_root_identity_changed/, + ); + assert.equal(isProcessRunning(parentPid), true, "a refused termination leaves the tree alive"); + assert.equal(isProcessRunning(grandchildPid), true, "no descendant was signalled either"); + // Missing identity is refused the same way (bare-PID trust is forbidden). + await assert.rejects( + terminateProcessTree(parentPid, { graceMs: 500, killWaitMs: 2_000 }), + /process_tree_root_identity_required/, + ); + assert.equal(isProcessRunning(parentPid), true, "still alive after the identity-required refusal"); + } catch (error) { + primaryFailed = true; + throw error; } finally { - rmSync(dir, { recursive: true, force: true }); + if (parentPid > 0) { + const pids = grandchildPid > 0 ? [parentPid, grandchildPid] : [parentPid]; + await cleanupOwnedProcessFixture({ pids, releasePath, dir, primaryFailed, preserveTree: grandchildPid === 0 }); + } } }); @@ -58,3 +295,40 @@ test("terminateProcessTree rejects non-positive PIDs", async () => { await assert.rejects(terminateProcessTree(0), /positive integer PID/); await assert.rejects(terminateProcessTree(-5), /positive integer PID/); }); + +test("when termination never succeeds the fixture still leaves no owned orphans", { timeout: 30_000 }, async () => { + // Failure-path coverage for the same fixture family: if the production + // terminator refuses or is never reached (the matrix2 shape), the + // cooperative release path alone must stop BOTH known fixture processes + // and clean the tree — without any PID-tree signalling and without + // weakening the primary test above, whose assertions are unchanged. + const dir = mkdtempSync(path.join(tmpdir(), "zcode-bridge-tree-refuses-")); + const { parentScript, releasePath } = writeFixtureScripts(dir); + const spawned = spawnFixtureParent(parentScript); + const parentPid = spawned.parentPid; + let grandchildPid = 0; + let primaryFailed = false; + let settledByPrimaryCleanup = false; + try { + grandchildPid = await spawned.receipt; + assert.ok(parentPid > 0 && isProcessRunning(parentPid), "fixture parent starts alive"); + assert.ok(isProcessRunning(grandchildPid), "fixture sleeper starts alive"); + + // Deliberately NO production termination here: the gated fixture release + // must suffice (both scripts watch the private release file; the parent + // additionally stops its own child after a confirmed close event). + await cleanupOwnedProcessFixture({ pids: [parentPid, grandchildPid], releasePath, dir, primaryFailed: false, timeoutMs: 15_000 }); + settledByPrimaryCleanup = true; + assert.equal(isProcessRunning(parentPid), false, "the fixture parent exited via the cooperative release"); + assert.equal(isProcessRunning(grandchildPid), false, "the fixture sleeper exited via its own release watch"); + assert.equal(existsSync(dir), false, "the tree is removed only after both exits are confirmed"); + } catch (error) { + primaryFailed = true; + throw error; + } finally { + if (!settledByPrimaryCleanup && parentPid > 0) { + const pids = grandchildPid > 0 ? [parentPid, grandchildPid] : [parentPid]; + await cleanupOwnedProcessFixture({ pids, releasePath, dir, primaryFailed, preserveTree: grandchildPid === 0 }); + } + } +}); diff --git a/test/runtime-fixture.ts b/test/runtime-fixture.ts index d0783c5..21dc7aa 100644 --- a/test/runtime-fixture.ts +++ b/test/runtime-fixture.ts @@ -2,7 +2,7 @@ import { mkdtemp, mkdir, writeFile, rm } from "node:fs/promises"; import { tmpdir } from "node:os"; import path from "node:path"; import type { ZCodeRuntimeConfig } from "../src/interfaces.js"; -import { validReport } from "./helpers.js"; +import { TEMP_TREE_REMOVE_OPTIONS, validReport } from "./helpers.js"; export async function runtimeFixture(options: { interaction?: boolean; replay?: boolean; output?: string; fail?: boolean; cancel?: boolean } = {}) { const root = await mkdtemp(path.join(tmpdir(), "bridge-regression-")); @@ -39,5 +39,5 @@ export async function runtimeFixture(options: { interaction?: boolean; replay?: const env = { ...process.env, HOME: home, USERPROFILE: home }; for (const key of Object.keys(env)) if (key.startsWith("ZCODE_")) delete env[key as keyof typeof env]; Object.assign(env, { ZCODE_BRIDGE_NODE: process.execPath, ZCODE_BRIDGE_ZCODE_CJS: entrypoint, ZCODE_BUILTIN_PROVIDER_CONFIG_FILE: builtin, ZCODE_PERSONAL_PROVIDER_CONFIG_FILE: personal, ZCODE_HOME: path.join(home, ".zcode"), ZCODE_BRIDGE_DATA_DIR: root, ZCODE_BRIDGE_MODE: "build" }); - return { root, home, log, config, env, host: { name: "test-host-bridge", settingsDirectory }, cleanup: () => rm(root, { recursive: true, force: true }) }; + return { root, home, log, config, env, host: { name: "test-host-bridge", settingsDirectory }, cleanup: () => rm(root, { recursive: true, force: true, ...TEMP_TREE_REMOVE_OPTIONS }) }; } diff --git a/test/runtime-remediation.test.ts b/test/runtime-remediation.test.ts index 5f6db10..0d3c035 100644 --- a/test/runtime-remediation.test.ts +++ b/test/runtime-remediation.test.ts @@ -9,7 +9,7 @@ import { TaskStore } from "../src/store/task-store.js"; import { runWorkerTask } from "../src/worker/run-task.js"; import { BridgeTaskManager } from "../src/manager/task-manager.js"; import { DirectWorkspaceProvider } from "../src/workspace/direct-provider.js"; -import { terminateProcessTree } from "../src/adapters/process-spawn.js"; +import { captureSpawnGenerationIdentity, createPlatformProbe, terminateProcessTree } from "../src/adapters/process-spawn.js"; import { ZCodeModelSettings } from "../src/runtime/model-settings.js"; import { runtimeFixture } from "./runtime-fixture.js"; import { makeTask, makeWorkspace } from "./helpers.js"; @@ -99,11 +99,21 @@ test("a real worker delivers a permission reply and exits; a continuation gets a assert.equal(exit, 0, stderr); assert.equal(store.readResult("task_1")?.status, "completed"); priorId = id; - } finally { if (child.exitCode === null && child.signalCode === null && child.pid) await terminateProcessTree(child.pid); } + } finally { if (child.exitCode === null && child.signalCode === null && child.pid) await terminateFixtureChild(child); } } } finally { manager.dispose(); await fixture.cleanup(); } }); +/** ADR-009: holds the ORIGINAL ChildProcess. The shared capture re-verifies + * the spawn generation after every await (exit state, not the close flag), + * so a fingerprint read after an exit-and-recycle race is discarded and the + * Windows terminator then refuses to signal. */ +async function terminateFixtureChild(child: import("node:child_process").ChildProcess): Promise { + if (child.pid === undefined) return; + const identity = await captureSpawnGenerationIdentity(child, createPlatformProbe()); + await terminateProcessTree(child.pid, { expectedRootFingerprint: identity?.fingerprint ?? null, killWaitMs: 5_000 }); +} + test("real worker reaches its 60-second timeout and exits with an unanswered interaction", { timeout: 80_000 }, async () => { const fixture = await runtimeFixture({ interaction: true }); const store = new TaskStore(fixture.root); @@ -119,7 +129,7 @@ test("real worker reaches its 60-second timeout and exits with an unanswered int assert.equal(store.readStatus("task_1").worker_pid, null); assert.equal(store.readStatus("task_1").zcode_pid, null); } finally { - if (child.exitCode === null && child.signalCode === null && child.pid) await terminateProcessTree(child.pid); + if (child.exitCode === null && child.signalCode === null && child.pid) await terminateFixtureChild(child); await fixture.cleanup(); } }); diff --git a/test/spawn-generation-identity.test.ts b/test/spawn-generation-identity.test.ts new file mode 100644 index 0000000..b567cca --- /dev/null +++ b/test/spawn-generation-identity.test.ts @@ -0,0 +1,267 @@ +// ADR-009 deterministic spawn-generation continuity tests (live-test safety +// gate 1). All cases use fake event state and injected probes — no real +// processes, no signals. They prove the two safety properties the gate +// demanded: (1) continuity is decided by the child's EXIT state, not by the +// later stdio close (Node delivers exit first and libuv releases the native +// handle at exit delivery, so a close flag alone leaves an exit→close window +// in which the PID can be recycled); (2) a fingerprint that arrives after +// the child exited — i.e. possibly describing a recycled generation — is +// discarded and never stored as the spawn identity, and without a stored +// identity the Windows terminator refuses to signal. +import assert from "node:assert/strict"; +import test from "node:test"; +import { captureSpawnGenerationIdentity, childStillOnSpawnGeneration, type ProcessProbe, type TerminateProcessTree } from "../src/adapters/process-spawn.js"; +import { ZCodeAdapter } from "../src/adapters/zcode-adapter.js"; +import { FakeSpawn, makeTask, makeWorkspace, stubResolver, successEnvelope, validReport, makeTempDir, removeTempDir } from "./helpers.js"; + +interface FakeChild { + pid?: number; + exitCode: number | null; + signalCode: string | null; + exitSeen: boolean; +} + +function liveChild(pid = 4_321): FakeChild { + return { pid, exitCode: null, signalCode: null, exitSeen: false }; +} + +test("continuity: a child that has not exited is on its original spawn generation", () => { + assert.equal(childStillOnSpawnGeneration(liveChild()), true); + assert.equal(childStillOnSpawnGeneration({ ...liveChild(), exitSeen: false }), true); +}); + +test("continuity: exit evidence ends the provable generation even before any close event", () => { + // Node delivers 'exit' before stdio 'close': the close flag is still + // false here, yet the exit already released the native handle. + assert.equal(childStillOnSpawnGeneration({ ...liveChild(), exitCode: 0 }), false); + assert.equal(childStillOnSpawnGeneration({ ...liveChild(), signalCode: "SIGTERM" }), false); + assert.equal(childStillOnSpawnGeneration({ ...liveChild(), exitSeen: true }), false); + // No pid means no provable object at all. + assert.equal(childStillOnSpawnGeneration({ pid: undefined, exitCode: null, signalCode: null, exitSeen: false }), false); +}); + +test("capture: an identity that resolves after the child exited is discarded (recycled generation)", async () => { + const child = liveChild(); + const probe = { + // The probe's await straddles the exit: by the time the fingerprint is + // available, the child has exited — the fingerprint may describe whoever + // recycled the PID and must never be stored as the spawn identity. + identityOf: async () => { + child.exitCode = 0; + return { fingerprint: "recycled-generation-ticks", fingerprint_precision: "exact" }; + }, + }; + const identity = await captureSpawnGenerationIdentity(child, probe, { retryDelayMs: 1 }); + assert.equal(identity, null, "a post-exit fingerprint is never adopted"); +}); + +test("capture: an exit during the bounded retry discards the second read too", async () => { + const child = liveChild(); + let reads = 0; + const probe = { + identityOf: async () => { + reads += 1; + if (reads === 1) return { fingerprint: null }; + return { fingerprint: "late-ticks", fingerprint_precision: "exact" }; + }, + }; + const capture = captureSpawnGenerationIdentity(child, probe, { retryDelayMs: 5 }); + child.exitSeen = true; // exit fires while the retry delay is pending + assert.equal(await capture, null); + assert.equal(reads, 1, "the second read never ran after the exit"); +}); + +test("capture: a live child's fingerprint is adopted and returned", async () => { + const child = liveChild(9_999); + const probe = { identityOf: async () => ({ fingerprint: "original-ticks", fingerprint_precision: "exact" }) }; + const identity = await captureSpawnGenerationIdentity(child, probe, { retryDelayMs: 1 }); + assert.deepEqual(identity, { pid: 9_999, fingerprint: "original-ticks" }); +}); + +test("capture: a failed first read retries once on a still-live child", async () => { + const child = liveChild(); + let reads = 0; + const probe = { + identityOf: async () => { + reads += 1; + return { fingerprint: reads === 1 ? null : "second-read-ticks", fingerprint_precision: "exact" }; + }, + }; + const identity = await captureSpawnGenerationIdentity(child, probe, { retryDelayMs: 1 }); + assert.deepEqual(identity, { pid: child.pid, fingerprint: "second-read-ticks" }); +}); + +test("capture: a throwing or empty probe result never yields an identity", async () => { + const child = liveChild(); + assert.equal(await captureSpawnGenerationIdentity(child, { identityOf: async () => { throw new Error("query failed"); } }, { retryDelayMs: 1 }), null); + assert.equal(await captureSpawnGenerationIdentity(child, { identityOf: async () => ({ fingerprint: null }) }, { retryDelayMs: 1 }), null); + assert.equal(await captureSpawnGenerationIdentity({ pid: undefined, exitCode: null, signalCode: null, exitSeen: false }, { identityOf: async () => ({ fingerprint: "x" }) }, { retryDelayMs: 1 }), null); +}); + +test("continuity: missing exit state is unknown, never trusted continuity", () => { + // A fake/legacy child object without exitCode/signalCode cannot prove the + // generation; unknown fields must fail closed (gate 2: undefined must not + // be coerced into a trusted null). + assert.equal(childStillOnSpawnGeneration({ pid: 5, exitCode: undefined, signalCode: undefined, exitSeen: false }), false); + assert.equal(childStillOnSpawnGeneration({ pid: 5, exitCode: null, signalCode: undefined, exitSeen: false }), false); + assert.equal(childStillOnSpawnGeneration({ pid: 5, exitCode: undefined, signalCode: null, exitSeen: false }), false); +}); + +test("capture: a coarse-precision fingerprint is never adopted as a termination identity", async () => { + const child = liveChild(); + const probe = { identityOf: async () => ({ fingerprint: "coarse-lstart", fingerprint_precision: "coarse" }) }; + assert.equal(await captureSpawnGenerationIdentity(child, probe, { retryDelayMs: 1 }), null); +}); +test("capture: close flags are irrelevant to continuity — only exit state decides", async () => { + // A child whose stdio close flag would say "still open" but whose exit + // already happened must NOT pass continuity: exit-before-close ordering. + const child = liveChild(); + assert.equal(childStillOnSpawnGeneration(child), true); + const probe = { + identityOf: async () => { + child.exitCode = 1; // exit delivered; stdio close still pending + return { fingerprint: "whoever-owns-the-pid-now", fingerprint_precision: "exact" }; + }, + }; + assert.equal(await captureSpawnGenerationIdentity(child, probe, { retryDelayMs: 1 }), null); + assert.equal(childStillOnSpawnGeneration(child), false); +}); + +// -- Consumer-level regression: the real adapter call chain (gate 2) -------- +// +// Exercises the LEGACY ZCodeAdapter with its fake spawn infrastructure, an +// injected probe, and an injected terminator. This proves the shared capture +// and the exit-based guard are actually wired through the adapter's real +// spawn → identity capture → cancel → terminate path, not only correct in +// isolation. Fake events only; the injected terminator never signals. + +interface TerminateRecord { pid: number; expectedRootFingerprint: string | null } + +async function waitForSpawn(adapter: ZCodeAdapter, handle: Parameters[0]): Promise { + for (let i = 0; i < 500; i++) { + const status = await adapter.getStatus(handle); + if (status.zcodePid !== null) return; + await new Promise((resolve) => setTimeout(resolve, 5)); + } + throw new Error("the adapter never spawned a child process"); +} + +function scriptedProbe(behavior: (pid: number) => Promise<{ fingerprint: string | null; fingerprint_precision?: string | null }>): ProcessProbe { + return { + platform: process.platform, + identityOf: behavior as ProcessProbe["identityOf"], + selfIdentity: behavior.bind(null, process.pid) as ProcessProbe["selfIdentity"], + probe: async () => [], + }; +} + +test("consumer: a live child's exact identity is handed to the terminator through the adapter chain", { timeout: 20_000 }, async () => { + const fake = new FakeSpawn(); + fake.script.push(() => { + // The child stays alive until the close emitted in the finally cleanup. + }); + const terminations: TerminateRecord[] = []; + const terminate: TerminateProcessTree = async (pid, options) => { + terminations.push({ pid, expectedRootFingerprint: options?.expectedRootFingerprint ?? null }); + return { pid, signal: "SIGKILL", verified: true }; + }; + const promptDir = await makeTempDir("consumer-identity"); + const adapter = new ZCodeAdapter({ + resolver: stubResolver(), + spawnImpl: fake.spawnFn, + terminateProcessTreeImpl: terminate, + probe: scriptedProbe(async () => ({ fingerprint: "consumer-exact-ticks", fingerprint_precision: "exact" })), + promptTmpDir: promptDir, + childEnvBase: { PATH: "C:\Windows" }, + }); + try { + const handle = await adapter.startTask({ task: makeTask(), workspace: makeWorkspace(), attempt: 1 }); + await waitForSpawn(adapter, handle); + await adapter.cancelTask(handle); + if (process.platform === "win32") { + assert.equal(terminations.length, 1, "exactly one termination request on Windows"); + assert.equal(terminations[0]!.expectedRootFingerprint, "consumer-exact-ticks", "the captured exact identity is handed to the terminator"); + } else { + assert.equal(terminations.length, 1, "POSIX keeps its termination behaviour"); + assert.equal(terminations[0]!.expectedRootFingerprint, null, "POSIX does not consume the fingerprint"); + } + fake.records[0]!.child.emitStdout(successEnvelope(validReport())); + fake.records[0]!.child.emitClose(0, null); + await adapter.getResult(handle); + } finally { + fake.records[0]?.child.emitClose(0, null); + await removeTempDir(promptDir); + } +}); + +test("consumer: an identity that raced the child's exit is never handed over (null expected, refused in production)", { timeout: 20_000 }, async () => { + const fake = new FakeSpawn(); + fake.script.push(() => { + // Alive until the cancel below. + }); + const terminations: TerminateRecord[] = []; + const terminate: TerminateProcessTree = async (pid, options) => { + terminations.push({ pid, expectedRootFingerprint: options?.expectedRootFingerprint ?? null }); + return { pid, signal: "SIGKILL", verified: true }; + }; + const promptDir = await makeTempDir("consumer-race"); + const adapter = new ZCodeAdapter({ + resolver: stubResolver(), + spawnImpl: fake.spawnFn, + terminateProcessTreeImpl: terminate, + probe: scriptedProbe(async () => { + // The probe's await straddles the child's exit: the fingerprint that + // comes back may describe a recycled PID generation. + fake.records[0]!.child.emitExit(0, null); + return { fingerprint: "recycled-generation-ticks", fingerprint_precision: "exact" }; + }), + promptTmpDir: promptDir, + childEnvBase: { PATH: "C:\Windows" }, + }); + try { + const handle = await adapter.startTask({ task: makeTask(), workspace: makeWorkspace(), attempt: 1 }); + await waitForSpawn(adapter, handle); + // Give the capture's retry path a moment to observe the exit, then cancel. + await new Promise((resolve) => setTimeout(resolve, 30)); + await adapter.cancelTask(handle).catch(() => undefined); + fake.records[0]!.child.emitClose(0, null); + await adapter.getResult(handle).catch(() => undefined); + for (const record of terminations) { + assert.notEqual(record.expectedRootFingerprint, "recycled-generation-ticks", "a raced post-exit fingerprint is never handed to the terminator"); + } + } finally { + fake.records[0]?.child.emitClose(0, null); + await removeTempDir(promptDir); + } +}); + +test("consumer: an already-exited child is never terminated (exit ends the provable generation)", { timeout: 20_000 }, async () => { + const fake = new FakeSpawn(); + fake.script.push((child) => { + child.emitExit(0, null); + child.emitClose(0, null); + }); + const terminations: TerminateRecord[] = []; + const terminate: TerminateProcessTree = async (pid, options) => { + terminations.push({ pid, expectedRootFingerprint: options?.expectedRootFingerprint ?? null }); + return { pid, signal: "SIGKILL", verified: true }; + }; + const promptDir = await makeTempDir("consumer-exited"); + const adapter = new ZCodeAdapter({ + resolver: stubResolver(), + spawnImpl: fake.spawnFn, + terminateProcessTreeImpl: terminate, + probe: scriptedProbe(async () => ({ fingerprint: "late-ticks", fingerprint_precision: "exact" })), + promptTmpDir: promptDir, + childEnvBase: { PATH: "C:\Windows" }, + }); + try { + const handle = await adapter.startTask({ task: makeTask(), workspace: makeWorkspace(), attempt: 1 }); + await adapter.getResult(handle).catch(() => undefined); + if (process.platform === "win32") { + assert.deepEqual(terminations, [], "an exited child's PID is never signalled on Windows"); + } + } finally { + await removeTempDir(promptDir); + } +}); diff --git a/test/teardown-retry.test.ts b/test/teardown-retry.test.ts new file mode 100644 index 0000000..6de67b6 --- /dev/null +++ b/test/teardown-retry.test.ts @@ -0,0 +1,139 @@ +// Windows teardown ENOTEMPTY regressions (2026-10-17): a full-suite run on +// Node 22.18.0/Windows flaked once in runtime-remediation teardown, and the +// PR #45 review attributed it to the same Windows filesystem timing family as +// the state.lock EPERM/ENOTEMPTY races (attribution, not a proven unique +// cause). Node documents EBUSY/EMFILE/ENFILE/ENOTEMPTY/EPERM as retryable for +// recursive rm, so the fixtures remove trees with an explicit bounded retry +// budget (TEMP_TREE_REMOVE_OPTIONS). Attempt-2 note: Node 22's rmSync with +// default options still performs one internal re-walk, consuming two injected +// failures by itself — each arm below therefore builds its OWN tree with a +// FRESH injection counter, the no-retry arm must fail visibly, the retry arm +// must actually observe failures and survive them, and a permanent error must +// still fail after the budget. The natural flake itself was not reproduced in +// 150 probe rounds (attempt-1 record); these are deterministic injections. +import assert from "node:assert/strict"; +import fs from "node:fs"; +import fsPromises from "node:fs/promises"; +import { mkdtempSync, mkdirSync, writeFileSync, rmSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { syncBuiltinESMExports } from "node:module"; +import path from "node:path"; +import test from "node:test"; +import { TEMP_TREE_REMOVE_OPTIONS, makeTempDir, removeTempDir } from "./helpers.js"; +import { runtimeFixture } from "./runtime-fixture.js"; + +const errnoError = (code: string): NodeJS.ErrnoException => { + const error = new Error(`injected ${code} (deterministic teardown regression)`) as NodeJS.ErrnoException; + error.code = code; + return error; +}; + +const makeTree = (root: string): string => { + const victim = path.join(root, `victim-${Math.random().toString(36).slice(2, 8)}`); + mkdirSync(path.join(victim, "sub"), { recursive: true }); + writeFileSync(path.join(victim, "sub", "a.txt"), "a"); + writeFileSync(path.join(victim, "b.txt"), "b"); + return victim; +}; + +test("the shared fixture teardown passes a bounded retry budget to recursive rm", async () => { + const root = await makeTempDir("teardown-wiring-"); + try { + const victim = makeTree(root); + const realRm = fsPromises.rm; + let observed: unknown = null; + fsPromises.rm = (async (dir: string, options: unknown) => { + observed = options; + return realRm(dir, options as object); + }) as typeof fsPromises.rm; + syncBuiltinESMExports(); + try { await removeTempDir(victim); } + finally { fsPromises.rm = realRm; syncBuiltinESMExports(); } + const options = observed as { recursive?: boolean; force?: boolean; maxRetries?: number; retryDelay?: number }; + assert.equal(options.recursive, true, "removal must stay recursive"); + assert.equal(options.force, true, "removal must stay forced"); + assert.ok((options.maxRetries ?? 0) >= 1, `a retry budget is required, saw ${JSON.stringify(options)}`); + assert.ok((options.retryDelay ?? 0) >= 1, "retries need an explicit delay"); + assert.equal(fs.existsSync(victim), false, "tree must be gone"); + } finally { await removeTempDir(root); } +}); + +test("the async runtimeFixture cleanup carries the same bounded retry budget", async () => { + const fixture = await runtimeFixture(); + const realRm = fsPromises.rm; + let observed: unknown = null; + fsPromises.rm = (async (dir: string, options: unknown) => { + observed = options; + return realRm(dir, options as object); + }) as typeof fsPromises.rm; + syncBuiltinESMExports(); + try { await fixture.cleanup(); } + finally { fsPromises.rm = realRm; syncBuiltinESMExports(); } + const options = observed as { recursive?: boolean; force?: boolean; maxRetries?: number; retryDelay?: number }; + assert.equal(options.recursive, true, "the fixture cleanup must stay recursive"); + assert.equal(options.force, true, "the fixture cleanup must stay forced"); + assert.ok((options.maxRetries ?? 0) >= 1, `the fixture cleanup needs the retry budget, saw ${JSON.stringify(options)}`); + assert.ok((options.retryDelay ?? 0) >= 1, "the fixture cleanup needs an explicit retry delay"); + assert.equal(fs.existsSync(fixture.root), false, "the fixture tree must be gone"); +}); + +// On Node 22 the recursive rm implementation calls the exported rmdirSync, so +// a patch can inject the observed errno codes end-to-end. Node 24's rimraf +// binds the internal operation directly and the patch is not observable +// there; the wiring tests above cover the option contract on every version. +test("injected ENOTEMPTY: default rm fails visibly, the fixture budget survives it, permanent errors still fail", { skip: Number(process.versions.node.split(".")[0]) !== 22, timeout: 60_000 }, () => { + const root = mkdtempSync(path.join(tmpdir(), "zcode-bridge-test-teardown-inject-")); + try { + const realRmdirSync = fs.rmdirSync; + const injectVictimFailures = (victim: string, times: number) => { + let failuresLeft = times; + let observed = 0; + const fail = (target: string): NodeJS.ErrnoException | null => { + if (target !== victim || failuresLeft <= 0) return null; + failuresLeft -= 1; + observed += 1; + return errnoError("ENOTEMPTY"); + }; + return { fail, observedCount: () => observed }; + }; + const runRm = (victim: string, injector: ReturnType, options: fs.RmOptions): void => { + fs.rmdirSync = ((target: fs.PathLike, rmdirOptions?: fs.RmDirOptions) => { + const injected = injector.fail(String(target)); + if (injected) throw injected; + return realRmdirSync(target, rmdirOptions); + }) as typeof fs.rmdirSync; + syncBuiltinESMExports(); + try { + fs.rmSync(victim, { recursive: true, force: true, ...options }); + } finally { + fs.rmdirSync = realRmdirSync; + syncBuiltinESMExports(); + } + }; + + // Arm 1 — default options on its OWN fresh tree: the injected transient + // failures must stay visible (no masking without a budget). + const victim1 = makeTree(root); + const arm1 = injectVictimFailures(victim1, 3); + assert.throws(() => runRm(victim1, arm1, {}), (error: NodeJS.ErrnoException) => error.code === "ENOTEMPTY", + "default options must keep the transient failure visible"); + assert.ok(arm1.observedCount() >= 1, "the default arm must actually observe injected failures"); + assert.equal(fs.existsSync(victim1), true, "the tree survives a visible failure"); + rmSync(victim1, { recursive: true, force: true, ...TEMP_TREE_REMOVE_OPTIONS }); + + // Arm 2 — the fixture budget on its OWN fresh tree with a FRESH counter: + // it must also observe failures and survive them through retries. + const victim2 = makeTree(root); + const arm2 = injectVictimFailures(victim2, 3); + runRm(victim2, arm2, { ...TEMP_TREE_REMOVE_OPTIONS }); + assert.ok(arm2.observedCount() >= 1, "the retry arm must actually receive the injected errors (attempt-1 defect: it saw none)"); + assert.equal(fs.existsSync(victim2), false, "the fixture retry budget must survive the injected ENOTEMPTY"); + + // Arm 3 — a permanent error with the full budget still fails visibly. + const victim3 = makeTree(root); + const arm3 = injectVictimFailures(victim3, Number.MAX_SAFE_INTEGER); + assert.throws(() => runRm(victim3, arm3, { ...TEMP_TREE_REMOVE_OPTIONS }), (error: NodeJS.ErrnoException) => error.code === "ENOTEMPTY", + "budget exhaustion must keep genuine permanent failures visible"); + assert.equal(fs.existsSync(victim3), true, "the tree survives the exhausted budget"); + } finally { rmSync(root, { recursive: true, force: true, ...TEMP_TREE_REMOVE_OPTIONS }); } +}); diff --git a/test/validate-plugin-root-pattern.test.ts b/test/validate-plugin-root-pattern.test.ts index b7ebf7e..162ecc2 100644 --- a/test/validate-plugin-root-pattern.test.ts +++ b/test/validate-plugin-root-pattern.test.ts @@ -12,7 +12,7 @@ import { tmpdir } from "node:os"; import path from "node:path"; import test from "node:test"; import { fileURLToPath, pathToFileURL } from "node:url"; -import { buildRepoRootLeakPattern } from "../scripts/lib/repo-root-pattern.mjs"; +import { assertNotFilesystemRoot, buildRepoRootLeakPattern } from "../scripts/lib/repo-root-pattern.mjs"; test("a short root like /app does not match unrelated URL or path segments", () => { const pattern = buildRepoRootLeakPattern("/app"); @@ -155,6 +155,103 @@ test("a repeated probe does not skip later matches (no sticky/global state)", () assert.equal(pattern.test('"https://json-schema.org/draft/2019-09/vocab/applicability"'), false); }); +test("filesystem roots are rejected with an explanation instead of silently disabling the leak check (2026-10-17)", () => { + const filesystemRoots = [ + ["/", "POSIX"], + ["//", "POSIX"], + ["///", "POSIX"], + ["/./", "normalized POSIX"], + ["/alpha/..", "normalized POSIX"], + ["/a/../..", "normalized POSIX"], + ["C:\\", "drive"], + ["C:", "drive"], + ["C:/", "drive"], + ["c:\\", "drive"], + ["C:\\\\", "drive (repeated separators)"], + ["c:/./", "normalized drive"], + ["C:/alpha/..", "normalized drive"], + ["\\\\?\\C:\\", "extended drive"], + ["\\\\?\\C:/", "extended drive"], + ["\\\\.\\C:\\", "device drive"], + ["\\\\server\\share", "share"], + ["\\\\server\\share\\", "share"], + ["\\\\server\\share\\repo\\..", "normalized share"], + ["\\\\?\\UNC\\server\\share", "extended share"], + ] as const; + for (const [root, kind] of filesystemRoots) { + assert.throws( + () => buildRepoRootLeakPattern(root), + (error: unknown) => { + assert.ok(error instanceof Error, `${kind} root ${JSON.stringify(root)} must be rejected with an Error`); + assert.match(error.message, /filesystem root/u, "the rejection must say it is a filesystem root"); + assert.match(error.message, /false positive/u, "the rejection must explain why the check is meaningless"); + assert.ok(error.message.includes(JSON.stringify(root)), "the rejection must name the rejected input"); + if (kind.startsWith("normalized")) { + assert.match(error.message, /lexically equivalent/u, `normalized spelling ${JSON.stringify(root)} must say which root it collapses to`); + } + return true; + }, + `${kind} root ${JSON.stringify(root)} must be rejected`, + ); + } + // Non-string and blank inputs are refused for the same reason: no silent skip. + for (const invalid of [undefined, null, 0, "", " "]) { + assert.throws(() => buildRepoRootLeakPattern(invalid as string), /non-empty repository root/u, `invalid root ${JSON.stringify(invalid)} must be refused`); + } +}); + +test("roots one level below a filesystem root keep building usable patterns", () => { + const nearRoots = [ + "/app", + "/home/开发者", + "/a/b/../c", + "C:\\Program Files\\Bridge Dev (x86)", + "C:\\repo", + "C:\\repo\\..\\repo2", + "D:\\x", + "\\\\server\\share\\repo", + "\\\\server\\share\\repo\\..\\repo2", + "\\\\?\\C:\\repo", + "E:/Programs/example repository", + ]; + for (const root of nearRoots) { + let pattern: RegExp | undefined; + assert.doesNotThrow(() => { pattern = buildRepoRootLeakPattern(root); }, `root ${JSON.stringify(root)} is a checkout path and must build`); + assert.ok(pattern instanceof RegExp); + if (root === "D:\\x") { + assert.equal(pattern.test('"D:\\\\x\\\\src\\\\leak.ts"'), true, "a one-level drive path still detects its own leaks"); + assert.equal(pattern.test('"D:\\\\y\\\\src\\\\leak.ts"'), false, "and still ignores sibling directories"); + } + } + // Sanity: the guard is exported so hosts can share the exact semantics. + assert.doesNotThrow(() => assertNotFilesystemRoot("/app")); + assert.throws(() => assertNotFilesystemRoot("/"), /filesystem root/u); + assert.throws(() => assertNotFilesystemRoot("/alpha/.."), /lexically equivalent/u); +}); + +test("an HTTP URL whose host equals a short root is not a development-path leak (2026-10-17)", () => { + const pattern = buildRepoRootLeakPattern("/app"); + for (const url of [ + '"https://app/config"', + '"http://app/x"', + '"https://app.example/config"', + '"https://app#fragment"', + ]) { + assert.equal(pattern.test(url), false, `an ordinary URL must not be flagged: ${url}`); + pattern.lastIndex = 0; + } + // The genuine references on the same short root stay flagged. + for (const leak of [ + '"/app/src/main.ts"', + JSON.stringify("file:///app/src/main.ts"), + '"// /app/src/leak.ts\n"', + '"/app"', + ]) { + assert.equal(pattern.test(leak), true, `a genuine reference must stay flagged: ${leak}`); + pattern.lastIndex = 0; + } +}); + /** * Runs the real scripts/validate-plugin.mjs against a fixture repository * (space + unicode + regex characters in the root name) with the current diff --git a/test/windows-cleanup-review-regressions.test.ts b/test/windows-cleanup-review-regressions.test.ts new file mode 100644 index 0000000..2a84441 --- /dev/null +++ b/test/windows-cleanup-review-regressions.test.ts @@ -0,0 +1,264 @@ +// Deterministic regressions for evaluator/review-round-1 blockers R2/R3/R4. +// All cases run on scripted fake transactions — no OS processes, no signals. +import assert from "node:assert/strict"; +import test from "node:test"; +import { + terminateWindowsProcessTree, + verifyWindowsRecordedTreeClaimants, + type HoldOutcome, + type KillOutcome, + type WindowsProcessRow, + type WindowsTreeTransactionLike, +} from "../src/adapters/windows-process-cleanup.js"; + +const BASE = 134_367_000_000_000_000n; +const MICROS = 10n; +const FT = (value: bigint | number): string => value.toString(); + +function row(pid: number, parentPid: number, cimOffset: bigint, ticks?: string | null): WindowsProcessRow { + return { pid, parentPid, creationFileTime: FT(BASE + cimOffset * MICROS), startTicks: ticks === undefined ? `${pid}-ticks` : ticks }; +} + +class FakeTransaction implements WindowsTreeTransactionLike { + rows: WindowsProcessRow[]; + readonly holds: number[][] = []; + readonly kills: Array<{ pid: number; ticks: string }> = []; + closeCount = 0; + holdBehavior: ((pids: readonly number[]) => Promise) | null = null; + killBehavior: ((pid: number, ticks: string) => KillOutcome) = (pid) => { + this.rows = this.rows.filter((r) => r.pid !== pid); + return { status: "signaled", exitFileTime: FT(BASE + 1_000_000n) }; + }; + enumerateBehavior: (() => Promise) | null = null; + closeBehavior: (() => Promise) | null = null; + + constructor(rows: readonly WindowsProcessRow[]) { + this.rows = [...rows]; + } + + async enumerate(): Promise { + if (this.enumerateBehavior) return this.enumerateBehavior(); + return [...this.rows]; + } + + async hold(pids: readonly number[]): Promise { + this.holds.push([...pids]); + if (this.holdBehavior) return this.holdBehavior(pids); + return pids.map((pid) => { + const target = this.rows.find((r) => r.pid === pid); + if (!target || !target.startTicks || !target.creationFileTime) return { pid, status: "absent" } as HoldOutcome; + return { pid, status: "ok", creationFileTime: FT(BigInt(target.creationFileTime) + 3n), creationTicks: target.startTicks } as HoldOutcome; + }); + } + + async kill(pid: number, expectedTicks: string, waitMs: number): Promise { + void waitMs; + this.kills.push({ pid, ticks: expectedTicks }); + return this.killBehavior(pid, expectedTicks); + } + + async close(): Promise { + this.closeCount += 1; + if (this.closeBehavior) await this.closeBehavior(); + } +} + +function drive(fake: FakeTransaction, clock: { now: number } = { now: 0 }): Promise { + return terminateWindowsProcessTree(101, "101-ticks", 5_000, { + open: async () => fake, + now: () => clock.now, + sleep: async () => undefined, + }); +} + +test("R2: a permanently denied hold fails the round with every observed candidate, before any signal", async () => { + const fake = new FakeTransaction([row(101, 4, 0n), row(102, 101, 40n)]); + fake.holdBehavior = async (pids) => pids.map((pid) => { + if (pid === 102) return { pid, status: "denied", error: 5 } as HoldOutcome; + const target = fake.rows.find((r) => r.pid === pid); + if (!target || !target.startTicks) return { pid, status: "absent" } as HoldOutcome; + return { pid, status: "ok", creationFileTime: FT(BigInt(target.creationFileTime!) + 3n), creationTicks: target.startTicks } as HoldOutcome; + }); + await assert.rejects(drive(fake), (error: unknown) => { + assert.match((error as Error).message, /process_tree_hold_unresolved/); + const memberPids = (error as { memberPids?: number[] }).memberPids ?? []; + assert.ok(memberPids.includes(102), "the denied candidate stays in the failure set"); + assert.ok(memberPids.includes(101), "the already-proven root stays in the failure set"); + return true; + }); + assert.deepEqual(fake.kills, [], "no member is signalled while a candidate is unprovable"); + assert.equal(fake.closeCount, 1); +}); + +test("R2: the review timeline — denied child vanishes while its grandchild lives — never yields success", async () => { + const fake = new FakeTransaction([row(101, 4, 0n), row(102, 101, 40n)]); + let rounds = 0; + fake.enumerateBehavior = async () => { + rounds += 1; + fake.rows = rounds === 1 ? [row(101, 4, 0n), row(102, 101, 40n)] : [row(103, 102, 2_000n)]; + return [...fake.rows]; + }; + fake.holdBehavior = async (pids) => pids.map((pid) => { + if (pid === 102) return { pid, status: "denied", error: 5 } as HoldOutcome; + const target = fake.rows.find((r) => r.pid === pid); + if (!target || !target.startTicks || !target.creationFileTime) return { pid, status: "absent" } as HoldOutcome; + return { pid, status: "ok", creationFileTime: FT(BigInt(target.creationFileTime) + 3n), creationTicks: target.startTicks } as HoldOutcome; + }); + await assert.rejects(drive(fake), /process_tree_hold_unresolved/); +}); + +test("R2: hold responses that are missing, duplicated, or unexpected are protocol failures with the known set", async () => { + for (const variant of ["missing", "duplicate", "unexpected"] as const) { + const fake = new FakeTransaction([row(101, 4, 0n)]); + fake.holdBehavior = async (pids) => { + const outcomes = await new FakeTransaction(fake.rows).hold(pids); + if (variant === "missing") return outcomes.slice(0, -1); + if (variant === "duplicate") return [...outcomes, outcomes[0]!]; + return [...outcomes, { pid: 9_999, status: "ok" } as HoldOutcome]; + }; + await assert.rejects(drive(fake), (error: unknown) => { + assert.match((error as Error).message, /process_tree_hold_invalid/); + assert.deepEqual((error as { memberPids?: number[] }).memberPids, [101]); + return true; + }); + assert.equal(fake.kills.length, 0, `variant ${variant}: nothing is signalled`); + assert.equal(fake.closeCount, 1, `variant ${variant}: the helper still closes`); + } +}); + +test("R2 addendum: a candidate that vanishes between enumeration and hold fails closed with its subtree unattributable", async () => { + // The candidate was live microseconds ago; if it exited just now, its own + // descendants may survive and can no longer be attributed to a proven + // parent identity. Dropping the row silently could return success while a + // grandchild lives — the round must fail and keep the candidate recorded. + const fake = new FakeTransaction([row(101, 4, 0n), row(104, 101, 60n)]); + fake.holdBehavior = async (pids) => { + const outcomes = await new FakeTransaction(fake.rows).hold(pids); + return outcomes.map((outcome) => (outcome.pid === 104 ? { pid: 104, status: "absent" } as HoldOutcome : outcome)); + }; + await assert.rejects(drive(fake), (error: unknown) => { + assert.match((error as Error).message, /process_tree_candidate_absent/); + const memberPids = (error as { memberPids?: number[] }).memberPids ?? []; + assert.ok(memberPids.includes(104), "the vanished candidate stays recorded"); + return true; + }); + assert.equal(fake.kills.length, 0, "nothing is signalled for an unattributable subtree"); + assert.equal(fake.closeCount, 1); +}); + +test("R2: a deep tree terminates children-first with a complete receipt regardless of row order", async () => { + // Review-round-1 addendum (test quality): a true three-plus-level chain, + // asserted PAIRWISE — every member is signalled strictly before its own + // parent — and converged within a single round even when enumeration + // delivers rows before their parents. + const chain = [row(101, 4, 0n), row(102, 101, 40n), row(103, 102, 80n), row(104, 103, 120n)]; + const parentOf = new Map([[102, 101], [103, 102], [104, 103]]); + const orders: Record = { + forward: [...chain], + reverse: [...chain].reverse(), + shuffled: [chain[2]!, chain[0]!, chain[3]!, chain[1]!], + }; + for (const [name, rows] of Object.entries(orders)) { + const fake = new FakeTransaction(rows); + let enumerateCalls = 0; + const killEnumerationCycles: number[] = []; + const originalKill = fake.kill.bind(fake); + fake.kill = async (pid: number, ticks: string, waitMs: number) => { + killEnumerationCycles.push(enumerateCalls); + return originalKill(pid, ticks, waitMs); + }; + fake.enumerateBehavior = async () => { + enumerateCalls += 1; + return [...fake.rows]; + }; + const receipt = await drive(fake) as { signaled: number[]; members: unknown[] }; + assert.deepEqual([...receipt.signaled].sort((a, b) => a - b), [101, 102, 103, 104], `${name}: every level is signalled exactly once`); + for (const [child, parent] of parentOf) { + assert.ok( + receipt.signaled.indexOf(child) < receipt.signaled.indexOf(parent), + `${name}: pid ${String(child)} is signalled strictly before its own parent ${String(parent)}`, + ); + } + assert.equal(receipt.members.length, 4, `${name}: the receipt records every member identity`); + assert.deepEqual(killEnumerationCycles, [1, 1, 1, 1], `${name}: every row was resolved in the first enumeration despite delivery order`); + } +}); + +test("R3: a mid-transaction enumeration failure escapes with every known member identity", async () => { + const fake = new FakeTransaction([row(101, 4, 0n), row(102, 101, 40n)]); + fake.killBehavior = (pid) => { + if (pid === 102) return { status: "wait_timeout", exitFileTime: null }; + fake.rows = fake.rows.filter((r) => r.pid !== pid); + return { status: "signaled", exitFileTime: FT(BASE + 1_000_000n) }; + }; + let rounds = 0; + fake.enumerateBehavior = async () => { + rounds += 1; + if (rounds === 1) return [row(101, 4, 0n), row(102, 101, 40n)]; + throw new Error("synthetic incomplete query after partial termination"); + }; + await assert.rejects(drive(fake), (error: unknown) => { + assert.match((error as Error).message, /process_tree_transaction_error: synthetic incomplete query/); + const memberPids = (error as { memberPids?: number[] }).memberPids ?? []; + assert.ok(memberPids.includes(102), "the still-unconfirmed child is preserved for later coverage"); + assert.ok(memberPids.includes(101), "the terminated root is preserved"); + return true; + }); +}); + +test("R3: a transport hold failure is wrapped with the known member set", async () => { + const fake = new FakeTransaction([row(101, 4, 0n), row(102, 101, 40n)]); + fake.holdBehavior = async (pids) => { + if (pids.includes(102)) throw new Error("synthetic hold transport failure"); + return new FakeTransaction(fake.rows).hold(pids); + }; + await assert.rejects(drive(fake), (error: unknown) => { + assert.match((error as Error).message, /process_tree_transaction_error: synthetic hold transport failure/); + assert.ok(((error as { memberPids?: number[] }).memberPids ?? []).includes(101)); + return true; + }); +}); + +test("R4: a helper close failure after a clean body fails the whole transaction", async () => { + const fake = new FakeTransaction([row(101, 4, 0n)]); + fake.closeBehavior = async () => { + throw new Error("synthetic helper close failure"); + }; + await assert.rejects(drive(fake), (error: unknown) => { + assert.match((error as Error).message, /process_tree_close_failed: synthetic helper close failure/); + assert.deepEqual((error as { memberPids?: number[] }).memberPids, [101]); + return true; + }); +}); + +test("R4: a helper close failure after a body failure keeps both visible", async () => { + const fake = new FakeTransaction([row(101, 4, 0n)]); + fake.killBehavior = () => ({ status: "wait_timeout", exitFileTime: null }); + fake.closeBehavior = async () => { + throw new Error("synthetic helper close failure"); + }; + const clock = { now: 0 }; + await assert.rejects(terminateWindowsProcessTree(101, "101-ticks", 1_000, { + open: async () => fake, + now: () => clock.now, + sleep: async () => { + clock.now += 600; + }, + }), (error: unknown) => { + const message = (error as Error).message; + assert.match(message, /process_tree_cleanup_unverified/); + assert.match(message, /process_tree_close_failed/); + return true; + }); +}); + +test("R4: the recorded-tree scan fails visibly when its helper close fails", async () => { + const fake = new FakeTransaction([]); + fake.closeBehavior = async () => { + throw new Error("synthetic scan close failure"); + }; + await assert.rejects( + verifyWindowsRecordedTreeClaimants([{ pid: 101, fingerprint: "101-ticks" }], { open: async () => fake, now: () => 0, sleep: async () => undefined }), + /process_tree_close_failed: synthetic scan close failure/, + ); +}); diff --git a/test/windows-process-cleanup.test.ts b/test/windows-process-cleanup.test.ts index 1ba6eba..1cb6d4a 100644 --- a/test/windows-process-cleanup.test.ts +++ b/test/windows-process-cleanup.test.ts @@ -1,75 +1,414 @@ +// ADR-009 deterministic unit tests: interval-exact membership resolution, +// the handle-transaction termination driver, and the recorded-tree claimant +// scan. Everything here runs on injected data — no OS processes, no signals. import assert from "node:assert/strict"; import test from "node:test"; -import { parseWindowsTreeCapture, terminateWindowsProcessTree, type WindowsCleanupDependencies } from "../src/adapters/windows-process-cleanup.js"; -import type { ProbeVerdict } from "../src/runtime/process-probe.js"; +import { + resolveWindowsTreeMembership, + terminateWindowsProcessTree, + verifyWindowsRecordedTreeClaimants, + type HeldProcessInfo, + type HoldOutcome, + type KillOutcome, + type MembershipInput, + type WindowsProcessRow, + type WindowsTreeTransactionLike, +} from "../src/adapters/windows-process-cleanup.js"; -const requests = parseWindowsTreeCapture("bridge_tree_snapshot_v1\n101|123\n102|456", 101); -function verdict(state: ProbeVerdict["state"]): ProbeVerdict { - return { state, reason_code: state === "exited" ? "pid_absent" : "test", observed_at: new Date().toISOString() }; +// Realistic 100ns FILETIME-scale anchors (values only need internal +// consistency). CIM values are microsecond-truncated (last digit 0); exact +// handle times sit inside [cim, cim + 1us]. +const FT = (value: bigint | number): string => value.toString(); +const BASE = 134_367_000_000_000_000n; // arbitrary in-range filetime +const MICROS = 10n; + +function row(pid: number, parentPid: number, cimOffset: bigint, ticks?: string | null): WindowsProcessRow { + return { pid, parentPid, creationFileTime: FT(BASE + cimOffset * MICROS), startTicks: ticks === undefined ? `${pid}-ticks` : ticks }; +} +function heldFor(rows: readonly WindowsProcessRow[], pid: number, exactOffset: bigint): HeldProcessInfo { + const target = rows.find((r) => r.pid === pid); + if (!target || !target.startTicks) throw new Error(`fixture: row ${String(pid)} missing`); + return { pid, creationFileTime: FT(BASE + exactOffset * MICROS + 3n), creationTicks: target.startTicks }; } -function fixture(after: ProbeVerdict["state"][], code: number | null = 0, before: ProbeVerdict["state"][] = ["alive", "alive"]): WindowsCleanupDependencies & { kills: number[] } { - let probes = 0; - const kills: number[] = []; +function input(overrides: Partial & { rows: readonly WindowsProcessRow[] }): MembershipInput { return { - kills, capture: async () => requests, - probe: async () => (probes++ === 0 ? before : after).map(verdict), - kill: async (pid) => { kills.push(pid); return code; }, + rootPid: 101, + expectedRootTicks: "101-ticks", + held: new Map(), + killed: new Map(), + naturalGonePids: new Set(), + naturalGoneCreation: new Map(), + ...overrides, }; } -test("taskkill nonzero is resolved only by fresh exit evidence for root and descendants", async () => { - const dependencies = fixture(["exited", "exited"], 128); - await terminateWindowsProcessTree(101, 0, dependencies); - assert.deepEqual(dependencies.kills, [101]); +test("membership: a fresh transaction refuses a root without a caller-bound identity", () => { + const rows = [row(101, 4, 0n)]; + const resolution = resolveWindowsTreeMembership(input({ rows, expectedRootTicks: null })); + assert.equal(resolution.kind, "refuse"); + assert.equal(resolution.kind === "refuse" && resolution.code, "process_tree_root_identity_required"); }); -test("successful taskkill and root exit cannot hide a surviving descendant", async () => { - await assert.rejects(terminateWindowsProcessTree(101, 0, fixture(["exited", "alive"])), /recorded executors remain alive/); +test("membership: a root missing from a fresh transaction refuses conservatively (no provable tree state)", () => { + const resolution = resolveWindowsTreeMembership(input({ rows: [row(102, 101, 900n)] })); + assert.equal(resolution.kind, "refuse"); + assert.equal(resolution.kind === "refuse" && resolution.code, "process_tree_root_exited"); }); -test("a localized command failure does not leak its text or defeat verified natural exit", async () => { - const dependencies = fixture(["exited", "exited"]); - dependencies.kill = async () => { throw new Error("本地化错误与私有路径 �"); }; - await terminateWindowsProcessTree(101, 0, dependencies); - const live = fixture(["exited", "alive"]); - live.kill = dependencies.kill; - await assert.rejects(terminateWindowsProcessTree(101, 0, live), (error: Error) => { - assert.match(error.message, /command_error/); - assert.doesNotMatch(error.message, /本地化|私有|�/); - return true; - }); +test("membership: a root owned by a different generation (PID reuse) is never expanded", () => { + const rows = [row(101, 4, 0n, "reuser-ticks")]; + const resolution = resolveWindowsTreeMembership(input({ rows })); + assert.equal(resolution.kind, "refuse"); + assert.equal(resolution.kind === "refuse" && resolution.code, "process_tree_root_identity_changed"); +}); + +test("membership: a root whose identity cannot be read refuses (unknown is never trusted)", () => { + const rows = [row(101, 4, 0n, null)]; + const resolution = resolveWindowsTreeMembership(input({ rows })); + assert.equal(resolution.kind, "refuse"); + assert.equal(resolution.kind === "refuse" && resolution.code, "process_tree_root_identity_unknown"); +}); + +test("membership: an unheld root requests a hold before any membership decision", () => { + const resolution = resolveWindowsTreeMembership(input({ rows: [row(101, 4, 0n)] })); + assert.equal(resolution.kind, "members"); + assert.deepEqual(resolution.kind === "members" ? resolution.holdRequired : [], [101]); +}); + +test("membership: root handle generation mismatch with its row refuses", () => { + const rows = [row(101, 4, 0n)]; + const bad = new Map([[101, { pid: 101, creationFileTime: FT(BASE + 5_000n), creationTicks: "101-ticks" }]]); + const resolution = resolveWindowsTreeMembership(input({ rows, held: bad })); + assert.equal(resolution.kind, "refuse"); + assert.equal(resolution.kind === "refuse" && resolution.code, "process_tree_root_generation_mismatch"); +}); + +test("membership: a stale row that predates the parent is excluded without being held", () => { + // The old-record PID-reuse shape: an unrelated process created long before + // the root whose ParentProcessId equals the root pid. + const rows = [row(101, 4, 0n), row(909, 101, -50_000n)]; + const held = new Map([[101, heldFor(rows, 101, 0n)]]); + const resolution = resolveWindowsTreeMembership(input({ rows, held })); + assert.equal(resolution.kind, "members"); + assert.deepEqual(resolution.kind === "members" ? resolution.members.map((m) => m.pid) : [], [101]); + // The stale row was never asked to be held: no handle was opened on it. + assert.equal(resolution.kind === "members" && resolution.holdRequired.includes(909), false); +}); + +test("membership: a live child created after its parent joins; equal exact ticks refuse", () => { + const rows = [row(101, 4, 0n), row(102, 101, 40n)]; + const held = new Map([ + [101, heldFor(rows, 101, 0n)], + [102, heldFor(rows, 102, 40n)], + ]); + const member = resolveWindowsTreeMembership(input({ rows, held })); + assert.deepEqual(member.kind === "members" ? member.members.map((m) => m.pid) : [], [101, 102]); + + // Same 100ns creation instant (representation intervals overlap): ordering + // is indistinguishable; refuse. + const sameInstant = new Map(held); + sameInstant.set(102, { pid: 102, creationFileTime: FT(BASE + 3n), creationTicks: "102-ticks" }); + const refused = resolveWindowsTreeMembership(input({ rows: [row(101, 4, 0n), row(102, 101, 0n, "102-ticks")], held: sameInstant })); + assert.equal(refused.kind === "refuse" && refused.code, "process_tree_edge_indistinguishable"); +}); + +test("membership: a candidate overlapping but provably older than the parent is excluded as stale", () => { + // Overlapping representation intervals: the exact times must decide. + const rows = [row(101, 4, 0n), row(103, 101, 0n, "103-ticks")]; // both cims anchor at BASE + const held = new Map([ + [101, { pid: 101, creationFileTime: FT(BASE + 8n), creationTicks: "101-ticks" }], // parent exact at BASE+8 + [103, { pid: 103, creationFileTime: FT(BASE + 6n), creationTicks: "103-ticks" }], // child exact at BASE+6: older + ]); + const resolution = resolveWindowsTreeMembership(input({ rows, held })); + assert.equal(resolution.kind, "members"); + assert.deepEqual(resolution.kind === "members" ? resolution.members.map((m) => m.pid) : [], [101]); +}); + +test("membership: children of a killed member are decided by the exact [creation, exit] window", () => { + const parentCreation = FT(BASE + 3n); + const parentExit = FT(BASE + 90_000n); + const killed = new Map([[201, { pid: 201, creationFileTime: parentCreation, exitFileTime: parentExit }]]); + const inside = [row(301, 201, 2_000n)]; + const heldInside = new Map([[301, heldFor(inside, 301, 2_000n)]]); + const member = resolveWindowsTreeMembership(input({ rows: inside, rootPid: 201, expectedRootTicks: null, killed, held: heldInside })); + assert.deepEqual(member.kind === "members" ? member.members.map((m) => m.pid) : [], [301]); + + const stale = [row(302, 201, -2_000n)]; + const excluded = resolveWindowsTreeMembership(input({ rows: stale, rootPid: 201, expectedRootTicks: null, killed })); + assert.deepEqual(excluded.kind === "members" ? excluded.members.map((m) => m.pid) : [], []); + + const afterExit = [row(303, 201, 500_000n)]; + const foreign = resolveWindowsTreeMembership(input({ rows: afterExit, rootPid: 201, expectedRootTicks: null, killed })); + assert.deepEqual(foreign.kind === "members" ? foreign.members.map((m) => m.pid) : [], []); }); -test("unknown post-termination probe does not release the process tree", async () => { - await assert.rejects(terminateWindowsProcessTree(101, 0, fixture(["exited", "unknown"], 128)), /probe_unverified/); +test("membership: a killed member without a confirmed exit time makes candidates refuse", () => { + const killed = new Map([[201, { pid: 201, creationFileTime: FT(BASE + 3n), exitFileTime: null }]]); + const resolution = resolveWindowsTreeMembership(input({ + rows: [row(301, 201, 2_000n)], + rootPid: 201, + expectedRootTicks: null, + killed, + })); + assert.equal(resolution.kind === "refuse" && resolution.code, "process_tree_parent_exit_unconfirmed"); }); -test("unknown pre-termination identity prevents signaling", async () => { - const dependencies = fixture(["exited", "exited"], 0, ["unknown", "alive"]); - await assert.rejects(terminateWindowsProcessTree(101, 0, dependencies), /probe_unverified/); - assert.deepEqual(dependencies.kills, []); +test("membership: a naturally-exited member's candidates refuse unless provably older", () => { + // The root is alive and held; mid-tree member 102 vanished without our + // signal; a live process now claims 102 as its parent. + const rows = [row(101, 4, 0n), row(301, 102, 2_000n)]; + const held = new Map([[101, heldFor(rows, 101, 0n)]]); + const naturalGone = new Set([102]); + const naturalCreation = new Map([[102, FT(BASE + 400n)]]); + const candidate = resolveWindowsTreeMembership(input({ rows, held, naturalGonePids: naturalGone, naturalGoneCreation: naturalCreation })); + assert.equal(candidate.kind === "refuse" && candidate.code, "process_tree_parent_unprovable"); + + const stale = resolveWindowsTreeMembership(input({ + rows: [row(101, 4, 0n), row(302, 102, -20_000n)], + held, + naturalGonePids: naturalGone, + naturalGoneCreation: naturalCreation, + })); + assert.deepEqual(stale.kind === "members" ? stale.members.map((m) => m.pid) : [], [101]); }); -test("a root PID whose old identity exited is never signaled even with a surviving child", async () => { - const dependencies = fixture(["exited", "alive"], 0, ["exited", "alive"]); - await assert.rejects(terminateWindowsProcessTree(101, 0, dependencies), /root exited/); - assert.deepEqual(dependencies.kills, []); - const exited = fixture(["exited", "exited"], 0, ["exited", "exited"]); - await terminateWindowsProcessTree(101, 0, exited); - assert.deepEqual(exited.kills, []); +test("membership: a member whose identity cannot be read refuses instead of being signalled", () => { + const rows = [row(101, 4, 0n), row(102, 101, 40n, null)]; + const held = new Map([ + [101, heldFor(rows, 101, 0n)], + [102, { pid: 102, creationFileTime: FT(BASE + 403n), creationTicks: "whatever" }], + ]); + const resolution = resolveWindowsTreeMembership(input({ rows, held })); + assert.equal(resolution.kind === "refuse" && resolution.code, "process_tree_member_identity_unknown"); }); -test("incomplete probe batches cannot prove omitted descendants exited", async () => { - const dependencies = fixture(["exited", "exited"]); - dependencies.probe = async () => [verdict("exited")]; - await assert.rejects(terminateWindowsProcessTree(101, 0, dependencies), /probe_unverified/); - assert.deepEqual(dependencies.kills, []); +test("membership: duplicate rows refuse the snapshot as invalid", () => { + const rows = [row(101, 4, 0n), row(101, 4, 0n)]; + const resolution = resolveWindowsTreeMembership(input({ rows })); + assert.equal(resolution.kind === "refuse" && resolution.code, "process_tree_snapshot_invalid"); }); -test("tree capture requires a complete ASCII protocol and retains absent root for recheck", () => { - const absent = parseWindowsTreeCapture("bridge_tree_snapshot_v1\n101|absent\n102|456", 101); - assert.equal(absent[0]?.identity, null); - for (const bad of ["101|123", "bridge_tree_snapshot_v1\n102|456", "bridge_tree_snapshot_v1\n101|", "bridge_tree_snapshot_v1\n101|123\n101|456", "bridge_tree_snapshot_v1\n101|123\ntruncated"]) { - assert.throws(() => parseWindowsTreeCapture(bad, 101), /snapshot_invalid/); +test("membership: a transaction that killed the whole tree succeeds when nothing new claims it", () => { + const killed = new Map([ + [101, { pid: 101, creationFileTime: FT(BASE + 3n), exitFileTime: FT(BASE + 500n) }], + ]); + const resolution = resolveWindowsTreeMembership(input({ rows: [], killed })); + assert.equal(resolution.kind, "success"); +}); + +// -- driver over a scripted fake transaction --------------------------------- + +class FakeTransaction implements WindowsTreeTransactionLike { + rows: WindowsProcessRow[]; + readonly holds: number[][] = []; + readonly kills: Array<{ pid: number; ticks: string; waitMs: number }> = []; + closeCount = 0; + killBehavior: (pid: number, ticks: string) => KillOutcome = (pid) => { + this.rows = this.rows.filter((r) => r.pid !== pid); + return { status: "signaled", exitFileTime: FT(BASE + 1_000_000n) }; + }; + + constructor(rows: readonly WindowsProcessRow[]) { + this.rows = [...rows]; + } + + async enumerate(): Promise { + return [...this.rows]; + } + + async hold(pids: readonly number[]): Promise { + this.holds.push([...pids]); + return pids.map((pid) => { + const target = this.rows.find((r) => r.pid === pid); + if (!target) return { pid, status: "absent" } as HoldOutcome; + if (!target.startTicks || !target.creationFileTime) return { pid, status: "denied", error: 5 } as HoldOutcome; + return { + pid, + status: "ok", + creationFileTime: FT(BigInt(target.creationFileTime) + 3n), + creationTicks: target.startTicks, + } as HoldOutcome; + }); + } + + async kill(pid: number, expectedTicks: string, waitMs: number): Promise { + this.kills.push({ pid, ticks: expectedTicks, waitMs }); + return this.killBehavior(pid, expectedTicks); + } + + async close(): Promise { + this.closeCount += 1; } +} + +test("driver: signals exactly the verified members, children first, then verifies an empty closure", async () => { + const rows = [row(101, 4, 0n), row(102, 101, 40n)]; + const fake = new FakeTransaction(rows); + let opened = 0; + const receipt = await terminateWindowsProcessTree(101, "101-ticks", 5_000, { + open: async () => { + opened += 1; + return fake; + }, + now: () => 0, + sleep: async () => undefined, + }); + assert.equal(opened, 1); + assert.deepEqual(receipt.signaled, [102, 101], "leaf first, root last"); + assert.deepEqual(receipt.members.map((m) => m.pid), [101, 102]); + assert.equal(receipt.kills.every((k) => k.status === "signaled"), true); + assert.equal(fake.closeCount, 1, "the transaction helper is closed exactly once"); + // Every signal carried the member's captured startup identity. + assert.deepEqual(fake.kills.map((k) => k.ticks), ["102-ticks", "101-ticks"]); +}); + +test("driver: a reused member PID is never signalled again; its candidates refuse conservatively", async () => { + const rows = [row(101, 4, 0n), row(102, 101, 40n)]; + const fake = new FakeTransaction(rows); + fake.killBehavior = (pid) => { + if (pid === 102) return { status: "identity_changed", exitFileTime: null, detail: "other-ticks" }; + fake.rows = fake.rows.filter((r) => r.pid !== pid); + return { status: "signaled", exitFileTime: FT(BASE + 1_000_000n) }; + }; + // Round 2: the vanished member's child appears; ownership is unprovable. + let rounds = 0; + fake.enumerate = async () => { + rounds += 1; + if (rounds === 1) return [row(101, 4, 0n), row(102, 101, 40n)]; + if (rounds === 2) return [row(103, 102, 2_000n)]; + return []; + }; + await assert.rejects( + terminateWindowsProcessTree(101, "101-ticks", 5_000, { + open: async () => fake, + now: () => 0, + sleep: async () => undefined, + }), + (error: unknown) => { + assert.ok(error instanceof Error); + assert.match(error.message, /process_tree_parent_unprovable/); + assert.deepEqual((error as { memberPids?: number[] }).memberPids, [101, 102]); + return true; + }, + ); + assert.equal(fake.kills.some((k) => k.pid === 103), false, "the unattributable candidate is never signalled"); + assert.equal(fake.closeCount, 1); +}); + +test("driver: a member that cannot be confirmed exited fails with its recorded member set", async () => { + const rows = [row(101, 4, 0n)]; + const fake = new FakeTransaction(rows); + fake.killBehavior = () => ({ status: "wait_timeout", exitFileTime: null }); + let clock = 0; + await assert.rejects( + terminateWindowsProcessTree(101, "101-ticks", 1_000, { + open: async () => fake, + now: () => clock, + sleep: async () => { + clock += 600; + }, + }), + (error: unknown) => { + assert.match((error as Error).message, /process_tree_cleanup_unverified/); + assert.deepEqual((error as { memberPids?: number[] }).memberPids, [101]); + return true; + }, + ); + assert.equal(fake.closeCount, 1); +}); + +test("driver: an absent root refuses immediately and signals nothing", async () => { + const fake = new FakeTransaction([row(555, 4, 0n)]); + await assert.rejects( + terminateWindowsProcessTree(101, "101-ticks", 1_000, { open: async () => fake, now: () => 0, sleep: async () => undefined }), + /process_tree_root_exited/, + ); + assert.deepEqual(fake.kills, []); + assert.equal(fake.closeCount, 1); +}); + +test("driver: a wrong expected root fingerprint refuses before any signal", async () => { + const fake = new FakeTransaction([row(101, 4, 0n, "actual-ticks")]); + await assert.rejects( + terminateWindowsProcessTree(101, "different-ticks", 1_000, { open: async () => fake, now: () => 0, sleep: async () => undefined }), + /process_tree_root_identity_changed/, + ); + assert.deepEqual(fake.kills, []); +}); + +test("driver: dynamic descendants inside a killed parent's window are terminated identity-bound", async () => { + // Round 1 sees only the root; a child spawns before the root's death and + // is discovered in round 2 through the killed root's exact window. + const fake = new FakeTransaction([row(101, 4, 0n)]); + let rounds = 0; + fake.enumerate = async () => { + rounds += 1; + if (rounds === 1) fake.rows = [row(101, 4, 0n)]; + else if (rounds === 2) fake.rows = [row(102, 101, 40n)]; + else fake.rows = []; + return [...fake.rows]; + }; + const receipt = await terminateWindowsProcessTree(101, "101-ticks", 5_000, { + open: async () => fake, + now: () => 0, + sleep: async () => undefined, + }); + assert.deepEqual(receipt.signaled, [101, 102]); + assert.deepEqual(receipt.members.map((m) => m.pid), [101, 102]); +}); + +// -- recorded-tree claimant scan ---------------------------------------------- + +test("scan: clean when nothing claims the recorded pids", async () => { + const fake = new FakeTransaction([row(700, 4, 0n), row(701, 700, 10n)]); + const scan = await verifyWindowsRecordedTreeClaimants([{ pid: 101, fingerprint: "101-ticks" }], { + open: async () => fake, + now: () => 0, + sleep: async () => undefined, + }); + assert.equal(scan.clean, true); + assert.deepEqual(scan.livePids, []); + assert.equal(fake.closeCount, 1); +}); + +test("scan: a live recorded pid with a matching fingerprint is a claimant", async () => { + const fake = new FakeTransaction([row(101, 4, 0n)]); + const scan = await verifyWindowsRecordedTreeClaimants([{ pid: 101, fingerprint: "101-ticks" }], { + open: async () => fake, + now: () => 0, + sleep: async () => undefined, + }); + assert.equal(scan.clean, false); + assert.deepEqual(scan.livePids, [101]); +}); + +test("scan: a proven foreign generation of a recorded pid is excluded together with its subtree", async () => { + const fake = new FakeTransaction([row(101, 4, 0n, "reuser-ticks"), row(155, 101, 40n)]); + const scan = await verifyWindowsRecordedTreeClaimants([{ pid: 101, fingerprint: "101-ticks" }], { + open: async () => fake, + now: () => 0, + sleep: async () => undefined, + }); + assert.equal(scan.clean, true, "the reuser and its own child are not ours"); +}); + +test("scan: children of a dead recorded pid stay claimants (attribution is impossible)", async () => { + const fake = new FakeTransaction([row(180, 101, 40n)]); + const scan = await verifyWindowsRecordedTreeClaimants([{ pid: 101, fingerprint: "101-ticks" }], { + open: async () => fake, + now: () => 0, + sleep: async () => undefined, + }); + assert.equal(scan.clean, false); + assert.deepEqual(scan.livePids, [180]); +}); + +test("scan: a live recorded pid without a fingerprint stays a claimant (unknown is never clean)", async () => { + const fake = new FakeTransaction([row(101, 4, 0n)]); + const scan = await verifyWindowsRecordedTreeClaimants([{ pid: 101, fingerprint: null }], { + open: async () => fake, + now: () => 0, + sleep: async () => undefined, + }); + assert.equal(scan.clean, false); }); diff --git a/test/windows-transport-bounds.test.ts b/test/windows-transport-bounds.test.ts new file mode 100644 index 0000000..6bbc3c4 --- /dev/null +++ b/test/windows-transport-bounds.test.ts @@ -0,0 +1,202 @@ +// Review-round-1 addendum R4 (incl. 13:13 review): the PowerShell +// transaction transport must be bounded against oversized lines, endless +// streams (bytes AND line counts, including empty lines), oversized +// enumerations, expired sessions with fully buffered data, and post-failure +// data — all with FAKE streams, no OS processes. Close must verify the +// helper's actual exit code/signal. +import assert from "node:assert/strict"; +import test from "node:test"; +import { EventEmitter } from "node:events"; +import { PowerShellTreeTransaction, type TransportBounds } from "../src/adapters/windows-process-cleanup.js"; + +class FakeStream extends EventEmitter { + setEncoding(): void { + // encoding is irrelevant for string pushes in the fake + } +} + +interface CloseScript { + readonly closed: string; + readonly failed: string; + readonly exitCode: number | null; + readonly exitSignal: string | null; +} + +class FakeChild { + readonly stdout = new FakeStream(); + readonly stderr = new FakeStream(); + readonly #stdinEmitter = new EventEmitter(); + readonly pid = 42_424; + killCount = 0; + closeScript: CloseScript = { closed: "0", failed: "0", exitCode: 0, exitSignal: null }; + #closeListener: ((code: number | null, signal: string | null) => void) | null = null; + #errorListener: ((error: Error) => void) | null = null; + #swallowCommands = false; + + constructor() { + // The transport writes protocol commands; script the close reply here. + (this.#stdinEmitter as unknown as { write: (chunk: string) => void }).write = (chunk: string) => { + if (this.#swallowCommands) return; + if (chunk.trim() === "close") { + this.stdout.emit("data", `CLOSED ${this.closeScript.closed} ${this.closeScript.failed}\n`); + setImmediate(() => this.exitHelper(this.closeScript.exitCode, this.closeScript.exitSignal)); + } + }; + } + + get stdin(): NodeJS.WritableStream { + return this.#stdinEmitter as unknown as NodeJS.WritableStream; + } + + /** When set, commands receive no reply and the helper never exits. */ + swallowCommands(): void { + this.#swallowCommands = true; + } + + emitStdout(chunk: string): void { + this.stdout.emit("data", chunk); + } + + /** Simulates the helper exiting. */ + exitHelper(code: number | null = 0, signal: string | null = null): void { + this.#closeListener?.(code, signal); + } + + on(event: "close", listener: (code: number | null, signal: string | null) => void): void; + on(event: "error", listener: (error: Error) => void): void; + on(event: string, listener: (...args: never[]) => void): void { + if (event === "close") this.#closeListener = listener as (code: number | null, signal: string | null) => void; + if (event === "error") this.#errorListener = listener as (error: Error) => void; + } + + kill(): boolean { + this.killCount += 1; + // A killed helper dies: signal exit shortly after the kill lands. + setImmediate(() => this.exitHelper(null, "SIGKILL")); + return true; + } +} + +function attach(fake: FakeChild, bounds: TransportBounds): PowerShellTreeTransaction { + return PowerShellTreeTransaction.attach(fake as unknown as Parameters[0], bounds); +} + +test("transport: an oversized protocol line fails the stream bound immediately", async () => { + const fake = new FakeChild(); + const transaction = attach(fake, { maxLineBytes: 64 }); + const pending = transaction.enumerate(); + fake.emitStdout("ENUM_BEGIN\n"); + fake.emitStdout(`${"x".repeat(200)}\n`); + await assert.rejects(pending, /process_tree_transport_overflow/); +}); + +test("transport: a burst beyond the buffered-LINE budget fails even when every line is empty", async () => { + const fake = new FakeChild(); + const transaction = attach(fake, { maxBufferedLines: 50 }); + const pending = transaction.enumerate(); + // One synchronous burst: ENUM_BEGIN is consumed by the pending waiter; the + // remaining empty lines have no reader and must trip the line-count budget. + fake.emitStdout(`ENUM_BEGIN\n${"\n".repeat(200)}`); + await assert.rejects(pending, /process_tree_transport_overflow/); +}); + +test("transport: data after a failure is neither buffered nor served", async () => { + const fake = new FakeChild(); + const transaction = attach(fake, { maxLineBytes: 64 }); + const first = transaction.enumerate(); + fake.emitStdout("ENUM_BEGIN\n"); + fake.emitStdout(`${"x".repeat(200)}\n`); + await assert.rejects(first, /process_tree_transport_overflow/); + // The failed transport keeps receiving but must not resurrect. + fake.emitStdout("ENUM_BEGIN\nENUM_END\n"); + fake.emitStdout(`${"y".repeat(500)}\n`); + await assert.rejects(transaction.enumerate(), /process_tree_transport_overflow/); +}); + +test("transport: an expired session does not serve already-buffered lines", async () => { + const fake = new FakeChild(); + const transaction = attach(fake, { totalSessionMs: 120, maxEnumRows: 4_096, maxBufferedLines: 65_536 }); + const pending = transaction.enumerate(); + // Attach the rejection expectation BEFORE the budget expires so the + // mid-test rejection is always handled. + const expectation = assert.rejects(pending, /process_tree_transport_(session_)?timeout/); + fake.emitStdout("ENUM_BEGIN\nR 1 1 1 1\nR 2 1 1 1\n"); // begin consumed; rows buffered + await new Promise((resolve) => setTimeout(resolve, 200)); + await expectation; +}); + +test("transport: an enumeration beyond the row cap fails the snapshot", async () => { + const fake = new FakeChild(); + const transaction = attach(fake, { maxEnumRows: 4 }); + const pending = transaction.enumerate(); + fake.emitStdout("ENUM_BEGIN\n"); + for (let index = 0; index < 6; index += 1) fake.emitStdout(`R ${String(1_000 + index)} 1 1 1\n`); + await assert.rejects(pending, /exceeded 4 rows/); +}); + +test("transport: a stalled session hits the TOTAL budget, not just a per-line timeout", async () => { + const fake = new FakeChild(); + const transaction = attach(fake, { totalSessionMs: 150, maxEnumRows: 4_096, maxBufferedLines: 65_536 }); + const pending = transaction.enumerate(); + // The helper keeps trickling VALID rows forever; without a total budget + // each line would reset a per-line wait indefinitely. The session cap + // bounds the whole exchange. + fake.emitStdout("ENUM_BEGIN\n"); + const trickle = setInterval(() => fake.emitStdout("R 1 1 1 1\n"), 20); + try { + await assert.rejects(pending, (error: unknown) => { + assert.match((error as Error).message, /process_tree_transport_(timeout|session_timeout)/); + return true; + }); + } finally { + clearInterval(trickle); + } +}); + +test("transport: an oversized unterminated line (no newline) fails the bound", async () => { + const fake = new FakeChild(); + const transaction = attach(fake, { maxLineBytes: 64 }); + const pending = transaction.enumerate(); + fake.emitStdout("ENUM_BEGIN\n"); + fake.emitStdout("y".repeat(300)); + await assert.rejects(pending, /process_tree_transport_overflow/); +}); + +test("transport: close verifies the receipt, a clean exit, and accepts the exchange", async () => { + const fake = new FakeChild(); + const transaction = attach(fake, {}); + await assert.doesNotReject(transaction.close()); + assert.equal(fake.killCount, 0, "a clean close never needs the kill fallback"); +}); + +test("transport: close fails when the helper exits abnormally despite a valid receipt", async () => { + const fake = new FakeChild(); + fake.closeScript = { closed: "0", failed: "0", exitCode: 1, exitSignal: null }; + const transaction = attach(fake, {}); + await assert.rejects(transaction.close(), /process_tree_close_abnormal/); +}); + +test("transport: close fails when CloseHandle reports failures in the receipt", async () => { + const fake = new FakeChild(); + fake.closeScript = { closed: "2", failed: "1", exitCode: 0, exitSignal: null }; + const transaction = attach(fake, {}); + await assert.rejects(transaction.close(), /CloseHandle call\(s\) failed/); +}); + +test("transport: close fails when the helper refuses to exit and the kill fallback converges visibly", async () => { + const fake = new FakeChild(); + fake.swallowCommands(); + const transaction = attach(fake, { totalSessionMs: 250 }); + // The fake child has no real OS handle, so keep the loop alive for the + // unref'd transport timers while the bounded close runs. + const keepAlive = setInterval(() => undefined, 50); + try { + await assert.rejects(transaction.close(), (error: unknown) => { + assert.match((error as Error).message, /process_tree_transport_(session_)?timeout|process_tree_close/); + return true; + }); + } finally { + clearInterval(keepAlive); + } + assert.ok(fake.killCount >= 1, "the bounded kill fallback ran"); +}); diff --git a/test/worker-env-portability.test.ts b/test/worker-env-portability.test.ts index aecdc55..3ed2e14 100644 --- a/test/worker-env-portability.test.ts +++ b/test/worker-env-portability.test.ts @@ -2,12 +2,19 @@ // createWorkerSpawner launches a REAL detached worker process, so assertions // here cross the actual spawn boundary instead of snapshotting // createWorkerEnv. The probe worker only records non-sensitive env fields. +// Fixture lifecycle (2026-10-17 matrix2 review): receipt is not exit — the +// detached worker's cwd is the fixture data root, so the tree may only be +// removed after the owned pid is confirmed exited. The probe watches a +// release file and the shared cleanupOwnedProcessFixture gate performs the +// release, exit confirmation, and gated tree removal on every path. import assert from "node:assert/strict"; -import { mkdtempSync, readFileSync, rmSync, writeFileSync } from "node:fs"; +import { existsSync, mkdtempSync, readFileSync, writeFileSync } from "node:fs"; import { tmpdir } from "node:os"; import path from "node:path"; import test from "node:test"; import { createWorkerSpawner } from "../src/manager/spawn-worker.js"; +import { cleanupOwnedProcessFixture } from "./helpers.js"; +import { isProcessRunning } from "../src/adapters/process-spawn.js"; interface ProbeReceipt { browser_use: string | null; @@ -27,11 +34,12 @@ function envLookup(entries: Record, key: string): st return null; } -function writeProbeWorker(scriptPath: string, receiptPath: string): void { +function writeProbeWorker(scriptPath: string, receiptPath: string, releasePath: string): void { writeFileSync(scriptPath, [ '"use strict";', "const fs = require('node:fs');", `const receipt = ${JSON.stringify(receiptPath)};`, + `const release = ${JSON.stringify(releasePath)};`, "function lookup(key) {", " for (const [name, value] of Object.entries(process.env)) {", " if (name.toUpperCase() === key && typeof value === 'string' && value !== '') return value;", @@ -48,6 +56,10 @@ function writeProbeWorker(scriptPath: string, receiptPath: string): void { " zcode_bridge_keys: Object.keys(process.env).filter((name) => name.startsWith('ZCODE_BRIDGE_')).sort(),", "};", "fs.writeFileSync(receipt, JSON.stringify(snapshot, null, 2));", + "// Receipt is not exit: stay alive (holding the fixture cwd) until the", + "// test's cooperative release file appears, then exit cleanly.", + "function watchRelease() { if (fs.existsSync(release)) process.exit(0); setTimeout(watchRelease, 50); }", + "watchRelease();", ].join("\n"), "utf8"); } @@ -70,15 +82,18 @@ function restoreEnv(before: NodeJS.ProcessEnv): void { test("a real detached worker receives env-only browser settings while secrets stay filtered", { timeout: 60_000 }, async () => { const dir = mkdtempSync(path.join(tmpdir(), "bridge-worker-env-probe-")); - try { - const dataRoot = path.join(dir, "data-root"); - const receiptPath = path.join(dir, "receipt.json"); - const probeWorker = path.join(dir, "probe-worker.cjs"); - writeProbeWorker(probeWorker, receiptPath); - // An existing absolute file standing in for an explicitly configured browser. - const fakeExecutable = path.join(dir, "fake-msedge.exe"); - writeFileSync(fakeExecutable, "probe fixture", "utf8"); + const dataRoot = path.join(dir, "data-root"); + const receiptPath = path.join(dir, "receipt.json"); + const releasePath = path.join(dir, "release"); + const probeWorker = path.join(dir, "probe-worker.cjs"); + writeProbeWorker(probeWorker, receiptPath, releasePath); + // An existing absolute file standing in for an explicitly configured browser. + const fakeExecutable = path.join(dir, "fake-msedge.exe"); + writeFileSync(fakeExecutable, "probe fixture", "utf8"); + let workerPid: number | null = null; + let primaryFailed = false; + try { const envBefore = { ...process.env }; try { process.env["ZCODE_BRIDGE_BROWSER_USE"] = "headless"; @@ -96,7 +111,7 @@ test("a real detached worker receives env-only browser settings while secrets st settingsDirectory: path.join(dir, "settings"), workerEntryPath: probeWorker, }); - spawnWorker(dataRoot, "task_env_probe", 1); + workerPid = spawnWorker(dataRoot, "task_env_probe", 1).pid; } finally { restoreEnv(envBefore); } @@ -110,19 +125,25 @@ test("a real detached worker receives env-only browser settings while secrets st assert.equal(receipt.programfiles, path.join(dir, "Program Files"), "ProgramFiles must survive the worker env filter (browser auto-discovery)"); assert.equal(receipt.programfiles_x86, path.join(dir, "Program Files (x86)"), "ProgramFiles(x86) must survive the worker env filter (browser auto-discovery)"); } + } catch (error) { + primaryFailed = true; + throw error; } finally { - rmSync(dir, { recursive: true, force: true }); + await cleanupOwnedProcessFixture({ pids: workerPid === null ? [] : [workerPid], releasePath, dir, primaryFailed }); } }); test("without browser settings the worker env stays default-off (keys absent, no broad ZCODE_BRIDGE_ forwarding)", { timeout: 60_000 }, async () => { const dir = mkdtempSync(path.join(tmpdir(), "bridge-worker-env-default-")); - try { - const dataRoot = path.join(dir, "data-root"); - const receiptPath = path.join(dir, "receipt.json"); - const probeWorker = path.join(dir, "probe-worker.cjs"); - writeProbeWorker(probeWorker, receiptPath); + const dataRoot = path.join(dir, "data-root"); + const receiptPath = path.join(dir, "receipt.json"); + const releasePath = path.join(dir, "release"); + const probeWorker = path.join(dir, "probe-worker.cjs"); + writeProbeWorker(probeWorker, receiptPath, releasePath); + let workerPid: number | null = null; + let primaryFailed = false; + try { const envBefore = { ...process.env }; try { delete process.env["ZCODE_BRIDGE_BROWSER_USE"]; @@ -132,7 +153,7 @@ test("without browser settings the worker env stays default-off (keys absent, no settingsDirectory: path.join(dir, "settings"), workerEntryPath: probeWorker, }); - spawnWorker(dataRoot, "task_env_default", 1); + workerPid = spawnWorker(dataRoot, "task_env_default", 1).pid; } finally { restoreEnv(envBefore); } @@ -143,7 +164,60 @@ test("without browser settings the worker env stays default-off (keys absent, no // Only the spawner's own host-profile injection may appear: forwarding must // stay an explicit key list, not a ZCODE_BRIDGE_* namespace copy. assert.deepEqual(receipt.zcode_bridge_keys, ["ZCODE_BRIDGE_HOST_PROFILE"]); + } catch (error) { + primaryFailed = true; + throw error; + } finally { + await cleanupOwnedProcessFixture({ pids: workerPid === null ? [] : [workerPid], releasePath, dir, primaryFailed }); + } +}); + +test("receipt is not exit: a held real worker is confirmed exited before its fixture tree is removed", { timeout: 60_000 }, async () => { + // Deterministic lifecycle regression for the Windows22 matrix2 EBUSY: the + // controlled proof held a real detached worker alive after its receipt and + // reproduced EBUSY on rmdir of the data root. Here the worker provably + // stays alive after its receipt (it awaits the release file), then the + // gated cleanup confirms exit and removes the tree — never the reverse + // order. + const dir = mkdtempSync(path.join(tmpdir(), "bridge-worker-env-held-")); + const dataRoot = path.join(dir, "data-root"); + const receiptPath = path.join(dir, "receipt.json"); + const releasePath = path.join(dir, "release"); + const probeWorker = path.join(dir, "probe-worker.cjs"); + writeProbeWorker(probeWorker, receiptPath, releasePath); + + let workerPid: number | null = null; + let primaryFailed = false; + let settledByPrimaryCleanup = false; + try { + const envBefore = { ...process.env }; + try { + delete process.env["ZCODE_BRIDGE_BROWSER_USE"]; + delete process.env["ZCODE_BRIDGE_BROWSER_EXECUTABLE"]; + const spawnWorker = createWorkerSpawner({ + name: "worker-env-held-host", + settingsDirectory: path.join(dir, "settings"), + workerEntryPath: probeWorker, + }); + workerPid = spawnWorker(dataRoot, "task_env_held", 1).pid; + } finally { + restoreEnv(envBefore); + } + assert.ok(workerPid !== null, "the spawner must report the owned worker pid"); + + const receipt = await waitForReceipt(receiptPath, 30_000); + assert.ok(receipt, "the held worker wrote its receipt"); + assert.equal(isProcessRunning(workerPid), true, "receipt is not exit: the held worker is still alive in its fixture cwd"); + await cleanupOwnedProcessFixture({ pids: [workerPid], releasePath, dir, primaryFailed: false, timeoutMs: 15_000 }); + assert.equal(isProcessRunning(workerPid), false, "the owned worker reports exited after the release signal"); + assert.equal(existsSync(dir), false, "the fixture tree is removed only after the exit confirmation"); + settledByPrimaryCleanup = true; // the inner gated call reaped the worker and removed the tree + } catch (error) { + primaryFailed = true; + throw error; } finally { - rmSync(dir, { recursive: true, force: true }); + if (!settledByPrimaryCleanup) { + await cleanupOwnedProcessFixture({ pids: workerPid === null ? [] : [workerPid], releasePath, dir, primaryFailed }); + } } }); From 73d4a731c5e8d34e4ff6f8ec146eb07c8598778b Mon Sep 17 00:00:00 2001 From: Bill Date: Sun, 18 Oct 2026 16:28:09 +0800 Subject: [PATCH 2/2] fix: preserve claimant evidence across process generations --- docs/ARCHITECTURE.md | 6 +- docs/PROJECT_STATE.md | 14 +- docs/README.md | 9 +- ...windows-identity-bound-tree-termination.md | 48 +- .../2026-10-18-pr47-claimant-repair.md | 52 ++ plugins/codex-zcode-bridge/dist/bridge.mjs | 127 +++- .../codex-zcode-bridge/worker/worker-main.mjs | 8 +- src/adapters/process-spawn.ts | 35 +- src/adapters/windows-process-cleanup.ts | 272 ++++++- src/manager/task-manager.ts | 29 +- test/manager-claimant-gate.test.ts | 386 ++++++++++ test/manager-helpers.ts | 21 +- test/windows-process-cleanup.test.ts | 679 +++++++++++++++++- 13 files changed, 1601 insertions(+), 85 deletions(-) create mode 100644 docs/reports/2026-10-18-pr47-claimant-repair.md create mode 100644 test/manager-claimant-gate.test.ts diff --git a/docs/ARCHITECTURE.md b/docs/ARCHITECTURE.md index 5d0fbd7..4f4b92a 100644 --- a/docs/ARCHITECTURE.md +++ b/docs/ARCHITECTURE.md @@ -1,8 +1,8 @@ # 当前架构 > Status: AUTHORITATIVE -> Last updated: 2026-10-17 -> Last verified: 2026-10-17，Windows 进程树身份绑定终止（ADR-009）经本机 Node 22.18.0/24.19.0 全量矩阵与 Codex 独立反例验证，证据在私有归档；其余沿承原核对范围。 +> Last updated: 2026-10-18 +> Last verified: 2026-10-18，PR #47 claimant 扫描返修（P1-A/P1-B、逐边持有证明的外来代排除、hold 响应校验、终止预算语义）经专项、Manager baseline-RED 复验与 evaluator 契约探针回归（worker 自测；Codex 独立矩阵待运行），证据索引见[公开返修报告](reports/2026-10-18-pr47-claimant-repair.md)；其余沿承 2026-10-17 的 ADR-009 全量矩阵核对范围。 > Verified against: 0902236c488adf7a3280dfdc38401dfa9f41cb49 + 本分支修复；受测源码指纹见各修复记录。 本文件描述当前代码，不是历史阶段的冻结设计。公共类型见 `src/interfaces.ts`，MCP schema 见 `src/mcp/schemas.ts`；共享边界见 [SHARED_CORE.md](SHARED_CORE.md)。 @@ -41,7 +41,7 @@ Manager 用进程内 promise 队列以及同一 data root 下的 `.tasks/.manage 同一 data root 内，FIFO 队列按 worker 上限和执行路径重叠规则启动 detached worker。Spawner 显式传 attempt。worker 在进入 adapter 前通过 attempt 目录的永久 `execution.claim` 抢占执行权，拒绝重复、旧 attempt 和终态入场。`state.lock` 保护状态更新和结果提交；worker 提交还校验当前 attempt/非终态。Manager 的延迟 PID 写入只作用于仍 running 的同一 attempt。 -未启动 worker 可重拉一次；抢占过的 attempt 不会重复执行。已开始的 worker 不自动重跑。worker 丢失但记录的 ZCode PID 仍存活时保留占用，要求 `zcode_cancel` 验证清理。清理失败的终态任务也保留目录与 slot；续跑被拒绝，再次 cancel 或只读恢复探测确认退出后才释放，原 failed 结果不改写。进程恢复使用 PID 和启动 fingerprint；unknown 不当作已退出。Windows 强制清理按 ADR-009 执行：一个持久 PowerShell 事务 helper 为每个已验证成员持有原生句柄（持有期间 PID 不可复用），归属按创建时间表示区间与句柄精确时间零容差证明（早于父创建的旧记录在触碰前排除；同 100ns 不可区分即拒绝；自然退出父、未确认退出保守拒绝）；Windows 终止必须携带调用者期望根指纹，信号仅经"同一句柄内验证、同一句柄内 TerminateProcess"发出，不使用 taskkill /T；未解析 hold、候选消失、传输异常、helper 关闭失败均携带完整已知成员集合显式失败并保留占用。已记录的 tree-cleanup 失败以 `cleanup_unverified` 锁存：恢复与 worker 提交不得抹除证据；部分或无法恢复完整成员的记录保守阻断，完整 marker 回退及 legacy 无失败记录的 probe 恢复按 ADR-009 具体分支执行；已记录树失败的正常释放由后续 cancel 的新鲜 probe 与 claimant 扫描门控。公共错误使用稳定诊断代码，不转发本地化 taskkill 文本。已记录 PID 的 claimant 闭包外（未观察中间祖先）不可证明，不以完整清理名义宣称。见 ADR-005、ADR-009。 +未启动 worker 可重拉一次；抢占过的 attempt 不会重复执行。已开始的 worker 不自动重跑。worker 丢失但记录的 ZCode PID 仍存活时保留占用，要求 `zcode_cancel` 验证清理。清理失败的终态任务也保留目录与 slot；续跑被拒绝，再次 cancel 或只读恢复探测确认退出后才释放，原 failed 结果不改写。进程恢复使用 PID 和启动 fingerprint；unknown 不当作已退出。Windows 强制清理按 ADR-009 执行：一个持久 PowerShell 事务 helper 为每个已验证成员持有原生句柄（持有期间 PID 不可复用），归属按创建时间表示区间与句柄精确时间零容差证明（早于父创建的旧记录在触碰前排除；同 100ns 不可区分即拒绝；自然退出父、未确认退出保守拒绝）；Windows 终止必须携带调用者期望根指纹，信号仅经"同一句柄内验证、同一句柄内 TerminateProcess"发出，不使用 taskkill /T；未解析 hold、候选消失、传输异常、helper 关闭失败均携带完整已知成员集合显式失败并保留占用。已记录的 tree-cleanup 失败以 `cleanup_unverified` 锁存：恢复与 worker 提交不得抹除证据；部分或无法恢复完整成员的记录保守阻断，完整 marker 回退及 legacy 无失败记录的 probe 恢复按 ADR-009 具体分支执行；已记录树失败的正常释放由后续 cancel 的新鲜 probe 与 claimant 扫描门控。公共错误使用稳定诊断代码，不转发本地化 taskkill 文本。claimant 扫描（PR #47 返修后语义）：存活记录 PID 指纹匹配即 claimant；枚举与 hold 之间消失的记录 PID 视为已死，其存活子代计入；已证外国换代的 PID 本身排除，其后代仅在每条边持有证明成立时排除——边的父代被持有且与枚举行同代一致、子代创建表示区间严格晚于该持有精确创建时间、子代自身再被持有核验后其更深后代递归同判；相等精度、更早、跨界、未知时间，或中间代消失/拒绝/换代，一律保持占用。多记录根证据单调合并：相对某一记录根证明外来不抹除相对另一记录根的 claimant 路径（如已死记录成员的存活子代复用了另一记录 PID），claimant 证据优先；记录目标自身的外来判定须行/句柄同代一致才可用该行作证，且仍接受同一严格入边检验（缓存句柄不是全局排除证书，严格更晚的正例复用缓存证据不重复 hold）。hold 响应按请求校验（数量/PID/精确身份字段），协议畸形拒绝扫描；行/句柄代矛盾按归属未证明保持占用。扫描 false 或拒绝时的任务相关观察（原始记录目标、请求过但未证实的候选、claimant；不新增已证外来后代候选）经既有 cleanup-failure 记录与 marker 单调持久化，后续 cancel 与重建后的扫描取并集；已观察中间代不属未观察祖先边界。kill 预算只约束等待退出：截止仅对无句柄确认退出时间的成员报 process_tree_cleanup_unverified，已确认退出的成员继续闭包确认轮。已记录 PID 的 claimant 闭包外（未观察中间祖先）不可证明，不以完整清理名义宣称。见 ADR-005、ADR-009。 worker 每 3 秒原子写入一次绑定 attempt 与 PID 的私有 heartbeat，字段含 session、turn、Bridge event 序号和已观测到的 ZCode event 序号。管理器遇到一次负向 PID 探测时，若 heartbeat 不超过 15 秒则暂缓失联判定。ZCode turn 完成后，worker 在清理进程前写入 outcome checkpoint，并在清理成功后更新验证标志；worker 在提交最终 result 之前退出时，管理器可据 checkpoint 恢复报告，清理未验证则保留 `cleanup_failed` 与 workspace 占用。 diff --git a/docs/PROJECT_STATE.md b/docs/PROJECT_STATE.md index 4824466..82d019a 100644 --- a/docs/PROJECT_STATE.md +++ b/docs/PROJECT_STATE.md @@ -1,9 +1,9 @@ # Project State > Status: AUTHORITATIVE -> Last updated: 2026-10-17 -> Last verified: 2026-10-17，本轮 Windows 进程树身份绑定终止（ADR-009）经本机 Node 22.18.0（461 项 460 pass/1 skip）与 24.19.0（459 pass/2 skip）全量矩阵、94 项专项单测、有界真实 fixture 4/4 与 Codex 九脚本十用例独立反例验证，证据在本轮私有归档；前轮文件系统根守卫的独立矩阵与 Linux 回归见[修复记录](reports/2026-10-17-filesystem-root-guards.md)，Linux 侧对本轮 ADR-009 改动的回归由 Codex 收尾执行。 -> Verified against: 0902236c488adf7a3280dfdc38401dfa9f41cb49 + 本分支修复；受测源码指纹见修复记录与本轮私有归档。 +> Last updated: 2026-10-18 +> Last verified: 2026-10-18，PR #47 返修行为已由 Codex 独立验收：冻结候选上六组 fake 探针（契约 10、已死父跨根 4、记录外来边 5、根行代 3、观察历史 2 模式、真实 Manager 重建 2 模式）× Node 22.18.0/24.16.0 全过；完整独立平台矩阵——Windows 22.18.0 全量 509/508/1 skip、Windows 24.16.0 全量 509/507/2 skip、Windows 24.19.0 受影响九套件 152/152/0、Linux 22.18.0 全量 511/508/3 skip、Linux 24.20.0 全量 511/507/4 skip，均 0 fail，各平台 typecheck/build/validate:plugin/clean-entrypoints(4/4) 通过（git diff --check 于 Windows 工作区通过；Linux 首次聚合 bundle 比较失败为评估器符号链接致来源注释差异，实体依赖重建后字节一致并 validate 通过，原失败保留）；跳过项为平台专有用例如实记录。维护者机器特定 process-lock 失败未在本机自然复现，仅机制修复有确定性证据（见 ADR-009 与返修报告）。本轮（attempt-7）仅注释/文档/记录收尾，源码行为字节不变（bundle 重建后字节一致）。worker 分版本自测计数见各 attempt 记录，与独立结果不混计；最终源码冻结、任务资源清理与 PR 更新由 Codex 负责。 +> Verified against: 628984ca625a4d307b859d9f32d49e52e2d21ee6 + 本分支修复；受测源码指纹见修复记录与本轮私有归档。 当前状态快照，回答"现在什么能用、什么不能用"。路线图和优先级不在这里。 @@ -17,15 +17,17 @@ ## 本分支改动 -本分支基于上游 1.2.6（0902236），加入两项修复。 +本分支基于上游 1.2.6（0902236），加入三项修复。 Windows 文件系统时序与根守卫（前轮）：`state.lock` 发布前存在性预检(已存在的空/未知 owner 锁不被 POSIX rename 覆盖)、发布 rename 的 EEXIST/ENOTEMPTY 按确定目的在场竞争移交 existing-owner 策略(不逃逸、不烧重试预算;EPERM/EACCES/EBUSY 目的缺失保持有界重试后可见抛出)、撤回 rename 对白名单瞬时码有界重试(撤回耗尽后锁保持完整、存活 owner 不驱逐;非瞬时码与共享 reclaim guard 清理失败保持可见,guard 清理在全部出口单点执行);worker 的迟到 self/runtime identity 写盘在终态前有界收束、超时丢弃,不再复活已清理的 store;测试 fixture 树清理带 Node 文档化有界重试;`buildRepoRootLeakPattern` 拒绝文件系统根输入（POSIX 根、盘符根及其词法等价写法、UNC 共享根）并修复 URL 主机名等于短根的误报。版本号未提升，不代表已发布；证据与边界见[修复记录](reports/2026-10-17-filesystem-root-guards.md)。 -Windows 进程树身份绑定归属与终止（本轮，ADR-009）：生产 Windows 清理重写为单持久 PowerShell 事务——每个已验证成员持有原生句柄（持有期间 PID 不可复用），归属按 CIM 表示区间与句柄精确创建/退出时间零容差证明，早于父创建的旧记录在触碰前排除；信号仅在"同一句柄内 GetProcessTimes 验证、同一句柄内 TerminateProcess"发出，`taskkill /T` 移除；Windows 终止必须携带调用者期望根指纹（spawn 代捕获/持久化 identity），POSIX 语义不变。未解析 hold、候选消失、传输异常、helper 关闭失败均携带完整已知成员集合显式失败；manager 端 `cleanup_unverified` 单调锁存（恢复路径、worker 晚到提交、连续失败、记录损坏/截断均不得抹除），释放由后续 cancel 的新鲜 probe 与 recorded-tree claimant 扫描门控。传输有界（1 MiB/行、8 MiB/65536 行缓冲、4096 枚举行、120 s 会话）。 +Windows 进程树身份绑定归属与终止（ADR-009，2026-10-17 轮）：生产 Windows 清理重写为单持久 PowerShell 事务——每个已验证成员持有原生句柄（持有期间 PID 不可复用），归属按 CIM 表示区间与句柄精确创建/退出时间零容差证明，早于父创建的旧记录在触碰前排除；信号仅在"同一句柄内 GetProcessTimes 验证、同一句柄内 TerminateProcess"发出，`taskkill /T` 移除；Windows 终止必须携带调用者期望根指纹（spawn 代捕获/持久化 identity），POSIX 语义不变。未解析 hold、候选消失、传输异常、helper 关闭失败均携带完整已知成员集合显式失败；manager 端 `cleanup_unverified` 单调锁存（恢复路径、worker 晚到提交、连续失败、记录损坏/截断均不得抹除），释放由后续 cancel 的新鲜 probe 与 recorded-tree claimant 扫描门控。传输有界（1 MiB/行、8 MiB/65536 行缓冲、4096 枚举行、120 s 会话）。 + +PR #47 claimant 扫描与预算返修（本轮，2026-10-18，含 review-round-1 R1–R4 与 review-round-2 S1/S2 返工）：S1——多记录根证据单调合并，claimant 路径优先于相对其他根的外来排除；S2——扫描 false/拒绝时任务相关观察经既有 cleanup-failure/marker 持久化并入后续 scanTargets（不含已证外来者与响应载荷任意 PID），空闭包正常释放。其余：P1-A——枚举与 hold 之间消失的记录 PID 视为已死，其存活子代计入 claimant，不再被过期枚举 live 集合漏掉；P1-B——外来换代父的后代仅在每条边持有证明成立时排除（边的父代被持有且与枚举行同代一致、子代创建表示区间严格晚于该持有精确创建时间、子代自身再被持有核验后递归同判），相等精度/更早/跨界/未知/中间代消失-拒绝-换代的子代保持占用，"外来换代父排除整棵子树"的旧保证撤回；hold 响应按请求校验（数量、PID、ok 必含可用精确身份字段），畸形证据拒绝扫描而非证明 clean；终止预算语义——kill 截止只对无句柄确认退出时间的成员报 `process_tree_cleanup_unverified`，全部已确认退出的成员继续闭包确认轮直至证明为空（仍受 120 s 传输会话预算约束），修复"全表枚举慢 + WMI 滞留已终止成员行"时对已证明退出的误报。维护者 Node 24.16.0 两项 process-lock 失败在本机 24.16.0 未复现（阶段计时与边界见 ADR-009 与[公开返修报告](reports/2026-10-18-pr47-claimant-repair.md)），不宣称版本归因或根治。 ## 稳定 -- Windows 进程树终止（ADR-009）：本机 Node 22/24 全量、专项单测 94 项、有界真实 fixture（正向身份绑定终止、错误指纹拒绝且零信号、缺身份拒绝、独立收束）与 Codex 九脚本十用例反例全部通过；claimant 闭包外（未观察中间祖先）不可证明，不以完整清理名义宣称。 +- Windows 进程树终止（ADR-009）：本机 Node 22/24 全量、专项单测、有界真实 fixture（正向身份绑定终止、错误指纹拒绝且零信号、缺身份拒绝、独立收束）与 Codex 九脚本十用例反例通过；2026-10-18 PR #47 返修后维护者原始反例、组合矩阵、真实 Manager 消费门禁与预算语义回归通过（attempt-1 版本 44/3/93/19 项为分文件历史计数；attempt-2 版本为 53/4 项加 evaluator 探针 10/10×3 Node）。claimant 闭包外（未观察中间祖先）不可证明，外来换代父不排除未证明的后代，不以完整清理名义宣称。 - TaskManager 生命周期：`queued` / `running` / `completed` / `failed` / `cancelled` / `waiting_for_master`，含 attempt 独占、续跑归档、清理验证。 - MCP 默认工具 13 个，清单见 `INTERFACES.md`。 - `zcode_events` 的增量事件、cursor 与 raw/summary 视图。 diff --git a/docs/README.md b/docs/README.md index 8457aab..a3c6d77 100644 --- a/docs/README.md +++ b/docs/README.md @@ -1,7 +1,7 @@ # 文档索引 / Documentation > Status: AUTHORITATIVE（仅指本索引） -> Last updated: 2026-10-17 +> Last updated: 2026-10-18 英文使用者从 [README.md](../README.md) 开始。技术文档为中文。本文是文档路由：先看状态等级，再按读者路径选文档。 @@ -48,8 +48,8 @@ English readers start at [README.md](../README.md). This index is bilingual; the |---|---|---|---| | [README.md](../README.md) | en | 2026-10-05 | 未记录 | | [README.zh-CN.md](../README.zh-CN.md) | zh | 2026-10-03 | 未记录 | -| [PROJECT_STATE.md](PROJECT_STATE.md) | zh | 2026-10-17 | 部分核对；详见文档顶部 | -| [ARCHITECTURE.md](ARCHITECTURE.md) | zh | 2026-10-14 | 部分核对；详见文档顶部 | +| [PROJECT_STATE.md](PROJECT_STATE.md) | zh | 2026-10-18 | 部分核对；详见文档顶部 | +| [ARCHITECTURE.md](ARCHITECTURE.md) | zh | 2026-10-18 | 部分核对；详见文档顶部 | | [INTERFACES.md](INTERFACES.md) | zh | 2026-10-14 | 部分核对；详见文档顶部 | | [SHARED_CORE.md](SHARED_CORE.md) | zh | 2026-10-06 | 未记录 | | [ZCODE_RUNTIME.md](ZCODE_RUNTIME.md) | zh | 2026-10-14 | 2026-10-08，本地配置/假运行时 | @@ -70,7 +70,7 @@ English readers start at [README.md](../README.md). This index is bilingual; the | [ADR-004](decisions/ADR-004-separate-control-state-and-observation.md) | 2026-10-07 | Accepted：分离运行时控制、权威状态与补充观察 | | [ADR-005](decisions/ADR-005-long-session-reliability.md) | 2026-10-08 | Accepted：长会话清理、锁释放、报告修复与 provider 兼容 | | [ADR-006](decisions/ADR-006-provider-quota-failure-diagnostics.md) | 2026-10-14 | Accepted：安全识别并明确提示 provider 用量额度耗尽 | -| [ADR-009](decisions/ADR-009-windows-identity-bound-tree-termination.md) | 2026-10-17 | DECISION：Windows 进程树身份绑定归属与终止 | +| [ADR-009](decisions/ADR-009-windows-identity-bound-tree-termination.md) | 2026-10-18 | DECISION：Windows 进程树身份绑定归属与终止（含 PR #47 返修语义） | | [decisions/roadmap-decisions-2026-09-27.md](decisions/roadmap-decisions-2026-09-27.md) | 2026-09-27 | 路线图与决策讨论 | | [decisions/reliability-repair-plan-v2-2026-10-03.md](decisions/reliability-repair-plan-v2-2026-10-03.md) | 2026-10-03 | 可靠性修复计划，含未完成项 | @@ -102,6 +102,7 @@ English readers start at [README.md](../README.md). This index is bilingual; the | [reports/task-feedback-v0-1-implementation-2026-10-06.md](reports/task-feedback-v0-1-implementation-2026-10-06.md) | Task Feedback v0.1 的交付报告，一次性材料 | | [reports/2026-10-08-long-session-reliability.md](reports/2026-10-08-long-session-reliability.md) | 长会话故障证据、本地修复、验证与未运行边界 | | [reports/2026-10-17-filesystem-root-guards.md](reports/2026-10-17-filesystem-root-guards.md) | Windows 锁时序重试、teardown 容错与文件系统根守卫的证据、修复与边界 | +| [reports/2026-10-18-pr47-claimant-repair.md](reports/2026-10-18-pr47-claimant-repair.md) | PR #47 claimant 扫描 P1 返修、逐边持有证明、hold 校验与预算语义的证据与边界 | ### 自动生成 Generated diff --git a/docs/decisions/ADR-009-windows-identity-bound-tree-termination.md b/docs/decisions/ADR-009-windows-identity-bound-tree-termination.md index 71c3ea8..470ec2d 100644 --- a/docs/decisions/ADR-009-windows-identity-bound-tree-termination.md +++ b/docs/decisions/ADR-009-windows-identity-bound-tree-termination.md @@ -1,9 +1,11 @@ # ADR-009 Windows 进程树身份绑定归属与终止 > Status: DECISION -> Last updated: 2026-10-17 -> Last verified: 2026-10-17（实现后本机验证：Windows Node 22.18.0 全量 461 项 460 pass/1 skip、Node 24.19.0 全量 461 项 459 pass/2 skip，均 0 fail；Codex 反例脚本 9 个（10 个运行用例，recovery-latch 含两种模式）全部 ok；有界真实 fixture 4/4。证据与命令输出存于本轮私有证据归档 `local/windows-process-ownership-20261017/worker/attempt-2/`，不在源码仓库内） -> Verified against: 0902236c488adf7a3280dfdc38401dfa9f41cb49 + 本分支修复（含 review-round-1 R1–R4、review-round-1-addendum（absent 候选、传输预算、missing/truncated/repeated/repeated-truncated 成员证据）、review-round-2-callers C1 的全部返工） +> Last updated: 2026-10-18 +> Last verified: 2026-10-18（attempt-6 行为修复经 Codex 独立验收：六组 fake 探针 × Node 22.18.0/24.16.0 全过；完整平台矩阵 Windows 22.18.0/24.16.0 全量 509 项（508/507 pass、1/2 平台专有 skip）、Windows 24.19.0 受影响九套件 152/152、Linux 22.18.0/24.20.0 全量 511 项（508/507 pass、3/4 平台专有 skip）均 0 fail，typecheck/build/validate:plugin/clean-entrypoints 通过——Linux 首次聚合 bundle 比较失败为评估器符号链接致来源注释差异，实体依赖重建后字节一致、原失败保留；维护者机器特定失败未复现边界保持。attempt-7 仅注释/文档收尾，行为字节不变。详见[公开返修报告](../reports/2026-10-18-pr47-claimant-repair.md)；原始输出存任务私有归档） +> Verified against: 628984ca625a4d307b859d9f32d49e52e2d21ee6 + 本分支修复（含 review-round-1 R1–R4、review-round-1-addendum、review-round-2-callers C1 返工与 2026-10-18 PR #47 claimant/预算返修） + +> 编号说明：公开基线 0902236c488adf7a3280dfdc38401dfa9f41cb49 的 `docs/decisions/` 只有 ADR-001..008；"ADR-009" 是本 PR 分支自选编号。维护者本地整合若已占用 009，最终编号由维护者在整合时协调重排，本文件不据未公开源码猜测。 ## 背景 @@ -44,7 +46,8 @@ - 多轮：枚举→归属→（无存活成员即成功；否则逐个句柄绑定终止）→下一轮。动态后代经证明窗口同样被绑定终止；不可证明即保守失败。verified 只由"闭包内无存活成员"给出，根退出不构成证据。 - manager：终止调用携带期望根指纹；任何终止失败 → `CANCEL_FAILED` + `cleanup_unverified` 锁存 + 成员记录；**对已记录 tree-cleanup 失败（记录或完整 marker 存在）的任务，恢复路径（persistedResult/checkpoint/worker_lost/terminal probe）在锁存期间保持占用**；无记录无 marker 的 legacy 终态锁存仍由根 probe 释放。worker 终态提交（`TaskStore.commitWorkerResult`）可保存结果但不得清锁存或覆盖成员证据（无论 patch 清除、重申还是省略该字段）；释放只能由后续 cancel 经新鲜 probe + recorded-tree claimant 扫描门控完成。 -- claimant 扫描的 `clean` 证明："**已记录 PID 名下没有存活的、属于我们或不可归属的 claimant**"——含指纹匹配的存活记录 PID 本身、其存活子代（传递闭包）、已死记录 PID 的存活子代（归属不可判定，保守计入）；已证外国换代的存活 PID 及其子树排除。未观察到的中间祖先在任何证明之外，调用方须记录该边界而非宣称完整。 +- claimant 扫描的 `clean` 证明："**已记录 PID 名下没有存活的、属于我们或不可归属的 claimant**"——含指纹匹配的存活记录 PID 本身、其存活子代（传递闭包）、已死记录 PID 的存活子代（归属不可判定，保守计入）；枚举与 hold 之间消失的记录 PID 视为已死，其存活子代同样计入（PR #47 P1-A：不得以过期枚举 live 集合证明 clean）。已证外国换代的存活 PID 本身排除，但其后代**逐边持有证明**后才排除（review-round-1 R1）：每条边的父代必须被持有（同句柄精确创建时间，且与枚举行同代一致），子代 CIM 表示区间**严格晚于**该精确创建时间才可证明属于外来代，随后子代自身也必须被持有且行代一致，其更深后代再依其持有精确时间递归判定；相等精度（表示区间触界）、更早、跨界、创建时间未知，以及中间代 absent/denied/换代/行句柄矛盾，一律保持占用。hold 响应按请求逐一校验（数量、PID、ok 必含可用精确身份字段），畸形证据拒绝扫描而非证明 clean（review-round-1 R2）。观察证据持久化（review-round-2 S2）：扫描 false 或拒绝时，任务相关观察（原始记录目标——即使某目标当前代已证换代，目标 PID 本身仍保留供按记录身份复扫；claimant；请求过但未证实的外来谱系候选；不新增已证外来的后代候选，不采信响应载荷任意 PID）经既有 cleanup-failure 记录与 marker 单调并入，后续 cancel/恢复/Manager 重建后的 scanTargets 取并集；已观察过的中间代不属于"未观察祖先"免责边界。空闭包最终仍正常释放。外来换代父不排除任何未经此证明的后代（PR #47 P1-B："外来换代父排除整棵子树"不是成立保证）。多记录根的证据单调合并（review-round-2 S1，review-round-3 T1 收紧）：某 PID 相对根 A 的记录证明为外来换代，不抹除它相对根 B 的 claimant 路径——它是已死记录成员的存活子代、claimant 后代或不可归属谱系后代时，claimant 证据优先，占用保持；排除只适用于对全部记录根都无 claimant 路径的 PID。记录目标自身的外来判定须先通过行/句柄同代一致（ticks 相等且精确时间落在行表示区间内）才可用该行为换代作证，否则按归属未证明保持占用；已判外来的记录目标仍按同一严格边规则接受入边检验（创建不严格晚于外来父的持有精确时间即降级为 claimant），缓存句柄不是全局排除证书；严格更晚且行句柄一致时复用缓存证据不重复 hold。行/句柄代矛盾（ticks 不符或精确时间出表示区间）按归属未证明保持占用，不按协议畸形抛错；协议畸形（响应数量、PID 对应、ok 缺精确身份字段或非十进制格式）才抛 process_tree_hold_invalid 拒绝扫描（review-round-2 复核口径）。未观察到的中间祖先在任何证明之外，调用方须记录该边界而非宣称完整。 +- 终止预算语义（PR #47 process-lock 返修）：kill 预算约束的是**等待退出**，不是已在持有句柄上证明的退出。截止时刻只对无确认退出 FILETIME 的成员触发 `process_tree_cleanup_unverified`；全部成员均已确认退出时继续确认轮直至闭包证明为空，整体仍受传输会话预算约束。已证明的机制：本机实测单次全表枚举 1.1–1.5s、两次枚举即占 5s 预算约六成，旧代码在枚举耗时或 WMI 滞留已终止成员行时会对"已确认退出"的成员误报 not provably exited（确定性回归覆盖）；维护者 24.16.0 失败在本机未复现，不做版本或根治宣称。 ### D4 调用链身份交接（每代状态） @@ -75,6 +78,43 @@ - Codex 反例脚本（存私有证据归档 evaluator/，新 attempt id 可重跑）：9 个脚本 / 10 个运行用例全部 ok——recovery-latch-probe（recover 与 corrupt-meta 两模式）、adversarial-cleanup、absent-candidate、missing-members、truncated-members、repeated-failure、repeated-truncated、worker-commit-latch、legacy-retry。 - 全量：Windows Node 22.18.0 `node scripts/test-isolated.mjs` 461 项 460 pass/1 skip；Node 24.19.0 同命令 459 pass/2 skip；`npm run typecheck`、`npm run build`、`npm run validate:plugin`、`git diff --check` 均 exit 0。 +## PR #47 返修补充验证 — attempt-1（2026-10-18 源码版本；原始输出存任务私有归档 attempt-1/，下列计数属该版源码，勿与 attempt-2 计数叠加） + +- 维护者原始两 fake 反例（absent-after-enumeration、reused-parent-old-child）在反馈 head 记录 RED（clean=true 缺陷复现，exit 1），修复后同脚本 GREEN（clean=false、livePids=[102]，exit 0），零信号。 +- `node --test dist-test/test/windows-process-cleanup.test.js`：44/44（含 P1-A/P1-B 及组合矩阵、边界零容差、预算语义回归）。 +- `node --test dist-test/test/manager-claimant-gate.test.js`：3/3——真实 TaskManager 消费路径（cancel→probe→锁存→扫描门控→pump），fake 只作 OS 传输；false/reject 保持 `cleanup_unverified`、同目录后续任务不放行，可证明 clean 正常释放。 +- 受影响套件 `node --test dist-test/test/{windows-process-cleanup,windows-cleanup-review-regressions,manager-recovery-latch,manager-claimant-gate,task-manager}.test.js`：93/93；`process-lock` 全量 19/19。 +- 便携 Node 24.16.0（官方 SHASUMS256 校验）下两 process-lock 用例 2/2（修复前后各一次）；阶段计时探针（自建登记进程、信号前成员身份门禁）：24.16.0 与 24.19.0 各 8/8，terminate 全程 2.8–3.3s / 预算 5s，单次全表枚举 1.1–1.5s；维护者失败未在本机自然复现，边界如实记录。 +- 预算机制 RED→GREEN：head 驱动对"已确认退出成员的滞留行 + 截止超时"抛 `process_tree_cleanup_unverified`（缺陷复现），修复后 3 轮完成闭包证明。 + +## PR #47 返修补充验证 — attempt-2 / review-round-1 R1–R4（2026-10-18 源码版本；以下均为 worker 自测，原始输出存任务私有归档 attempt-2/；本轮源码全量 492/490 pass/2 skip，受影响七套件 128/128，typecheck/build/validate:plugin/clean-entrypoints/git diff --check exit 0） + +- evaluator 契约探针（10 例，含 6 个 R1/R2 失败反例与 2 个正向对照）经 worker 在 Node 22.18.0 / 24.16.0 / 24.19.0 各 10/10 通过，零信号、helper 各关闭一次。 +- `node --test dist-test/test/windows-process-cleanup.test.js`（attempt-2 源码）：53/53——新增中间代 absent/denied/换代/行句柄矛盾保持占用、相等精度不判外来、被排除成员必须被持有且行代一致、R2 形状违规（错 PID、超量、缺精确字段、非数字时间）拒绝扫描。 +- `node --test dist-test/test/manager-claimant-gate.test.js`（attempt-2 源码）：4/4——P1-A 经指纹化记录根真正触发 absent hold（断言 hold 发生）；maxConcurrency=1 且不同执行目录验证 slot 保留；reject/畸形 hold 同路径保持占用；可证明 clean 释放并放行排队任务。 +- baseline-loader 复验：同一 Manager 测试在原缺陷扫描器（head 转译）下 4/4 全部 "Missing expected rejection" RED——包括 review-round-1 指出原先误 PASS 的 P1-A。 +- R1/R2 未改变终止驱动与预算语义；attempt-1 的预算机制独立复验（6 例）与 24.16.0 未复现边界继续有效，不重复执行真实阶段探针。 + +## PR #47 返修补充验证 — attempt-3 / review-round-2 S1/S2（2026-10-18 源码版本；以下均为 worker 自测，原始输出存任务私有归档 attempt-3/） + +- evaluator `probe-recorded-roots.mjs`（只读复用）S1 四例 × Node 24.19.0/24.16.0/22.18.0：跨根 claimant（含目标序反转、孙代传播）保持占用，全外来正向释放。 +- evaluator `probe-manager-observed-history.mjs`（只读复用，新编译 dist-test）S2 两模式（正负检查）× 同三版本全过：false 与拒绝两路径均持久化观察，第二次 scanTargets 含并入 PID，task_2 保持 queued，空表后正常释放且 task_2 转 running。 +- 单元 `windows-process-cleanup` 59/59（新增 S1 四例与 S2 观察证据两例）；Manager 门禁 `manager-claimant-gate` 7/7（新增 S1 跨根、S2 false 持久化、S2 拒绝持久化），baseline-loader 复验原扫描器下 6/7 RED（S2-false 模式由 Manager 侧 livePids 兜底在旧扫描器下亦过；该模式旧候选 RED 由评估方 round2 探针证明）。 +- 受影响十套件 175 tests/174 pass/0 fail/1 skip；全量 501 tests/499 pass/0 fail/2 skip；typecheck/build/validate:plugin/clean-entrypoints(4/4)/git diff --check exit 0。前轮计数见 attempt-1/attempt-2 节，按版本分开不叠加。 + +## PR #47 返修补充验证 — attempt-6 / review-round-3 T1（2026-10-18 源码版本；以下均为 worker 自测，原始输出存任务私有归档 attempt-6/；attempt-4/5 为 provider 额度中断，无实现产出，失败由调用方记录保留） + +- evaluator `probe-recorded-foreign-edges.mjs`（只读复用）5/5 × Node 22.18.0/24.16.0：修复前当前源码记录 RED（4 负例 clean:true、1 正例过），修复后同探针 GREEN。 +- evaluator `probe-root-row-generation.mjs`（只读复用）3/3 × 同两版本：修复前两处行/句柄代不一致误 clean，修复后占用保持；行句柄一致正例仍 clean。 +- 既有证据复验（修复后源码）：`probe-claimant-contract` 10/10 × Node 22.18.0/24.16.0/24.19.0；`probe-recorded-roots` 4/4 × 同三版本；`probe-manager-observed-history` 两模式 × Node 22.18.0/24.16.0。 +- 单元 `windows-process-cleanup` 67/67（新增 T1 八例：换代记录父子更早/等时/孙代传播/顺序反转/严格更晚正例、行句柄代不一致两形态与一致正例）；受影响九套件 116 tests/115 pass/0 fail/1 skip；全量 509 tests/507 pass/0 fail/2 skip；typecheck/build/validate:plugin/clean-entrypoints(4/4)/git diff --check exit 0（EOF 空白一次告警最小修复后 run2 exit 0，未重跑套件）。 + +## 最终独立验收（2026-10-18，Codex 于冻结候选执行；本节为公开摘要，原始证据存任务私有归档） + +- 六组 fake 探针 × Node 22.18.0/24.16.0 全过（含正向释放对照，零真实信号）：契约 10、已死父跨根 4、记录外来边 5、根行代 3、观察历史 2 模式、真实 Manager 重建 2 模式。 +- 平台矩阵（测试/通过/跳过，均 0 fail；平台专有 skip 如实记录）：Windows 22.18.0 全量 509/508/1；Windows 24.16.0 全量 509/507/2；Windows 24.19.0 受影响九套件 152/152/0；Linux 22.18.0 全量 511/508/3；Linux 24.20.0 全量 511/507/4。typecheck/build/validate:plugin/clean-entrypoints(4/4) 通过；git diff --check 于 Windows 工作区通过。Linux 首次聚合 bundle 比较失败保留：评估器符号链接使 bundle 来源注释指向 ../deps 源路径，注释前缀归一化后无其他字节差异，实体依赖重建后与原始 bundle 字节一致并 validate 通过（未重跑全套）。 +- attempt-7 仅按 review-round-6 完成注释/文档/记录措辞修正，源码行为与 bundle 字节不变。 + ## 影响 - 修改 `src/adapters/windows-process-cleanup.ts`（重写：区间归属、句柄事务、传输预算）、`src/adapters/process-spawn.ts`（期望根指纹、代连续性 helper、claimant 扫描）、`src/adapters/zcode-app-server-adapter.ts`、`src/adapters/zcode-adapter.ts`（每代重置）、`src/runtime/model-settings.ts`、`src/manager/task-manager.ts`（单调锁存、扫描门控）、`src/store/task-store.ts`（commitWorkerResult 锁存保留）及测试；`docs/ARCHITECTURE.md` Windows 清理段随本决策更新。 diff --git a/docs/reports/2026-10-18-pr47-claimant-repair.md b/docs/reports/2026-10-18-pr47-claimant-repair.md new file mode 100644 index 0000000..660909e --- /dev/null +++ b/docs/reports/2026-10-18-pr47-claimant-repair.md @@ -0,0 +1,52 @@ +# PR #47 Claimant 扫描返修报告（2026-10-18） + +> Status: REPORT（一次性交付材料；验证结论以 ADR-009 与 PROJECT_STATE 的核对记录为准） +> 范围：PR #47 反馈 head `628984c` 上的两项 P1 修复、真实 process-lock 失败调查、组合与真实 Manager 门禁回归；含集中评估 review-round-1 R1–R4 返工。原始日志与脚本存任务私有证据归档（本仓库外），本报告只保留可公开摘要。 + +## 修复内容 + +1. **P1-A（枚举后消失）**：`verifyWindowsRecordedTreeClaimants` 中，记录 PID 在枚举与 hold 之间消失时按已死处理（从 live 集合移除），其存活子代经已死记录 PID 路径计入 claimant；不再以过期枚举快照证明 clean。 +2. **P1-B（换代父与旧子）**：撤回"外来换代父排除整棵子树"的不成立保证。现行为逐边持有证明：每条边的父代必须被持有句柄核验（同句柄精确创建时间、与枚举行同代一致），子代创建表示区间**严格晚于**该精确创建时间才可判属外来代；随后子代自身也必须被持有且行代一致，更深后代递归同判。相等精度（ADR-009 同 100ns 规则同样适用于排除方向）、更早、跨界、创建时间未知，以及中间代消失/拒绝/换代/行句柄矛盾，一律保持占用。正向 clean 场景（空记录树；可证明新代链）保留。 +3. **hold 响应校验（review-round-1 R2，口径修正于 review-round-2）**：扫描对每次单 PID hold 校验响应数量、PID 对应关系与 ok 响应的精确身份字段（十进制、非空）。协议畸形（数量/PID 不符、字段缺失或非法格式）抛 `process_tree_hold_invalid` 拒绝扫描；行与句柄的代矛盾（ticks 不符或精确时间落在表示区间外）属归属未证明，保持占用而非抛错——两类出口都不用于证明 foreign/clean；helper 仍恰好在所有出口关闭一次。 +4. **终止预算语义**：kill 截止只对无句柄确认退出 FILETIME 的成员触发 `process_tree_cleanup_unverified`；全部成员已确认退出时继续闭包确认轮直至枚举证明为空（受 120 s 传输会话预算约束）。修复"全表枚举耗时 + WMI 滞留已终止成员行"时对已证明退出成员的误报（与本轮实测机制一致：单次全表枚举约 1.1–1.5 s，两轮枚举占 5 s 预算约六成）。 + +5. **多记录根单调合并（review-round-2 S1）**：某 PID 相对一个记录根证明为外来换代，不抹除它相对另一记录根的 claimant 证据（如已死记录成员的存活子代复用了另一记录 PID）；claimant 路径优先于外来排除，对全部记录根均无 claimant 路径时才排除。目标/行序不敏感，全外来正向仍释放。 +6. **观察证据持久化（review-round-2 S2）**：扫描返回 false 或拒绝时，任务相关观察——原始记录目标（即使某目标的当前代已证换代，目标 PID 本身仍保留，供后续按记录身份复扫）、claimant、请求过但未证实的外来谱系候选（不新增已证外来的后代候选，不采信响应载荷任意 PID）——经既有 cleanup-failure 记录与状态 marker 单调持久化；后续 cancel、恢复与 Manager 重建后的 scanTargets 取并集，已观察过的中间代不属"未观察祖先"边界；全部相关 PID 真正消失后正常释放。 + +7. **记录目标的统一边证明（review-round-3 T1）**：记录 PID 的换代判定须先通过行/句柄同代一致（ticks 相等且精确时间落在行表示区间）才可用该行为换代作证，否则保持占用；已判外来的记录目标仍按同一严格规则接受入边检验（创建不严格晚于外来父的持有精确时间即降级 claimant，孙代随之传播），缓存的外来状态不是全局排除证书；严格更晚且一致的正例复用缓存证据、不重复 hold，正向外来代释放保持。 + +## 维护者 process-lock 失败的排查边界 + +在官方校验的便携 Node 24.16.0（SHASUMS256 核对通过）与新编译输出下，两项 process-lock 用例修复前后均 2/2 通过，失败未自然复现；带登记与信号前成员身份门禁的阶段计时探针（自建 node 子进程与其 conhost，无信号观察取证确认成员构成）在 24.16.0 与 24.19.0 各 8/8。结论：已证明的误报机制已修复并有确定性回归，但不宣称对维护者机器特定失败的根治或版本归因。 + +## 验证摘要（worker 自测；Codex 独立复验与平台矩阵另行执行） + +- 维护者原始两 fake 反例：反馈 head RED（clean=true 缺陷复现）→ 修复后 GREEN（clean=false、livePids=[102]、零信号）。 +- evaluator 契约探针（10 例：两个原始反例、相等精度、中间代 absent/换代/denied、错 PID hold、缺精确字段、两个正向对照）：Node 22.18.0 / 24.16.0 / 24.19.0 各 10/10。 +- `windows-process-cleanup` 专项（attempt-2 源码）53/53：P1 组合矩阵（消失/换代+旧子/可证明新子/混合/多层/同精度/未知时间/缺 hold/身份读取失败）+ R1/R2 新增回归 + 预算两态。 +- `manager-claimant-gate`（attempt-2 源码）4/4：真实 TaskManager 消费路径（cancel→新鲜 probe→锁存→扫描门控→pump），fake 仅作 OS 传输；P1-A 经指纹化记录根真正触发 absent hold（断言 hold 发生）；maxConcurrency=1 且不同执行目录验证 slot 保留；扫描 false/reject/畸形 hold 保持 `cleanup_unverified` 与占用；可证明 clean 正常释放并放行排队任务。 +- baseline 对照：同一 Manager 测试在反馈 head 的原扫描器（转译加载）下 4/4 全部 "Missing expected rejection"——包括 review-round-1 指出原先误 PASS 的 P1-A 用例。 +- 预算机制独立复验（6 例，attempt-1）：基线误拒已确认退出、候选完成第三轮空闭包、未确认退出仍拒绝、会话预算拒绝保留成员集合。 +- 全量套件与 typecheck/build/validate:plugin/clean-entrypoints/git diff --check 的最终计数以 PROJECT_STATE 核对记录为准（按源码版本分别记录，不跨版本叠加）。 + +## 最终验证（Codex 独立平台矩阵，冻结候选上执行） + +| 平台/运行时 | 范围 | 测试 | 通过 | 失败/取消 | 跳过 | +|---|---|---:|---:|---:|---:| +| Windows 22.18.0 | 全量 | 509 | 508 | 0/0 | 1 | +| Windows 24.16.0 | 全量 | 509 | 507 | 0/0 | 2 | +| Windows 24.19.0 | 受影响九套件 | 152 | 152 | 0/0 | 0 | +| Linux 22.18.0 | 全量 | 511 | 508 | 0/0 | 3 | +| Linux 24.20.0 | 全量 | 511 | 507 | 0/0 | 4 | + +- 独立探针（fake 传输、零真实信号）于 Node 22.18.0 与 24.16.0 各六组全过：契约 10 例、已死父跨根 4 例、记录外来边 5 例、根行代 3 例、观察历史 2 模式、真实 Manager 重建 2 模式；各含正向释放对照。 +- 前述各平台检查（typecheck、全量/受影响套件、build、validate:plugin、clean-entrypoints 4/4）均通过，源码在验证期间未变；git diff --check 在 Windows 源码工作区通过（Linux tar 验收未运行 git diff）。Linux 首次聚合比较失败被保留：评估器 node_modules 符号链接使两 bundle 内来源注释指向 ../deps 源路径；经精确注释前缀归一化证明无其他字节差异，并在两运行时以实体依赖目录重建后 bundle 与原始字节一致、validate 通过（仅重建复检，未重跑全套）。 +- 跳过项如实记录不作通过论：Windows 的 POSIX 模式用例；Node24 专有 injected-ENOTEMPTY 按既有测试条件跳过；Linux 的 Windows startup hook 与两个 Windows 专有进程树子测试。各平台测试总数差异源于平台子测试分别计数。 +- 以上为 Codex 独立结果；worker 自测（Windows 24.19.0 会话全量 509/507/2 等）见各 attempt 记录，二者不混计。Windows 为目录隔离非 OS 沙箱；Linux 隔离为 rootless、非 root、仅回环网络、capEff 0、noNewPrivs 1、无宿主挂载。 + +## 未覆盖与边界 + +- 维护者机器特定失败未在本机自然复现（见上）。 +- 组合矩阵为 fake 事务覆盖；真实进程路径限于自建登记 fixture（node 子进程与其 conhost）。 +- 观察历史：早期一次探针存在 close 监听晚挂的观察缺陷、另一次缺少信号前成员名记录，均保留为历史且不并入门禁化后的结论；后续成员构成（node + conhost）由无信号自然退出 fixture 单独取证。 +- 未观察中间祖先仍在任何证明之外；ADR-009 为本 PR 分支自选编号，整合期重编号由维护者协调。 diff --git a/plugins/codex-zcode-bridge/dist/bridge.mjs b/plugins/codex-zcode-bridge/dist/bridge.mjs index 7bc6a0a..4d5ece8 100644 --- a/plugins/codex-zcode-bridge/dist/bridge.mjs +++ b/plugins/codex-zcode-bridge/dist/bridge.mjs @@ -24241,10 +24241,14 @@ async function terminateWindowsProcessTree(pid, expectedRootTicks, killWaitMs, d naturalGoneCreation.set(member.pid, member.creationFileTime); } } - if (now() >= deadline) { + const unconfirmedMembers = members2.filter((m) => { + const record3 = killed.get(m.pid); + return record3 === void 0 || record3.exitFileTime === null; + }); + if (now() >= deadline && unconfirmedMembers.length > 0) { throw new WindowsTreeCleanupError( "process_tree_cleanup_unverified", - `process_tree_cleanup_unverified: ${String(resolution.members.length)} recorded member(s) not provably exited`, + `process_tree_cleanup_unverified: ${String(unconfirmedMembers.length)} recorded member(s) not provably exited`, knownPids() ); } @@ -24360,12 +24364,53 @@ async function verifyWindowsRecordedTreeClaimants(targets, dependencies) { const transaction = await dependencies.open(); let bodyError = null; const targetPids = targets.map((t) => t.pid).filter((p) => Number.isInteger(p) && p > 0); + const claimantPids = /* @__PURE__ */ new Set(); + const foreignHeld = /* @__PURE__ */ new Map(); + const unattributableLineagePids = /* @__PURE__ */ new Set(); + const edgeVerifiedForeignPids = /* @__PURE__ */ new Set(); + const requestedPids = /* @__PURE__ */ new Set(); + const evidencePids = () => [ + .../* @__PURE__ */ new Set([...targetPids, ...claimantPids, ...[...requestedPids].filter((pid) => !foreignHeld.has(pid))]) + ]; + const DECIMAL = /^\d{1,25}$/u; + const holdOneValidated = async (pid) => { + requestedPids.add(pid); + const outcomes = await transaction.hold([pid]); + if (outcomes.length !== 1) { + throw new WindowsTreeCleanupError( + "process_tree_hold_invalid", + `process_tree_hold_invalid: ${String(outcomes.length)} responses for 1 requested pid`, + evidencePids() + ); + } + const outcome = outcomes[0]; + if (outcome === void 0 || outcome.pid !== pid) { + throw new WindowsTreeCleanupError( + "process_tree_hold_invalid", + `process_tree_hold_invalid: unexpected pid ${String(outcome?.pid ?? "missing")} in hold response`, + evidencePids() + ); + } + if (outcome.status === "ok") { + if (typeof outcome.creationFileTime !== "string" || !DECIMAL.test(outcome.creationFileTime) || typeof outcome.creationTicks !== "string" || !DECIMAL.test(outcome.creationTicks)) { + throw new WindowsTreeCleanupError( + "process_tree_hold_invalid", + `process_tree_hold_invalid: ok hold response for pid ${String(pid)} lacks usable exact identity fields`, + evidencePids() + ); + } + return { status: "ok", creationFileTime: outcome.creationFileTime, creationTicks: outcome.creationTicks }; + } + return { status: outcome.status }; + }; try { const rows = await transaction.enumerate(); const live = /* @__PURE__ */ new Set(); for (const row of rows) live.add(row.pid); - const claimantPids = /* @__PURE__ */ new Set(); - const foreignPids = /* @__PURE__ */ new Set(); + const rowsByPid = /* @__PURE__ */ new Map(); + for (const row of rows) { + if (!rowsByPid.has(row.pid)) rowsByPid.set(row.pid, row); + } const recorded = /* @__PURE__ */ new Set(); for (const target of targets) { if (!Number.isInteger(target.pid) || target.pid <= 0) continue; @@ -24375,13 +24420,19 @@ async function verifyWindowsRecordedTreeClaimants(targets, dependencies) { claimantPids.add(target.pid); continue; } - const outcome = (await transaction.hold([target.pid]))[0]; - if (outcome === void 0) throw new WindowsTreeCleanupError("process_tree_scan_invalid", "recorded-tree scan received no hold outcome"); + const outcome = await holdOneValidated(target.pid); if (outcome.status === "ok") { if (outcome.creationTicks === target.fingerprint) claimantPids.add(target.pid); - else foreignPids.add(target.pid); + else { + const row = rowsByPid.get(target.pid); + if (row !== void 0 && row.startTicks !== null && row.creationFileTime !== null && outcome.creationTicks === row.startTicks && withinCimInterval(outcome.creationFileTime, row.creationFileTime)) { + foreignHeld.set(target.pid, { exact: outcome.creationFileTime }); + } else { + claimantPids.add(target.pid); + } + } } else if (outcome.status === "absent") { - continue; + live.delete(target.pid); } else { claimantPids.add(target.pid); } @@ -24389,20 +24440,49 @@ async function verifyWindowsRecordedTreeClaimants(targets, dependencies) { for (let changed = true; changed; ) { changed = false; for (const row of rows) { - if (claimantPids.has(row.pid) || foreignPids.has(row.pid)) continue; - const parentClaimed = claimantPids.has(row.parentPid); - const parentRecordedDead = recorded.has(row.parentPid) && !live.has(row.parentPid) && !foreignPids.has(row.parentPid); - if (parentClaimed || parentRecordedDead) { + if (claimantPids.has(row.pid)) continue; + if (claimantPids.has(row.parentPid) || unattributableLineagePids.has(row.parentPid) || recorded.has(row.parentPid) && !live.has(row.parentPid) && !foreignHeld.has(row.parentPid)) { claimantPids.add(row.pid); changed = true; } } + for (const row of rows) { + if (claimantPids.has(row.pid) || unattributableLineagePids.has(row.pid) || edgeVerifiedForeignPids.has(row.pid)) continue; + const foreignParent = foreignHeld.get(row.parentPid); + if (foreignParent === void 0) continue; + if (row.creationFileTime === null || BigInt(row.creationFileTime) <= BigInt(foreignParent.exact)) { + claimantPids.add(row.pid); + foreignHeld.delete(row.pid); + edgeVerifiedForeignPids.delete(row.pid); + changed = true; + continue; + } + if (foreignHeld.has(row.pid)) { + edgeVerifiedForeignPids.add(row.pid); + changed = true; + continue; + } + const outcome = await holdOneValidated(row.pid); + if (outcome.status === "absent") { + unattributableLineagePids.add(row.pid); + } else if (outcome.status === "ok" && row.startTicks !== null && outcome.creationTicks === row.startTicks && withinCimInterval(outcome.creationFileTime, row.creationFileTime)) { + foreignHeld.set(row.pid, { exact: outcome.creationFileTime }); + edgeVerifiedForeignPids.add(row.pid); + } else { + claimantPids.add(row.pid); + } + changed = true; + } } const liveList = [...claimantPids].sort((a, b) => a - b); - return { clean: liveList.length === 0, livePids: liveList }; + return { clean: liveList.length === 0, livePids: liveList, observedPids: evidencePids().sort((a, b) => a - b) }; } catch (error2) { bodyError = error2; - throw error2; + if (error2 instanceof WindowsTreeCleanupError) { + throw error2.memberPids.length > 0 ? error2 : new WindowsTreeCleanupError(error2.code, error2.message, evidencePids()); + } + const detail = error2 instanceof Error ? error2.message : String(error2); + throw new WindowsTreeCleanupError("process_tree_scan_failed", `process_tree_scan_failed: ${detail}`, evidencePids()); } finally { let closeError = null; try { @@ -24414,9 +24494,9 @@ async function verifyWindowsRecordedTreeClaimants(targets, dependencies) { const closeText = closeError instanceof Error ? closeError.message : String(closeError); if (bodyError !== null) { const originalText = bodyError instanceof Error ? bodyError.message : String(bodyError); - throw new WindowsTreeCleanupError("process_tree_scan_failed", `process_tree_scan_failed: ${originalText}; process_tree_close_failed: ${closeText}`, targetPids); + throw new WindowsTreeCleanupError("process_tree_scan_failed", `process_tree_scan_failed: ${originalText}; process_tree_close_failed: ${closeText}`, evidencePids()); } - throw new WindowsTreeCleanupError("process_tree_close_failed", `process_tree_close_failed: ${closeText}`, targetPids); + throw new WindowsTreeCleanupError("process_tree_close_failed", `process_tree_close_failed: ${closeText}`, evidencePids()); } } } @@ -24838,7 +24918,7 @@ async function verifyProcessTreeClaimantsClean(targets) { return verifyWindowsRecordedTreeClaimants(targets, { open: openPowerShellTreeTransaction }); } const live = targets.filter((t) => Number.isInteger(t.pid) && t.pid > 0 && isProcessTreeRunning(t.pid)).map((t) => t.pid); - return { clean: live.length === 0, livePids: live }; + return { clean: live.length === 0, livePids: live, observedPids: live }; } function childStillOnSpawnGeneration(child) { if (typeof child.pid !== "number" || !Number.isInteger(child.pid) || child.pid <= 0) return false; @@ -26083,10 +26163,23 @@ var BridgeTaskManager = class _BridgeTaskManager { } const scanTargets = this.#claimantTargets(taskId, status.attempt, probeSnapshot).filter((target) => !verifiedRoots.has(target.pid)); if (scanTargets.length > 0) { + const retainObserved = (observed, livePids, error2) => { + const fromError = error2 instanceof WindowsTreeCleanupError ? error2.memberPids : []; + const pids = [.../* @__PURE__ */ new Set([...observed ?? [], ...livePids ?? [], ...fromError])].filter((pid2) => Number.isSafeInteger(pid2) && pid2 > 0); + if (pids.length === 0) return; + const carrier = error2 instanceof WindowsTreeCleanupError && error2.memberPids.length > 0 ? error2 : new WindowsTreeCleanupError( + "process_tree_cleanup_unverified", + `recorded-tree scan keeps occupancy: ${pids.join(",")} still requires attribution`, + pids + ); + this.#recordCleanupFailure(taskId, status.attempt, carrier); + }; const scan = await this.#verifyTreeClean(scanTargets).catch((error2) => { + retainObserved(void 0, void 0, error2); throw new TaskManagerError("CANCEL_FAILED", `ZCode cleanup verification failed: ${error2 instanceof Error ? error2.message : String(error2)}`); }); if (!scan.clean) { + retainObserved(scan.observedPids, scan.livePids); this.#store.appendEvent(taskId, "cleanup_unverified", "Recorded-tree scan still observes live claimants; the task stays occupied", { live_pids: [...scan.livePids] }, this.#now().toISOString()); throw new TaskManagerError("CANCEL_FAILED", `ZCode cleanup could not be verified: ${String(scan.livePids.length)} recorded-tree process(es) still alive`); } diff --git a/plugins/codex-zcode-bridge/worker/worker-main.mjs b/plugins/codex-zcode-bridge/worker/worker-main.mjs index 1253b0f..75db629 100644 --- a/plugins/codex-zcode-bridge/worker/worker-main.mjs +++ b/plugins/codex-zcode-bridge/worker/worker-main.mjs @@ -1144,10 +1144,14 @@ async function terminateWindowsProcessTree(pid, expectedRootTicks, killWaitMs, d naturalGoneCreation.set(member.pid, member.creationFileTime); } } - if (now() >= deadline) { + const unconfirmedMembers = members2.filter((m) => { + const record2 = killed.get(m.pid); + return record2 === void 0 || record2.exitFileTime === null; + }); + if (now() >= deadline && unconfirmedMembers.length > 0) { throw new WindowsTreeCleanupError( "process_tree_cleanup_unverified", - `process_tree_cleanup_unverified: ${String(resolution.members.length)} recorded member(s) not provably exited`, + `process_tree_cleanup_unverified: ${String(unconfirmedMembers.length)} recorded member(s) not provably exited`, knownPids() ); } diff --git a/src/adapters/process-spawn.ts b/src/adapters/process-spawn.ts index 4acd8ba..47899c3 100644 --- a/src/adapters/process-spawn.ts +++ b/src/adapters/process-spawn.ts @@ -153,23 +153,44 @@ export const terminateProcessTree: TerminateProcessTree = async (pid, options = * Read-only verification used by the manager after a failed termination: * reports live processes that currently claim any recorded pid as their * parent (transitively), plus recorded pids still alive under their recorded - * generation. A live recorded pid whose current generation differs is a - * proven foreign owner and its subtree is excluded. It proves "nothing - * provably-ours-or-unattributable is alive right now"; unobserved - * intermediate ancestors remain outside any proof and callers must log that - * boundary instead of claiming completeness. + * generation. A live recorded pid whose current generation differs counts + * as foreign only after its row and held handle prove the same generation, + * and even then it keeps obeying the all-recorded-roots incoming-edge proof + * (review-round-3 T1): the pid itself is excluded only on that basis, and a + * descendant is excluded with it ONLY when every edge of its lineage carries + * held-handle evidence — the parent of each edge is held (exact creation from that same + * handle, row-generation consistent) and the child's creation representation + * lies STRICTLY after it; equal, earlier, or unknown times, absent/denied/ + * replaced intermediates, and malformed hold responses are unproven and stay + * claimants or refuse the scan (PR #47 P1-B, review-round-1 R1/R2). A + * recorded pid that vanishes between enumeration and hold is dead: its live + * children stay claimants like any dead recorded pid's (PR #47 P1-A). It + * proves "nothing provably-ours-or-unattributable is alive right now"; + * unobserved intermediate ancestors remain outside any proof and callers + * must log that boundary instead of claiming completeness. */ export interface TreeClaimantTarget { readonly pid: number; readonly fingerprint: string | null; } -export async function verifyProcessTreeClaimantsClean(targets: readonly TreeClaimantTarget[]): Promise<{ clean: boolean; livePids: readonly number[] }> { +export interface TreeClaimantsResult { + readonly clean: boolean; + readonly livePids: readonly number[]; + /** Task-relevant pids this scan actually involved (targets, requested + * candidates, claimants) for durable retention on a false/rejected result + * (review-round-2 S2). Original targets are always retained — even when + * their current generation is proven foreign — and only newly + * proven-foreign descendant candidates are omitted. */ + readonly observedPids: readonly number[]; +} + +export async function verifyProcessTreeClaimantsClean(targets: readonly TreeClaimantTarget[]): Promise { if (process.platform === "win32") { return verifyWindowsRecordedTreeClaimants(targets, { open: openPowerShellTreeTransaction }); } const live = targets.filter((t) => Number.isInteger(t.pid) && t.pid > 0 && isProcessTreeRunning(t.pid)).map((t) => t.pid); - return { clean: live.length === 0, livePids: live }; + return { clean: live.length === 0, livePids: live, observedPids: live }; } // -- Spawn-generation continuity (ADR-009) ---------------------------------- diff --git a/src/adapters/windows-process-cleanup.ts b/src/adapters/windows-process-cleanup.ts index 1ae2bd3..a822d2e 100644 --- a/src/adapters/windows-process-cleanup.ts +++ b/src/adapters/windows-process-cleanup.ts @@ -468,10 +468,22 @@ export async function terminateWindowsProcessTree( // wait_timeout / error: the member stays unresolved; the next round // retries until the deadline refuses. } - if (now() >= deadline) { + // The kill budget bounds WAITING for exits, not exits already proven on + // held handles (PR #47 process-lock investigation): a member whose exit + // FILETIME was read from its held handle IS provably exited, and a + // full-table enumeration that ran slow must not turn that proof into a + // false "not provably exited" failure. Only members WITHOUT a confirmed + // exit trip the deadline; confirmation rounds for confirmed members + // continue until the closure provably empties, still bounded end-to-end + // by the transport session budget. + const unconfirmedMembers = members.filter((m) => { + const record = killed.get(m.pid); + return record === undefined || record.exitFileTime === null; + }); + if (now() >= deadline && unconfirmedMembers.length > 0) { throw new WindowsTreeCleanupError( "process_tree_cleanup_unverified", - `process_tree_cleanup_unverified: ${String(resolution.members.length)} recorded member(s) not provably exited`, + `process_tree_cleanup_unverified: ${String(unconfirmedMembers.length)} recorded member(s) not provably exited`, knownPids(), ); } @@ -615,29 +627,145 @@ function orderChildrenFirst(members: readonly MemberRecord[]): MemberRecord[] { * Read-only recorded-tree scan: reports live processes that currently claim * any of the recorded pids as their parent, transitively. A recorded pid that * is alive with a MATCHING startup fingerprint is itself a claimant; one - * alive with a different fingerprint is a proven foreign generation — its - * subtree is excluded from the claim. A recorded pid without a fingerprint - * that is alive stays a claimant (unprovable, conservative). Children of + * alive with a different fingerprint is treated as a foreign generation ONLY + * after its enumeration row and held handle prove to describe the SAME + * generation, and even then it stays subject to the all-recorded-roots + * incoming-edge proof below (review-round-3 T1) — a mismatch alone neither + * excludes the pid nor certifies its row's lineage. A foreign-generation pid + * is excluded, and a descendant is excluded with it ONLY when every edge of + * its lineage is proven on held handles: + * - the parent of the edge is HELD (native handle, exact creation FILETIME + * and ticks from that same handle) and therefore provably owned its pid + * continuously from that creation until now, so a child whose CIM + * representation lies STRICTLY after the parent's exact creation was + * created during that ownership — equal, earlier, boundary-straddling, or + * unknown times are unproven and stay claimants (ADR-009 same-100ns rule; + * PR #47 review-round-1 R1: an unheld enumeration row alone proves + * nothing about pid ownership across the descendant's creation); + * - the excluded member itself must then be held with a row-consistent + * identity (same-generation handle) before ITS descendants may be + * classified — absent, denied, replaced-generation, or malformed evidence + * at any intermediate level keeps the subtree occupied. + * Evidence from ALL recorded roots combines MONOTONICALLY (review-round-2 + * S1): a pid excluded as foreign relative to one recorded identity can still + * be ours-or-unattributable relative to another (e.g. a live child of a dead + * recorded member that happens to reuse another recorded pid). Claimant + * evidence from any root always wins over foreign exclusion; the exclusion + * only ever applies to pids with no claimant path from any root. A recorded + * target's own foreign classification is held to the SAME proof as any + * candidate (review-round-3 T1): the fingerprint mismatch must be backed by + * a row/handle generation-consistent hold before the row may vouch for the + * replacement, and the row's incoming edge is judged against foreign-held + * parents like anyone else's — a cached foreign status is never a global + * exclusion certificate, and strictly-later proven edges reuse the cached + * hold without repeating it. + * A recorded pid without a fingerprint that is alive stays a claimant + * (unprovable, conservative). A recorded pid that vanishes between + * enumeration and hold is dead: its live children count as claimants exactly + * like children of any other dead recorded pid (PR #47 P1-A). Children of * dead recorded pids count as claimants: their creator is unattributable * after the fact, and releasing while one is alive could orphan our own - * process. This proves "nothing provably-ours-or-unattributable is alive - * right now"; it does NOT prove no process ever descended from the recorded - * pids (unobserved intermediate ancestors remain outside any proof), and - * callers must log that boundary instead of claiming completeness. + * process. Hold responses are validated against the request (count, pid, + * mandatory identity fields) before any proof is drawn from them + * (review-round-1 R2); malformed evidence refuses the scan instead of + * proving a false clean. This proves "nothing provably-ours-or-unattributable + * is alive right now"; it does NOT prove no process ever descended from the + * recorded pids (unobserved intermediate ancestors remain outside any + * proof), and callers must log that boundary instead of claiming + * completeness. + * + * Observation evidence (review-round-2 S2): `observedPids` (and the member + * set carried by every exception) reports the TASK-RELEVANT pids this scan + * actually involved — the recorded targets, every pid it asked the helper to + * hold (foreign-lineage edge candidates), and the claimants — so a caller + * can persist them before keeping occupancy and cover them on later scans. + * Pids taken from response payloads are never trusted as observations. The + * ORIGINAL recorded targets stay in the evidence even when their current + * generation is proven foreign (later scans re-judge them under their + * recorded identity); only NEWLY proven-foreign descendant candidates are + * omitted from the added evidence — persisting those would make a later + * fingerprint-less scan count them as claimants forever. A pid already + * observed here is not an "unobserved ancestor" for later scans. */ export async function verifyWindowsRecordedTreeClaimants( targets: readonly { pid: number; fingerprint: string | null }[], dependencies: WindowsCleanupDependencies, -): Promise<{ clean: boolean; livePids: readonly number[] }> { +): Promise<{ clean: boolean; livePids: readonly number[]; observedPids: readonly number[] }> { const transaction = await dependencies.open(); let bodyError: unknown = null; const targetPids = targets.map((t) => t.pid).filter((p) => Number.isInteger(p) && p > 0); + const claimantPids = new Set(); + /** Members whose foreign lineage is proven on a HELD, row-consistent + * handle: `exact` is the held handle's exact creation FILETIME. */ + const foreignHeld = new Map(); + /** Lineage members that are dead (absent at hold) but unattributable: + * not occupied themselves, but their live children cannot be proven + * foreign and stay claimants. */ + const unattributableLineagePids = new Set(); + /** Pids whose incoming parent edge has been PROVEN strictly-later against + * a held, row-consistent foreign parent (their cached hold evidence needs + * no re-verification in later sweeps). */ + const edgeVerifiedForeignPids = new Set(); + /** Every pid this scan asked the helper to hold — observation evidence + * only ever comes from our own requests, never from response payloads. */ + const requestedPids = new Set(); + /** Task-relevant observation evidence: targets + claimants + requested + * pids that were not proven foreign (proven-foreign pids are omitted so a + * later fingerprint-less scan cannot count them as claimants forever). */ + const evidencePids = (): number[] => [ + ...new Set([...targetPids, ...claimantPids, ...[...requestedPids].filter((pid) => !foreignHeld.has(pid))]), + ]; + const DECIMAL = /^\d{1,25}$/u; + /** A hold response whose shape was validated: ok outcomes carry both + * exact identity fields as well-formed decimal strings. */ + type ValidatedHold = + | { readonly status: "ok"; readonly creationFileTime: string; readonly creationTicks: string } + | { readonly status: "absent" | "denied" | "error" }; + /** Holds exactly one pid and validates the response shape before any + * caller draws identity evidence from it (review-round-1 R2): the count, + * the pid, and — for ok outcomes — both exact identity fields must be + * present and well-formed. Anything else is a protocol failure, never a + * proof of anything; the failure carries the observation evidence + * gathered so far (review-round-2 S2). */ + const holdOneValidated = async (pid: number): Promise => { + requestedPids.add(pid); + const outcomes = await transaction.hold([pid]); + if (outcomes.length !== 1) { + throw new WindowsTreeCleanupError( + "process_tree_hold_invalid", + `process_tree_hold_invalid: ${String(outcomes.length)} responses for 1 requested pid`, + evidencePids(), + ); + } + const outcome = outcomes[0]; + if (outcome === undefined || outcome.pid !== pid) { + throw new WindowsTreeCleanupError( + "process_tree_hold_invalid", + `process_tree_hold_invalid: unexpected pid ${String(outcome?.pid ?? "missing")} in hold response`, + evidencePids(), + ); + } + if (outcome.status === "ok") { + if (typeof outcome.creationFileTime !== "string" || !DECIMAL.test(outcome.creationFileTime) || + typeof outcome.creationTicks !== "string" || !DECIMAL.test(outcome.creationTicks)) { + throw new WindowsTreeCleanupError( + "process_tree_hold_invalid", + `process_tree_hold_invalid: ok hold response for pid ${String(pid)} lacks usable exact identity fields`, + evidencePids(), + ); + } + return { status: "ok", creationFileTime: outcome.creationFileTime, creationTicks: outcome.creationTicks }; + } + return { status: outcome.status }; + }; try { const rows = await transaction.enumerate(); const live = new Set(); for (const row of rows) live.add(row.pid); - const claimantPids = new Set(); - const foreignPids = new Set(); + const rowsByPid = new Map(); + for (const row of rows) { + if (!rowsByPid.has(row.pid)) rowsByPid.set(row.pid, row); + } const recorded = new Set(); for (const target of targets) { if (!Number.isInteger(target.pid) || target.pid <= 0) continue; @@ -647,13 +775,36 @@ export async function verifyWindowsRecordedTreeClaimants( claimantPids.add(target.pid); continue; } - const outcome = (await transaction.hold([target.pid]))[0]; - if (outcome === undefined) throw new WindowsTreeCleanupError("process_tree_scan_invalid", "recorded-tree scan received no hold outcome"); + const outcome = await holdOneValidated(target.pid); if (outcome.status === "ok") { if (outcome.creationTicks === target.fingerprint) claimantPids.add(target.pid); - else foreignPids.add(target.pid); + else { + // Review-round-3 T1(B): a fingerprint mismatch only proves the + // CURRENT holder differs from the RECORD — it says nothing about + // the enumerated row's lineage. The row may vouch for the held + // generation only when row and handle describe the SAME generation + // (ticks equal AND the exact time inside the row's representation + // interval), exactly as required of excluded intermediates. An + // inconsistent or unknown row cannot prove the replacement's + // parentage: keep the pid occupied. + const row = rowsByPid.get(target.pid); + if ( + row !== undefined && row.startTicks !== null && row.creationFileTime !== null && + outcome.creationTicks === row.startTicks && + withinCimInterval(outcome.creationFileTime, row.creationFileTime) + ) { + foreignHeld.set(target.pid, { exact: outcome.creationFileTime }); + } else { + claimantPids.add(target.pid); + } + } } else if (outcome.status === "absent") { - continue; // exited between enumeration and hold + // Vanished between enumeration and hold: it is dead now, so its live + // children are children of a dead recorded pid — claimants. Dropping + // it from the live set is what makes parentRecordedDead fire for them + // (PR #47 P1-A); keeping it would leave the stale enumeration snapshot + // vouching for a process that no longer exists. + live.delete(target.pid); } else { // denied/error: identity unreadable; never treat as provably gone. claimantPids.add(target.pid); @@ -661,21 +812,94 @@ export async function verifyWindowsRecordedTreeClaimants( } for (let changed = true; changed;) { changed = false; + // Claimant-path sweep (review-round-2 S1): evaluated for EVERY row — + // a pid already excluded as foreign relative to one recorded root can + // still be ours-or-unattributable relative to another (a live child of + // a dead recorded member that reused another recorded pid, a descendant + // of a claimant, or of an unattributable lineage member). Claimant + // evidence from ANY root wins over foreign exclusion; exclusion only + // ever applies to pids with no claimant path at all. for (const row of rows) { - if (claimantPids.has(row.pid) || foreignPids.has(row.pid)) continue; - const parentClaimed = claimantPids.has(row.parentPid); - const parentRecordedDead = recorded.has(row.parentPid) && !live.has(row.parentPid) && !foreignPids.has(row.parentPid); - if (parentClaimed || parentRecordedDead) { + if (claimantPids.has(row.pid)) continue; + if ( + claimantPids.has(row.parentPid) || + unattributableLineagePids.has(row.parentPid) || + (recorded.has(row.parentPid) && !live.has(row.parentPid) && !foreignHeld.has(row.parentPid)) + ) { claimantPids.add(row.pid); changed = true; } } + // Foreign-lineage exclusion sweep (review-round-3 T1: a cached + // foreign-held status — including one initialized from a recorded + // target — is NOT a global exclusion certificate). Every row whose + // incoming edge has not yet been proven is judged against a + // foreign-held parent with the SAME strict edge rule, recorded targets + // included: a row created at-or-before the parent's held exact creation + // is an old-generation child of the recorded parent pid or + // unattributable, and is demoted to a claimant even if it was already + // foreign relative to its OWN record. + for (const row of rows) { + if (claimantPids.has(row.pid) || unattributableLineagePids.has(row.pid) || edgeVerifiedForeignPids.has(row.pid)) continue; + const foreignParent = foreignHeld.get(row.parentPid); + if (foreignParent === undefined) continue; // parent outside this tree or still pending a later sweep + // The held foreign parent provably owned its pid continuously since + // its exact creation F. A row created STRICTLY after F (whole CIM + // representation interval beyond F; equality is the ADR-009 + // indistinguishable case, earlier rows are old-generation children) + // is a candidate foreign descendant — but the candidate itself must + // be held and row-consistent before it, in turn, may be excluded: an + // unheld snapshot row proves nothing about pid ownership across its + // own descendants (review-round-1 R1). + if (row.creationFileTime === null || BigInt(row.creationFileTime) <= BigInt(foreignParent.exact)) { + claimantPids.add(row.pid); + foreignHeld.delete(row.pid); + edgeVerifiedForeignPids.delete(row.pid); + changed = true; + continue; + } + if (foreignHeld.has(row.pid)) { + // Already held and row-consistent (target initialization above or a + // previous sweep pass): the strictly-later edge is proven with the + // CACHED evidence — no repeat hold is issued. + edgeVerifiedForeignPids.add(row.pid); + changed = true; + continue; + } + const outcome = await holdOneValidated(row.pid); + if (outcome.status === "absent") { + unattributableLineagePids.add(row.pid); + } else if ( + outcome.status === "ok" && + row.startTicks !== null && + outcome.creationTicks === row.startTicks && + withinCimInterval(outcome.creationFileTime, row.creationFileTime) + ) { + // Same-generation handle and row: excluded as a proven foreign + // descendant, and its own children will be judged against ITS held + // exact creation in a later sweep. + foreignHeld.set(row.pid, { exact: outcome.creationFileTime }); + edgeVerifiedForeignPids.add(row.pid); + } else { + // denied/error, or a replaced generation (handle ticks or exact + // time inconsistent with the enumerated row): alive but unprovable. + claimantPids.add(row.pid); + } + changed = true; + } } const liveList = [...claimantPids].sort((a, b) => a - b); - return { clean: liveList.length === 0, livePids: liveList }; + return { clean: liveList.length === 0, livePids: liveList, observedPids: evidencePids().sort((a, b) => a - b) }; } catch (error) { bodyError = error; - throw error; + // Review-round-2 S2: EVERY exception exit carries the task-relevant + // observation evidence (targets + claimants + requested-but-unproven + // candidates), never pids taken from response payloads. + if (error instanceof WindowsTreeCleanupError) { + throw error.memberPids.length > 0 ? error : new WindowsTreeCleanupError(error.code, error.message, evidencePids()); + } + const detail = error instanceof Error ? error.message : String(error); + throw new WindowsTreeCleanupError("process_tree_scan_failed", `process_tree_scan_failed: ${detail}`, evidencePids()); } finally { // R4: the read-only scan must not claim a result through a silently // failed helper close either; a close failure is chained onto a body @@ -690,9 +914,9 @@ export async function verifyWindowsRecordedTreeClaimants( const closeText = closeError instanceof Error ? closeError.message : String(closeError); if (bodyError !== null) { const originalText = bodyError instanceof Error ? bodyError.message : String(bodyError); - throw new WindowsTreeCleanupError("process_tree_scan_failed", `process_tree_scan_failed: ${originalText}; process_tree_close_failed: ${closeText}`, targetPids); + throw new WindowsTreeCleanupError("process_tree_scan_failed", `process_tree_scan_failed: ${originalText}; process_tree_close_failed: ${closeText}`, evidencePids()); } - throw new WindowsTreeCleanupError("process_tree_close_failed", `process_tree_close_failed: ${closeText}`, targetPids); + throw new WindowsTreeCleanupError("process_tree_close_failed", `process_tree_close_failed: ${closeText}`, evidencePids()); } } } diff --git a/src/manager/task-manager.ts b/src/manager/task-manager.ts index f2da504..2df4b8d 100644 --- a/src/manager/task-manager.ts +++ b/src/manager/task-manager.ts @@ -32,6 +32,7 @@ import type { ZCodeRunOutcome } from "../adapters/zcode-adapter.js"; import { TaskManagerError } from "./errors.js"; import { defaultSpawnWorker, type SpawnWorker } from "./spawn-worker.js"; import { isProcessRunning, terminateProcessTree, verifyProcessTreeClaimantsClean, type TerminateProcessTree, type TreeClaimantTarget } from "../adapters/process-spawn.js"; +import { WindowsTreeCleanupError } from "../adapters/windows-process-cleanup.js"; import { validateTaskTimeout } from "../runtime/task-timeout.js"; import { tryAcquireProcessLock, withProcessLock, publishSelfIdentity } from "../store/process-lock.js"; import { buildTaskPrompt, buildContinuePrompt } from "../prompts/task-prompt.js"; @@ -52,7 +53,7 @@ export interface TaskManagerOptions { /** Defaults to real process-tree termination with verification. */ terminateProcessTree?: TerminateProcessTree; /** Defaults to the read-only recorded-tree claimant scan (ADR-009). */ - verifyTreeClean?: (targets: readonly TreeClaimantTarget[]) => Promise<{ clean: boolean; livePids: readonly number[] }>; + verifyTreeClean?: (targets: readonly TreeClaimantTarget[]) => Promise<{ clean: boolean; livePids: readonly number[]; observedPids?: readonly number[] }>; /** Defaults to the platform ProcessProbe (Windows PowerShell batch / Linux * /proc / macOS ps). Recovery and cleanup identity checks use it. */ probe?: ProcessProbe; @@ -79,7 +80,7 @@ export class BridgeTaskManager implements ProgressTaskManager { readonly #spawnWorker: SpawnWorker; readonly #isProcessRunning: (pid: number) => boolean; readonly #terminateProcessTree: TerminateProcessTree; - readonly #verifyTreeClean: (targets: readonly TreeClaimantTarget[]) => Promise<{ clean: boolean; livePids: readonly number[] }>; + readonly #verifyTreeClean: (targets: readonly TreeClaimantTarget[]) => Promise<{ clean: boolean; livePids: readonly number[]; observedPids?: readonly number[] }>; readonly #probe: ProcessProbe | null; readonly #now: () => Date; readonly #dataRoot: string; @@ -1126,10 +1127,34 @@ export class BridgeTaskManager implements ProgressTaskManager { // outside any proof and are logged, not claimed complete). const scanTargets = this.#claimantTargets(taskId, status.attempt, probeSnapshot).filter((target) => !verifiedRoots.has(target.pid)); if (scanTargets.length > 0) { + // Review-round-2 S2: whenever the scan keeps occupancy — false OR + // rejected — the task-relevant pids it actually observed/requested + // join the durable member evidence through the SAME monotonic + // cleanup-failure record and marker used for termination + // failures, so a later cancel (or a recreated manager) still scans + // them; a pid already observed here is not an "unobserved + // ancestor" later. Only pids the scan itself reports as its own + // observation evidence are retained — never arbitrary pids from + // malformed response payloads — and a clean scan records nothing. + const retainObserved = (observed: readonly number[] | undefined, livePids: readonly number[] | undefined, error?: unknown): void => { + const fromError = error instanceof WindowsTreeCleanupError ? error.memberPids : []; + const pids = [...new Set([...(observed ?? []), ...(livePids ?? []), ...fromError])].filter((pid) => Number.isSafeInteger(pid) && pid > 0); + if (pids.length === 0) return; + const carrier = error instanceof WindowsTreeCleanupError && error.memberPids.length > 0 + ? error + : new WindowsTreeCleanupError( + "process_tree_cleanup_unverified", + `recorded-tree scan keeps occupancy: ${pids.join(",")} still requires attribution`, + pids, + ); + this.#recordCleanupFailure(taskId, status.attempt, carrier); + }; const scan = await this.#verifyTreeClean(scanTargets).catch((error: unknown) => { + retainObserved(undefined, undefined, error); throw new TaskManagerError("CANCEL_FAILED", `ZCode cleanup verification failed: ${error instanceof Error ? error.message : String(error)}`); }); if (!scan.clean) { + retainObserved(scan.observedPids, scan.livePids); this.#store.appendEvent(taskId, "cleanup_unverified", "Recorded-tree scan still observes live claimants; the task stays occupied", { live_pids: [...scan.livePids] }, this.#now().toISOString()); throw new TaskManagerError("CANCEL_FAILED", `ZCode cleanup could not be verified: ${String(scan.livePids.length)} recorded-tree process(es) still alive`); } diff --git a/test/manager-claimant-gate.test.ts b/test/manager-claimant-gate.test.ts new file mode 100644 index 0000000..cb08cde --- /dev/null +++ b/test/manager-claimant-gate.test.ts @@ -0,0 +1,386 @@ +// PR #47 consumption-path regressions: the REAL TaskManager release gate +// (cancel → fresh probe → latched cleanup_unverified → recorded-tree claimant +// scan → occupancy/pump) driven end to end. Per the repair plan, the fake +// transaction stands in ONLY for the OS scan transport; the claimant scan +// logic itself and every manager decision are production code. Review-round-1 +// R3: the P1-A shape below drives the absent-hold branch through a recorded +// root WITH a persisted fingerprint (verified against the ORIGINAL defective +// scanner via the evaluator's baseline loader, where it fails); the P1-B shape +// was already baseline-RED. Numeric decimal ticks throughout: hold responses +// are validated against the transport contract before proving anything. +import assert from "node:assert/strict"; +import test from "node:test"; +import { mkdtempSync, rmSync } from "node:fs"; +import { tmpdir } from "node:os"; +import path from "node:path"; +import { WindowsTreeCleanupError, type HoldOutcome, type KillOutcome, type WindowsProcessRow, type WindowsTreeTransactionLike } from "../src/adapters/windows-process-cleanup.js"; +import { FakeAdapter, makeManagerFixture } from "./manager-helpers.js"; + +/** Scripted OS process table for the real scan: fixed rows, per-pid hold + * overrides (absent/denied/replaced), and injectable transport failures. */ +class ManagerScanTable implements WindowsTreeTransactionLike { + rows: WindowsProcessRow[] = []; + readonly holdOverrides = new Map(); + enumerateError: Error | null = null; + closeError: Error | null = null; + readonly holds: number[][] = []; + closeCount = 0; + + async enumerate(): Promise { + if (this.enumerateError) throw this.enumerateError; + return [...this.rows]; + } + + /** Pids whose hold request returns NO response at all (protocol gap). */ + readonly holdEmptyFor = new Set(); + + async hold(pids: readonly number[]): Promise { + this.holds.push([...pids]); + if (pids.length === 1 && this.holdEmptyFor.has(pids[0]!)) return []; + return pids.map((pid) => { + const override = this.holdOverrides.get(pid); + if (override) return override; + const target = this.rows.find((r) => r.pid === pid); + if (!target) return { pid, status: "absent" } as HoldOutcome; + if (!target.startTicks || !target.creationFileTime) return { pid, status: "denied", error: 5 } as HoldOutcome; + return { + pid, + status: "ok", + creationFileTime: String(BigInt(target.creationFileTime) + 3n), + creationTicks: target.startTicks, + } as HoldOutcome; + }); + } + + async kill(): Promise { + throw new Error("read-only scan must not signal"); + } + + async close(): Promise { + this.closeCount += 1; + if (this.closeError) throw this.closeError; + } +} + +/** 100ns-tick anchors: one unit = 100 ticks (10µs), so parent/child edges + * stay far outside the 1µs CIM representation interval. Numeric ticks only. */ +const BASE_FT = 134_367_000_000_000_000n; +const T = (units: number): string => String(BASE_FT + BigInt(units) * 100n); +const ORIG_TICKS = "100000000000000000"; // the worker's persisted startup fingerprint +const REUSER_TICKS = "300000000000000000"; // a foreign generation now owning the pid + +/** Latches a terminal task_1 with recorded members [workerPid, 334455] and + * leaves task_2 (same execution directory) queued behind the occupancy. The + * persisted executor identity carries a NUMERIC fingerprint so the scan's + * fingerprint comparison (not the null branch) drives every release probe. */ +async function latchTerminalTask(table: ManagerScanTable, options: { maxConcurrentWorkers?: number; task2Workspace?: string } = {}) { + const fx = await makeManagerFixture({ scanOpen: async () => table, maxConcurrentWorkers: options.maxConcurrentWorkers }); + await fx.manager.createTask(fx.makeTask()); + await fx.manager.createTask(fx.makeTask({ task_id: "task_2", ...(options.task2Workspace ? { workspace: options.task2Workspace } : {}) })); + const workerPid = fx.spawned[0]!.pid; + // A failed termination latches cleanup_unverified with known members. + fx.setTerminateError(new WindowsTreeCleanupError("process_tree_cleanup_unverified", "known descendant still alive", [workerPid, 334_455])); + await assert.rejects(fx.manager.cancelTask("task_1"), /could not be verified/); + assert.equal(fx.store.readStatus("task_1").cleanup_unverified, true); + // The worker later exits naturally and commits its terminal result; the + // store must preserve the latch through commitWorkerResult. + fx.pidsAlive.delete(workerPid); + await fx.runWorker("task_1", new FakeAdapter()); + const committed = fx.store.readStatus("task_1"); + assert.equal(committed.status, "cancelled", "the raced worker commit finishes the task"); + assert.equal(committed.cleanup_unverified, true, "the worker commit cannot dissolve the latch"); + assert.match(String(committed.error), /\[cleanup-members: /, "the member marker survives the commit"); + // Deterministic release-probe identity: the recorded worker pid with its + // original startup fingerprint (real flows persist this from the worker). + fx.store.writeExecutorIdentity("task_1", 1, { worker: { pid: workerPid, fingerprint: ORIG_TICKS, precision: "exact" } }); + // The pump must NOT start task_2 while the latched task holds its slot/path. + await fx.manager.recoverTasks(); + assert.equal(fx.store.readStatus("task_2").status, "queued", "a queued task stays queued behind the latch"); + assert.equal(fx.spawned.length, 1); + return fx; +} + +test("manager gate P1-B: a foreign-reused worker pid with an old-generation child keeps the task and directory occupied", async () => { + const table = new ManagerScanTable(); + const fx = await latchTerminalTask(table); + try { + const workerPid = fx.spawned[0]!.pid; + // P1-B table shape: pid workerPid is alive as a FOREIGN generation and a + // child created BEFORE the reuser still claims it. Before the scan fix + // this returned clean=true and the manager released the directory. + table.rows = [ + { pid: workerPid, parentPid: 4, creationFileTime: T(3), startTicks: REUSER_TICKS }, + { pid: workerPid + 1, parentPid: workerPid, creationFileTime: T(1), startTicks: "200000000000000000" }, + ]; + table.holdOverrides.set(workerPid, { pid: workerPid, status: "ok", creationFileTime: T(3), creationTicks: REUSER_TICKS }); + + await assert.rejects(fx.manager.cancelTask("task_1"), /1 recorded-tree process\(es\) still alive/); + const after = fx.store.readStatus("task_1"); + assert.equal(after.status, "cancelled", "the terminal result is preserved"); + assert.equal(after.cleanup_unverified, true, "a false scan never releases the latch"); + assert.equal(fx.store.readStatus("task_2").status, "queued", "the same-directory task stays blocked"); + assert.equal(fx.spawned.length, 1, "no second worker started"); + assert.ok(table.holds.some((h) => h.length === 1 && h[0] === workerPid), "the fingerprinted recorded root was actually held"); + assert.ok(fx.scanCalls.some((targets) => targets.some((t) => t.pid === workerPid && t.fingerprint === ORIG_TICKS)), "the scan saw the recorded fingerprint"); + // The provably-clean variant of the SAME lineage releases: the observed + // old-generation child is gone (its S2-persisted fingerprint-less record + // scans it as a dead member) and only the provably-foreign reuser with a + // strictly-later, held, row-consistent new child remains. + table.rows = [ + { pid: workerPid, parentPid: 4, creationFileTime: T(3), startTicks: REUSER_TICKS }, + { pid: workerPid + 2, parentPid: workerPid, creationFileTime: T(9), startTicks: "900000000000000000" }, + ]; + const released = await fx.manager.cancelTask("task_1"); + assert.equal(released.status, "cancelled"); + assert.equal(fx.store.readStatus("task_1").cleanup_unverified ?? false, false, "a provably-clean scan releases the latch"); + await fx.manager.recoverTasks(); + assert.equal(fx.store.readStatus("task_2").status, "running", "the blocked same-directory task starts after the release"); + assert.equal(fx.spawned.length, 2); + } finally { + await fx.cleanup(); + } +}); + +test("manager gate P1-A: the recorded root vanishing between enumeration and hold keeps occupancy through its live child", async () => { + const table = new ManagerScanTable(); + const fx = await latchTerminalTask(table); + try { + const workerPid = fx.spawned[0]!.pid; + // P1-A table shape, driven through the FINGERPRINTED recorded root: the + // root was enumerated alive but vanishes before its hold; its live child + // stays occupied. The fingerprint comparison makes the scan actually call + // hold (the null-fingerprint shortcut never runs); the original defective + // scanner kept the stale live entry, proved clean, and released. + table.rows = [ + { pid: workerPid, parentPid: 4, creationFileTime: T(1), startTicks: "100000000000000001" }, + { pid: workerPid + 1, parentPid: workerPid, creationFileTime: T(2), startTicks: "200000000000000000" }, + ]; + table.holdOverrides.set(workerPid, { pid: workerPid, status: "absent" }); + + await assert.rejects(fx.manager.cancelTask("task_1"), /still alive/); + const after = fx.store.readStatus("task_1"); + assert.equal(after.cleanup_unverified, true, "the vanished root's live child keeps the latch"); + assert.equal(fx.store.readStatus("task_2").status, "queued"); + assert.equal(fx.spawned.length, 1); + assert.ok(table.holds.some((h) => h.length === 1 && h[0] === workerPid), "the absent hold on the fingerprinted root actually happened"); + + // The root is fully gone (not even enumerated): dead recorded pid, its + // child still lives — attribution stays impossible, occupancy stays. + table.rows = [{ pid: workerPid + 1, parentPid: workerPid, creationFileTime: T(2), startTicks: "200000000000000000" }]; + table.holdOverrides.clear(); + await assert.rejects(fx.manager.cancelTask("task_1"), /still alive/); + assert.equal(fx.store.readStatus("task_1").cleanup_unverified, true); + assert.equal(fx.store.readStatus("task_2").status, "queued"); + + // Everything provably gone: the release works and the queue moves. + table.rows = []; + const released = await fx.manager.cancelTask("task_1"); + assert.equal(released.status, "cancelled"); + await fx.manager.recoverTasks(); + assert.equal(fx.store.readStatus("task_2").status, "running"); + assert.equal(fx.spawned.length, 2); + } finally { + await fx.cleanup(); + } +}); + +test("manager gate: a latched task retains the worker SLOT against a different execution directory at max concurrency 1", async () => { + const table = new ManagerScanTable(); + const otherDir = mkdtempSync(path.join(tmpdir(), "zcode-bridge-slot-")); + const fx = await latchTerminalTask(table, { maxConcurrentWorkers: 1, task2Workspace: otherDir }); + try { + const workerPid = fx.spawned[0]!.pid; + // Different directory: only the worker SLOT (not path overlap) can keep + // task_2 queued while the latched task_1 holds the single slot. + assert.notEqual(path.resolve(fx.workspaceDir), path.resolve(otherDir)); + assert.equal(fx.store.readStatus("task_2").status, "queued", "the different-directory task is slot-blocked"); + assert.equal(fx.spawned.length, 1); + + // A false scan (foreign reuse with an old-generation child) keeps the slot. + table.rows = [ + { pid: workerPid, parentPid: 4, creationFileTime: T(3), startTicks: REUSER_TICKS }, + { pid: workerPid + 1, parentPid: workerPid, creationFileTime: T(1), startTicks: "200000000000000000" }, + ]; + table.holdOverrides.set(workerPid, { pid: workerPid, status: "ok", creationFileTime: T(3), creationTicks: REUSER_TICKS }); + await assert.rejects(fx.manager.cancelTask("task_1"), /still alive/); + await fx.manager.recoverTasks(); + assert.equal(fx.store.readStatus("task_2").status, "queued", "the slot stays retained while the latch holds"); + assert.equal(fx.spawned.length, 1); + + // Provable clean releases the slot and the different-directory task starts. + table.rows = []; + table.holdOverrides.clear(); + const released = await fx.manager.cancelTask("task_1"); + assert.equal(released.status, "cancelled"); + await fx.manager.recoverTasks(); + assert.equal(fx.store.readStatus("task_2").status, "running", "the slot frees after the verified release"); + assert.equal(fx.spawned.length, 2); + } finally { + await fx.cleanup(); + rmSync(otherDir, { recursive: true, force: true, maxRetries: 10, retryDelay: 50 }); + } +}); + +test("manager gate: a rejected scan keeps the latch instead of releasing or narrowing", async () => { + const table = new ManagerScanTable(); + const fx = await latchTerminalTask(table); + try { + const workerPid = fx.spawned[0]!.pid; + table.rows = [{ pid: workerPid, parentPid: 4, creationFileTime: T(1), startTicks: "300000000000000000" }]; + table.enumerateError = new Error("process_tree_transport_timeout: enum begin"); + await assert.rejects(fx.manager.cancelTask("task_1"), /ZCode cleanup verification failed: process_tree_scan_failed: process_tree_transport_timeout/); + let after = fx.store.readStatus("task_1"); + assert.equal(after.cleanup_unverified, true, "a rejected scan never releases"); + assert.equal(fx.store.readStatus("task_2").status, "queued"); + + // A malformed hold response (wrong pid for the request) refuses the scan + // through the same manager path — never a false clean. + table.enumerateError = null; + table.rows = [{ pid: workerPid, parentPid: 4, creationFileTime: T(1), startTicks: "300000000000000000" }]; + const originalHold = table.hold.bind(table); + table.hold = async (pids: readonly number[]) => { + const outcomes = await originalHold(pids); + return outcomes.map((o) => (o.pid === pids[0] ? { pid: 999, status: "ok", creationFileTime: T(3), creationTicks: "300000000000000000" } as HoldOutcome : o)); + }; + await assert.rejects(fx.manager.cancelTask("task_1"), /ZCode cleanup verification failed: process_tree_hold_invalid/); + after = fx.store.readStatus("task_1"); + assert.equal(after.cleanup_unverified, true); + assert.equal(fx.store.readStatus("task_2").status, "queued"); + assert.equal(fx.spawned.length, 1); + + // A helper close failure on an otherwise-clean table is equally visible. + table.hold = originalHold; + table.rows = []; + table.closeError = new Error("synthetic scan close failure"); + await assert.rejects(fx.manager.cancelTask("task_1"), /verification failed/); + after = fx.store.readStatus("task_1"); + assert.equal(after.cleanup_unverified, true); + assert.equal(fx.store.readStatus("task_2").status, "queued"); + assert.equal(fx.spawned.length, 1); + } finally { + await fx.cleanup(); + } +}); + +test("manager gate S1: a pid foreign to its own record but a live child of another dead recorded member keeps occupancy", async () => { + const table = new ManagerScanTable(); + const fx = await latchTerminalTask(table); + try { + const workerPid = fx.spawned[0]!.pid; + // workerPid currently holds a FOREIGN generation relative to its own + // recorded fingerprint, but its row claims the OTHER recorded member + // 334455 (dead) as parent: claimant evidence from that root wins over + // the foreign exclusion (review-round-2 S1). The attempt-2 scanner + // skipped it as foreign and wrongly proved clean. + table.rows = [ + { pid: workerPid, parentPid: 334_455, creationFileTime: T(3), startTicks: REUSER_TICKS }, + { pid: workerPid + 1, parentPid: workerPid, creationFileTime: T(4), startTicks: "400000000000000000" }, + ]; + table.holdOverrides.set(workerPid, { pid: workerPid, status: "ok", creationFileTime: T(3), creationTicks: REUSER_TICKS }); + + await assert.rejects(fx.manager.cancelTask("task_1"), /still alive/); + const after = fx.store.readStatus("task_1"); + assert.equal(after.cleanup_unverified, true, "cross-root claimant evidence keeps the latch"); + assert.equal(fx.store.readStatus("task_2").status, "queued"); + assert.equal(fx.spawned.length, 1); + assert.ok(table.holds.some((h) => h.length === 1 && h[0] === workerPid), "the reused pid was held and proven foreign relative to its own record"); + + // With the cross-root row gone the same lineage is provably foreign and + // the release works again. + table.rows = [ + { pid: workerPid, parentPid: 4, creationFileTime: T(3), startTicks: REUSER_TICKS }, + ]; + const released = await fx.manager.cancelTask("task_1"); + assert.equal(released.status, "cancelled"); + await fx.manager.recoverTasks(); + assert.equal(fx.store.readStatus("task_2").status, "running"); + assert.equal(fx.spawned.length, 2); + } finally { + await fx.cleanup(); + } +}); + +test("manager gate S2: a false scan persists its observed members and later cancels keep scanning them", async () => { + const table = new ManagerScanTable(); + const fx = await latchTerminalTask(table); + try { + const workerPid = fx.spawned[0]!.pid; + const child = workerPid + 1_000; + const grand = child + 1; + // First scan: the dead root's live child and grandchild are claimants. + table.rows = [ + { pid: child, parentPid: workerPid, creationFileTime: T(2), startTicks: "200000000000000000" }, + { pid: grand, parentPid: child, creationFileTime: T(3), startTicks: "300000000000000000" }, + ]; + await assert.rejects(fx.manager.cancelTask("task_1"), /still alive/); + const record = fx.store.readAttemptMeta<{ member_pids?: number[] }>("task_1", 1, "cleanup-failure.json"); + assert.ok(record?.member_pids?.includes(child), "the observed child joined the durable member record"); + assert.ok(record?.member_pids?.includes(grand), "the observed grandchild joined the durable member record"); + + // The OBSERVED intermediate disappears; its still-live child must not + // become an unobserved-ancestor exception: the next cancel scans the + // persisted union and stays occupied. + table.rows = [{ pid: grand, parentPid: child, creationFileTime: T(3), startTicks: "300000000000000000" }]; + await assert.rejects(fx.manager.cancelTask("task_1"), /still alive/); + assert.equal(fx.store.readStatus("task_1").cleanup_unverified, true); + assert.equal(fx.store.readStatus("task_2").status, "queued"); + assert.equal(fx.spawned.length, 1); + const laterTargets = fx.scanCalls.at(-1) ?? []; + assert.ok(laterTargets.some((t) => t.pid === child), "the persisted observed pid is scanned again"); + + // Once every observed pid is truly gone the release works and the queue + // moves again. + table.rows = []; + const released = await fx.manager.cancelTask("task_1"); + assert.equal(released.status, "cancelled"); + await fx.manager.recoverTasks(); + assert.equal(fx.store.readStatus("task_2").status, "running"); + assert.equal(fx.spawned.length, 2); + } finally { + await fx.cleanup(); + } +}); + +test("manager gate S2: a rejected scan after observing a candidate retains it across later cancels", async () => { + const table = new ManagerScanTable(); + const fx = await latchTerminalTask(table); + try { + const workerPid = fx.spawned[0]!.pid; + const child = workerPid + 1_000; + const grand = child + 1; + // The root is alive as a foreign generation; the strictly-later child is + // requested and its hold returns no response: the scan rejects, and the + // requested candidate (plus the root) must persist before the cancel + // fails. + table.rows = [ + { pid: workerPid, parentPid: 4, creationFileTime: T(3), startTicks: REUSER_TICKS }, + { pid: child, parentPid: workerPid, creationFileTime: T(4), startTicks: "400000000000000000" }, + ]; + table.holdOverrides.set(workerPid, { pid: workerPid, status: "ok", creationFileTime: T(3), creationTicks: REUSER_TICKS }); + table.holdEmptyFor.add(child); + await assert.rejects(fx.manager.cancelTask("task_1"), /ZCode cleanup verification failed: process_tree_hold_invalid/); + const record = fx.store.readAttemptMeta<{ member_pids?: number[] }>("task_1", 1, "cleanup-failure.json"); + assert.ok(record?.member_pids?.includes(child), "the requested-but-unresolved candidate joined the durable record"); + assert.equal(fx.store.readStatus("task_1").cleanup_unverified, true); + assert.equal(fx.store.readStatus("task_2").status, "queued"); + + // The candidate disappears while its own child stays live: the persisted + // observation keeps the grandchild covered (no unobserved-ancestor gap). + table.rows = [{ pid: grand, parentPid: child, creationFileTime: T(5), startTicks: "500000000000000000" }]; + table.holdEmptyFor.clear(); + table.holdOverrides.clear(); + await assert.rejects(fx.manager.cancelTask("task_1"), /still alive/); + assert.equal(fx.store.readStatus("task_1").cleanup_unverified, true); + assert.equal(fx.store.readStatus("task_2").status, "queued"); + assert.equal(fx.spawned.length, 1); + + // Empty closure releases normally. + table.rows = []; + const released = await fx.manager.cancelTask("task_1"); + assert.equal(released.status, "cancelled"); + await fx.manager.recoverTasks(); + assert.equal(fx.store.readStatus("task_2").status, "running"); + } finally { + await fx.cleanup(); + } +}); diff --git a/test/manager-helpers.ts b/test/manager-helpers.ts index 613eaa5..7e3a930 100644 --- a/test/manager-helpers.ts +++ b/test/manager-helpers.ts @@ -18,6 +18,7 @@ import { BridgeTaskManager } from "../src/manager/task-manager.js"; import { DirectWorkspaceProvider } from "../src/workspace/direct-provider.js"; import type { ZCodeRunOutcome } from "../src/adapters/zcode-adapter.js"; import type { TerminateProcessTree } from "../src/adapters/process-spawn.js"; +import { verifyWindowsRecordedTreeClaimants, type WindowsTreeTransactionLike } from "../src/adapters/windows-process-cleanup.js"; import { SESSION_ID, validReport } from "./helpers.js"; export type FakeBehavior = @@ -165,6 +166,11 @@ export async function makeManagerFixture(options: { maxConcurrentWorkers?: number; /** Defaults to 0 (immediate worker_lost finalization) for legacy tests. */ workerStartGraceMs?: number; + /** When provided, the manager's release gate runs the REAL recorded-tree + * claimant scan (verifyWindowsRecordedTreeClaimants) against the OS + * transaction returned by this factory — only the transport is faked; the + * scan logic and the whole manager cancel/occupancy path stay real. */ + scanOpen?: () => Promise; } = {}): Promise { const dataRoot = await mkdtemp(path.join(tmpdir(), "zcode-bridge-mgr-")); const workspaceDir = await mkdtemp(path.join(tmpdir(), "zcode-bridge-ws-")); @@ -187,11 +193,16 @@ export async function makeManagerFixture(options: { return { pid, signal: "SIGKILL", verified: true }; }; - const verifyTreeClean = async (targets: readonly { pid: number; fingerprint: string | null }[]) => { - scanCalls.push([...targets]); - if (scanResult instanceof Error) throw scanResult; - return scanResult; - }; + const verifyTreeClean = options.scanOpen + ? async (targets: readonly { pid: number; fingerprint: string | null }[]) => { + scanCalls.push([...targets]); + return verifyWindowsRecordedTreeClaimants(targets, { open: options.scanOpen! }); + } + : async (targets: readonly { pid: number; fingerprint: string | null }[]) => { + scanCalls.push([...targets]); + if (scanResult instanceof Error) throw scanResult; + return scanResult; + }; const manager = new BridgeTaskManager({ store, diff --git a/test/windows-process-cleanup.test.ts b/test/windows-process-cleanup.test.ts index 1cb6d4a..9cf51f0 100644 --- a/test/windows-process-cleanup.test.ts +++ b/test/windows-process-cleanup.test.ts @@ -7,6 +7,7 @@ import { resolveWindowsTreeMembership, terminateWindowsProcessTree, verifyWindowsRecordedTreeClaimants, + WindowsTreeCleanupError, type HeldProcessInfo, type HoldOutcome, type KillOutcome, @@ -357,11 +358,96 @@ test("driver: dynamic descendants inside a killed parent's window are terminated assert.deepEqual(receipt.members.map((m) => m.pid), [101, 102]); }); +// -- PR #47 process-lock budget regressions ------------------------------------ +// +// Measured on Windows (see the repair plan's stage-timing evidence): one +// full-table enumeration costs ~1.1-1.5s, so the two enumerations a normal +// reap needs already consume ~60% of the 5s kill budget. The old driver +// checked the deadline after EVERY kill pass unconditionally, so a slow +// enumeration plus a stale WMI row for an already-terminated member produced +// process_tree_cleanup_unverified — "not provably exited" — for members whose +// exit FILETIME had already been read from their held handles. These +// regressions pin the corrected semantics: the budget refuses only members +// WITHOUT a confirmed exit; confirmed members keep their proof. + +test("driver budget: a stale row for a confirmed-exit member past the deadline completes the closure proof", async () => { + const round1 = [row(101, 4, 0n), row(102, 101, 40n)]; + // Round 2 still lists both members (representation lag): each kill returns + // already_exited WITH the exit FILETIME read on the held handle. + const snapshots = [[...round1], [...round1], []]; + const kills: Array<{ pid: number; status: string }> = []; + const clock = { now: 0 }; + const tx: WindowsTreeTransactionLike = { + enumerate: async () => { + const snapshot = snapshots.shift() ?? []; + clock.now += 2_600; // each full-table enumeration outpaces the budget + return snapshot; + }, + hold: async (pids) => pids.map((pid) => { + const target = round1.find((r) => r.pid === pid)!; + return { pid, status: "ok", creationFileTime: FT(BigInt(target.creationFileTime!) + 3n), creationTicks: target.startTicks! } as HoldOutcome; + }), + kill: async (pid) => { + kills.push({ pid, status: "already_exited" }); + return { status: "already_exited", exitFileTime: FT(BASE + 900_000n) }; + }, + close: async () => undefined, + }; + const receipt = await terminateWindowsProcessTree(101, "101-ticks", 5_000, { + open: async () => tx, + now: () => clock.now, + sleep: async () => undefined, + }); + assert.equal(receipt.rounds, 3, "the stale-row round and the confirming round both ran"); + assert.deepEqual(receipt.signaled, [], "no signal is issued for an already-exited member"); + assert.ok(kills.length >= 4, "both members were re-checked across the stale round"); +}); + +test("driver budget: an unconfirmed member past the deadline still refuses conservatively", async () => { + const snapshots = [[row(101, 4, 0n)], []]; + const clock = { now: 0 }; + const tx: WindowsTreeTransactionLike = { + enumerate: async () => { + const snapshot = snapshots.shift() ?? []; + clock.now += 6_000; // the very first enumeration already crosses the deadline + return snapshot; + }, + hold: async (pids) => pids.map((pid) => { + const target = row(101, 4, 0n); + return { pid, status: "ok", creationFileTime: FT(BigInt(target.creationFileTime!) + 3n), creationTicks: target.startTicks! } as HoldOutcome; + }), + kill: async () => ({ status: "wait_timeout", exitFileTime: null }), // exit NOT confirmed + close: async () => undefined, + }; + await assert.rejects( + terminateWindowsProcessTree(101, "101-ticks", 5_000, { + open: async () => tx, + now: () => clock.now, + sleep: async () => undefined, + }), + (error: unknown) => { + assert.ok(error instanceof WindowsTreeCleanupError); + assert.equal(error.code, "process_tree_cleanup_unverified"); + assert.match(error.message, /1 recorded member\(s\) not provably exited/); + assert.deepEqual(error.memberPids, [101]); + return true; + }, + ); +}); + // -- recorded-tree claimant scan ---------------------------------------------- +// Scan-section rows use NUMERIC startup ticks, matching the real transport +// contract (decimal strings): hold responses are validated against that +// shape before any identity proof is drawn from them (review-round-1 R2). +const SCAN_TICKS = (off: bigint): string => FT(BASE + off * MICROS + 7n); +function srow(pid: number, parentPid: number, cimOffset: bigint, ticks?: string): WindowsProcessRow { + return { pid, parentPid, creationFileTime: FT(BASE + cimOffset * MICROS), startTicks: ticks ?? SCAN_TICKS(cimOffset) }; +} + test("scan: clean when nothing claims the recorded pids", async () => { - const fake = new FakeTransaction([row(700, 4, 0n), row(701, 700, 10n)]); - const scan = await verifyWindowsRecordedTreeClaimants([{ pid: 101, fingerprint: "101-ticks" }], { + const fake = new FakeTransaction([srow(700, 4, 0n), srow(701, 700, 10n)]); + const scan = await verifyWindowsRecordedTreeClaimants([{ pid: 101, fingerprint: SCAN_TICKS(0n) }], { open: async () => fake, now: () => 0, sleep: async () => undefined, @@ -372,8 +458,8 @@ test("scan: clean when nothing claims the recorded pids", async () => { }); test("scan: a live recorded pid with a matching fingerprint is a claimant", async () => { - const fake = new FakeTransaction([row(101, 4, 0n)]); - const scan = await verifyWindowsRecordedTreeClaimants([{ pid: 101, fingerprint: "101-ticks" }], { + const fake = new FakeTransaction([srow(101, 4, 0n)]); + const scan = await verifyWindowsRecordedTreeClaimants([{ pid: 101, fingerprint: SCAN_TICKS(0n) }], { open: async () => fake, now: () => 0, sleep: async () => undefined, @@ -382,19 +468,22 @@ test("scan: a live recorded pid with a matching fingerprint is a claimant", asyn assert.deepEqual(scan.livePids, [101]); }); -test("scan: a proven foreign generation of a recorded pid is excluded together with its subtree", async () => { - const fake = new FakeTransaction([row(101, 4, 0n, "reuser-ticks"), row(155, 101, 40n)]); - const scan = await verifyWindowsRecordedTreeClaimants([{ pid: 101, fingerprint: "101-ticks" }], { +test("scan: a proven foreign generation of a recorded pid does not inherit the recorded pid's old children", async () => { + // The reuser currently owns pid 101; row 155 was created AFTER the reuser's + // exact creation, so it provably belongs to the foreign generation and is + // excluded. This is the ONLY subtree the foreign parent can exclude. + const fake = new FakeTransaction([srow(101, 4, 0n, "1"), srow(155, 101, 40n)]); + const scan = await verifyWindowsRecordedTreeClaimants([{ pid: 101, fingerprint: SCAN_TICKS(0n) }], { open: async () => fake, now: () => 0, sleep: async () => undefined, }); - assert.equal(scan.clean, true, "the reuser and its own child are not ours"); + assert.equal(scan.clean, true, "the reuser and its provably-new child are not ours"); }); test("scan: children of a dead recorded pid stay claimants (attribution is impossible)", async () => { - const fake = new FakeTransaction([row(180, 101, 40n)]); - const scan = await verifyWindowsRecordedTreeClaimants([{ pid: 101, fingerprint: "101-ticks" }], { + const fake = new FakeTransaction([srow(180, 101, 40n)]); + const scan = await verifyWindowsRecordedTreeClaimants([{ pid: 101, fingerprint: SCAN_TICKS(0n) }], { open: async () => fake, now: () => 0, sleep: async () => undefined, @@ -404,7 +493,7 @@ test("scan: children of a dead recorded pid stay claimants (attribution is impos }); test("scan: a live recorded pid without a fingerprint stays a claimant (unknown is never clean)", async () => { - const fake = new FakeTransaction([row(101, 4, 0n)]); + const fake = new FakeTransaction([srow(101, 4, 0n)]); const scan = await verifyWindowsRecordedTreeClaimants([{ pid: 101, fingerprint: null }], { open: async () => fake, now: () => 0, @@ -412,3 +501,571 @@ test("scan: a live recorded pid without a fingerprint stays a claimant (unknown }); assert.equal(scan.clean, false); }); + +// -- PR #47 P1-A/P1-B: recorded-tree scan attribution regressions ------------- +// +// Scripted scan fixture: a fixed process table plus per-pid hold overrides, +// so a row can vanish between enumeration and hold (P1-A) or expose a +// foreign generation with an exact creation time (P1-B). No OS, no signals. + +class ScriptedScan implements WindowsTreeTransactionLike { + rows: WindowsProcessRow[]; + readonly holds: number[][] = []; + readonly holdOverrides = new Map(); + readonly holdResults = new Map(); + closeCount = 0; + closeError: Error | null = null; + + constructor(rows: readonly WindowsProcessRow[]) { + this.rows = [...rows]; + } + + async enumerate(): Promise { + return [...this.rows]; + } + + async hold(pids: readonly number[]): Promise { + this.holds.push([...pids]); + const scripted = this.holdResults.get(pids.length); + if (scripted) return scripted; + return pids.map((pid) => { + const override = this.holdOverrides.get(pid); + if (override) return override; + const target = this.rows.find((r) => r.pid === pid); + if (!target) return { pid, status: "absent" } as HoldOutcome; + if (!target.startTicks || !target.creationFileTime) return { pid, status: "denied", error: 5 } as HoldOutcome; + return { + pid, + status: "ok", + creationFileTime: FT(BigInt(target.creationFileTime) + 3n), + creationTicks: target.startTicks, + } as HoldOutcome; + }); + } + + async kill(): Promise { + throw new Error("read-only scan must not signal"); + } + + async close(): Promise { + this.closeCount += 1; + if (this.closeError) throw this.closeError; + } +} + +const scanDeps = (tx: ScriptedScan) => ({ open: async () => tx, now: () => 0, sleep: async () => undefined }); + +test("scan P1-A (maintainer counterexample): a recorded pid vanishing between enumeration and hold keeps its live child occupied", async () => { + // Exact maintainer shape: rows use raw small filetime values; hold for the + // enumerated parent returns absent. The child claims the vanished parent. + const tx = new ScriptedScan([ + { pid: 101, parentPid: 4, creationFileTime: "100", startTicks: "100" }, + { pid: 102, parentPid: 101, creationFileTime: "200", startTicks: "200" }, + ]); + tx.holdResults.set(1, [{ pid: 101, status: "absent" }]); + const scan = await verifyWindowsRecordedTreeClaimants([{ pid: 101, fingerprint: "100" }], scanDeps(tx)); + assert.equal(scan.clean, false, "a vanished parent's live child is never provably clean"); + assert.deepEqual(scan.livePids, [102]); + assert.equal(tx.closeCount, 1); +}); + +test("scan P1-A: an absent parent's whole live subtree stays occupied (multi-level)", async () => { + const tx = new ScriptedScan([srow(101, 4, 0n), srow(102, 101, 40n), srow(103, 102, 80n)]); + tx.holdOverrides.set(101, { pid: 101, status: "absent" }); + const scan = await verifyWindowsRecordedTreeClaimants([{ pid: 101, fingerprint: SCAN_TICKS(0n) }], scanDeps(tx)); + assert.equal(scan.clean, false); + assert.deepEqual(scan.livePids, [102, 103]); +}); + +test("scan P1-B (maintainer counterexample): an old-generation child under a reused recorded pid stays a claimant", async () => { + // Exact maintainer shape: the recorded fingerprint is 100 but pid 101 is now + // generation 300 (foreign); the child was created at 200, BEFORE the + // reuser's exact creation — the foreign generation cannot own it. + const tx = new ScriptedScan([ + { pid: 101, parentPid: 4, creationFileTime: "300", startTicks: "300" }, + { pid: 102, parentPid: 101, creationFileTime: "200", startTicks: "200" }, + ]); + tx.holdResults.set(1, [{ pid: 101, status: "ok", creationFileTime: "300", creationTicks: "300" }]); + const scan = await verifyWindowsRecordedTreeClaimants([{ pid: 101, fingerprint: "100" }], scanDeps(tx)); + assert.equal(scan.clean, false, "a foreign parent never proves its old-generation children foreign"); + assert.deepEqual(scan.livePids, [102]); +}); + +test("scan P1-B: old and provably-new children under one foreign parent are decided per edge", async () => { + // Foreign exact creation F = BASE+3 (hold adds 3 to the row's CIM). + const tx = new ScriptedScan([srow(101, 4, 0n, "1"), srow(102, 101, -20n), srow(103, 101, 40n)]); + const scan = await verifyWindowsRecordedTreeClaimants([{ pid: 101, fingerprint: SCAN_TICKS(0n) }], scanDeps(tx)); + assert.equal(scan.clean, false); + assert.deepEqual(scan.livePids, [102], "the old child stays; the provably-new child is excluded"); +}); + +test("scan P1-B boundary: a child whose CIM interval straddles the foreign creation is not provably foreign", async () => { + // Child CIM = BASE (same anchor as the parent row) while F = BASE+3: the + // child's true creation lies in [BASE, BASE+1us] and could be before F. + // Zero tolerance: only a representation wholly at/after F is excluded. + const tx = new ScriptedScan([srow(101, 4, 0n, "1"), srow(102, 101, 0n)]); + const scan = await verifyWindowsRecordedTreeClaimants([{ pid: 101, fingerprint: SCAN_TICKS(0n) }], scanDeps(tx)); + assert.equal(scan.clean, false); + assert.deepEqual(scan.livePids, [102]); +}); + +test("scan P1-B boundary: a child whose representation equals the foreign creation is indistinguishable, not foreign", async () => { + // Hold returns the exact foreign creation; a child anchored exactly there + // could have been created in the same 100ns instant as the reuser. The + // ADR-009 same-precision rule applies to exclusions too: only STRICTLY + // later representations prove the foreign edge (review-round-1 R1). + const tx = new ScriptedScan([ + { pid: 101, parentPid: 4, creationFileTime: "300", startTicks: "300" }, + { pid: 102, parentPid: 101, creationFileTime: "300", startTicks: "300c" }, + ]); + tx.holdResults.set(1, [{ pid: 101, status: "ok", creationFileTime: "300", creationTicks: "300" }]); + const scan = await verifyWindowsRecordedTreeClaimants([{ pid: 101, fingerprint: "100" }], scanDeps(tx)); + assert.equal(scan.clean, false); + assert.deepEqual(scan.livePids, [102]); + assert.deepEqual(tx.holds, [[101]], "an unprovable child is never held"); +}); + +test("scan P1-B boundary: a child strictly after the foreign creation is excluded only through its own held identity", async () => { + // The positive counterpart: the edge is strictly proven AND the child's + // own hold is row-consistent — the child is held before being excluded, + // so its descendants can be judged against ITS held exact creation. + const tx = new ScriptedScan([ + { pid: 101, parentPid: 4, creationFileTime: "300", startTicks: "300" }, + { pid: 102, parentPid: 101, creationFileTime: "400", startTicks: "400" }, + ]); + tx.holdOverrides.set(101, { pid: 101, status: "ok", creationFileTime: "300", creationTicks: "300" }); + const scan = await verifyWindowsRecordedTreeClaimants([{ pid: 101, fingerprint: "100" }], scanDeps(tx)); + assert.equal(scan.clean, true); + assert.deepEqual(scan.livePids, []); + assert.deepEqual(tx.holds, [[101], [102]], "the excluded child was itself held and verified"); +}); + +test("scan P1-B: a foreign descendant's own children are excluded only when created clearly after it", async () => { + // 102 is excluded as a provable foreign child (held, row-consistent); + // grandchild 103 created strictly after 102's HELD exact creation is + // excluded transitively through its own held identity, while a grandchild + // 104 predating that creation cannot be proven foreign and stays occupied. + const after = new ScriptedScan([ + srow(101, 4, 0n, "1"), + srow(102, 101, 40n), + srow(103, 102, 100n), + ]); + const cleanScan = await verifyWindowsRecordedTreeClaimants([{ pid: 101, fingerprint: SCAN_TICKS(0n) }], scanDeps(after)); + assert.equal(cleanScan.clean, true, "the whole provably-foreign chain is excluded, each member held"); + assert.deepEqual(after.holds, [[101], [102], [103]], "every excluded member was individually held and verified"); + + const before = new ScriptedScan([ + srow(101, 4, 0n, "1"), + srow(102, 101, 40n), + srow(104, 102, 20n), + ]); + const dirtyScan = await verifyWindowsRecordedTreeClaimants([{ pid: 101, fingerprint: SCAN_TICKS(0n) }], scanDeps(before)); + assert.equal(dirtyScan.clean, false); + assert.deepEqual(dirtyScan.livePids, [104]); +}); + +test("scan P1-B: a same-precision grandchild under an excluded foreign child stays a claimant", async () => { + // 103's CIM equals its excluded parent 102's CIM; 102's held exact creation + // sits at CIM+3 ticks, so 103's representation does not lie strictly after + // it — the edge direction is indistinguishable, refuse. + const tx = new ScriptedScan([ + srow(101, 4, 0n, "1"), + srow(102, 101, 40n), + srow(103, 102, 40n), + ]); + const scan = await verifyWindowsRecordedTreeClaimants([{ pid: 101, fingerprint: SCAN_TICKS(0n) }], scanDeps(tx)); + assert.equal(scan.clean, false); + assert.deepEqual(scan.livePids, [103]); +}); + +// -- review-round-1 R1: intermediate foreign-lineage members need held proof -- + +test("scan R1: an intermediate foreign descendant that vanishes before its hold keeps its subtree occupied", async () => { + // 101 is a proven foreign generation; 102 is created strictly after it and + // would be excludable, but it disappears between enumeration and hold: it + // is dead, unattributable, and its live grandchild 103 stays a claimant. + const tx = new ScriptedScan([ + srow(101, 4, 0n, "1"), + srow(102, 101, 40n), + srow(103, 102, 100n), + ]); + tx.holdOverrides.set(102, { pid: 102, status: "absent" }); + const scan = await verifyWindowsRecordedTreeClaimants([{ pid: 101, fingerprint: SCAN_TICKS(0n) }], scanDeps(tx)); + assert.equal(scan.clean, false); + assert.deepEqual(scan.livePids, [103], "the vanished intermediate itself is gone; its child stays occupied"); + assert.ok(tx.holds.some((h) => h.length === 1 && h[0] === 102), "the intermediate was actually held"); +}); + +test("scan R1: an intermediate foreign descendant whose identity read is denied stays occupied with its subtree", async () => { + const tx = new ScriptedScan([ + srow(101, 4, 0n, "1"), + srow(102, 101, 40n), + srow(103, 102, 100n), + ]); + tx.holdOverrides.set(102, { pid: 102, status: "denied", error: 5 }); + const scan = await verifyWindowsRecordedTreeClaimants([{ pid: 101, fingerprint: SCAN_TICKS(0n) }], scanDeps(tx)); + assert.equal(scan.clean, false); + assert.deepEqual(scan.livePids, [102, 103]); +}); + +test("scan R1: an intermediate whose held generation replaced its enumerated row stays occupied with its subtree", async () => { + // The row says generation 400, but the held handle reports ticks 600: the + // enumerated row and the current owner are different generations, so the + // grandchild (created 500, before the 600 generation began) cannot be + // attributed to the held one — everything stays occupied. + const tx = new ScriptedScan([ + srow(101, 4, 0n, "1"), + srow(102, 101, 40n), + srow(103, 102, 100n), + ]); + tx.holdOverrides.set(102, { pid: 102, status: "ok", creationFileTime: FT(BASE + 600n), creationTicks: "600" }); + const scan = await verifyWindowsRecordedTreeClaimants([{ pid: 101, fingerprint: SCAN_TICKS(0n) }], scanDeps(tx)); + assert.equal(scan.clean, false); + assert.deepEqual(scan.livePids, [102, 103]); +}); + +test("scan R1: an intermediate whose held exact time contradicts its row interval stays occupied", async () => { + // Ticks match the row but the held exact creation sits far outside the + // row's CIM representation interval: row and handle disagree about the + // generation; nothing under the contradiction is excludable. + const tx = new ScriptedScan([ + srow(101, 4, 0n, "1"), + srow(102, 101, 40n), + srow(103, 102, 100n), + ]); + tx.holdOverrides.set(102, { pid: 102, status: "ok", creationFileTime: FT(BASE + 5_000n), creationTicks: SCAN_TICKS(40n) }); + const scan = await verifyWindowsRecordedTreeClaimants([{ pid: 101, fingerprint: SCAN_TICKS(0n) }], scanDeps(tx)); + assert.equal(scan.clean, false); + assert.deepEqual(scan.livePids, [102, 103]); +}); + +// -- review-round-1 R2: malformed hold evidence can never prove clean --------- + +test("scan R2: a hold response for a different pid refuses the scan", async () => { + const tx = new ScriptedScan([srow(101, 4, 0n)]); + tx.holdResults.set(1, [{ pid: 999, status: "ok", creationFileTime: "300", creationTicks: "300" }]); + await assert.rejects( + verifyWindowsRecordedTreeClaimants([{ pid: 101, fingerprint: SCAN_TICKS(0n) }], scanDeps(tx)), + (error: unknown) => { + assert.ok(error instanceof WindowsTreeCleanupError); + assert.equal(error.code, "process_tree_hold_invalid"); + assert.match(error.message, /unexpected pid 999/); + return true; + }, + ); + assert.equal(tx.closeCount, 1, "the scan helper is closed on refusal"); +}); + +test("scan R2: more hold responses than requested refuse the scan", async () => { + const tx = new ScriptedScan([srow(101, 4, 0n)]); + tx.holdResults.set(1, [ + { pid: 101, status: "ok", creationFileTime: "100", creationTicks: "100" }, + { pid: 101, status: "ok", creationFileTime: "100", creationTicks: "100" }, + ]); + await assert.rejects( + verifyWindowsRecordedTreeClaimants([{ pid: 101, fingerprint: SCAN_TICKS(0n) }], scanDeps(tx)), + /2 responses for 1 requested pid/, + ); +}); + +test("scan R2: an ok hold response without the exact creation filetime refuses the scan", async () => { + const tx = new ScriptedScan([srow(101, 4, 0n), srow(102, 101, 40n)]); + tx.holdResults.set(1, [{ pid: 101, status: "ok", creationTicks: "300" }]); + await assert.rejects( + verifyWindowsRecordedTreeClaimants([{ pid: 101, fingerprint: SCAN_TICKS(0n) }], scanDeps(tx)), + /lacks usable exact identity fields/, + ); + assert.equal(tx.closeCount, 1); +}); + +test("scan R2: an ok hold response with a non-numeric creation time refuses the scan", async () => { + const tx = new ScriptedScan([srow(101, 4, 0n)]); + tx.holdResults.set(1, [{ pid: 101, status: "ok", creationFileTime: "not-a-number", creationTicks: "300" }]); + await assert.rejects( + verifyWindowsRecordedTreeClaimants([{ pid: 101, fingerprint: SCAN_TICKS(0n) }], scanDeps(tx)), + /lacks usable exact identity fields/, + ); +}); + +test("scan P1-B: a child with unknown creation time under a foreign parent stays a claimant", async () => { + const tx = new ScriptedScan([ + srow(101, 4, 0n, "1"), + { pid: 102, parentPid: 101, creationFileTime: null, startTicks: "102-ticks" }, + ]); + const scan = await verifyWindowsRecordedTreeClaimants([{ pid: 101, fingerprint: SCAN_TICKS(0n) }], scanDeps(tx)); + assert.equal(scan.clean, false); + assert.deepEqual(scan.livePids, [102]); +}); + +test("scan: a foreign parent's child with unknown creation time stays a claimant with its whole subtree", async () => { + // 102 under the foreign parent has no CIM: it cannot be excluded, so it is + // a claimant, and its live child 103 propagates through it. + const tx = new ScriptedScan([ + srow(101, 4, 0n, "1"), + { pid: 102, parentPid: 101, creationFileTime: null, startTicks: "102-ticks" }, + srow(103, 102, 40n), + ]); + const scan = await verifyWindowsRecordedTreeClaimants([{ pid: 101, fingerprint: SCAN_TICKS(0n) }], scanDeps(tx)); + assert.equal(scan.clean, false); + assert.deepEqual(scan.livePids, [102, 103]); +}); + +test("scan: a claimant under a foreign parent propagates to its own subtree", async () => { + // Mixed lineage: 102 is an old-generation child (claimant); its live child + // 103 is a claimant through it even though 103 was created after F. + const tx = new ScriptedScan([ + srow(101, 4, 0n, "1"), + srow(102, 101, -20n), + srow(103, 102, 60n), + ]); + const scan = await verifyWindowsRecordedTreeClaimants([{ pid: 101, fingerprint: SCAN_TICKS(0n) }], scanDeps(tx)); + assert.equal(scan.clean, false); + assert.deepEqual(scan.livePids, [102, 103]); +}); + +test("scan: multi-level claimant propagation under a dead recorded pid", async () => { + const tx = new ScriptedScan([srow(102, 101, 40n), srow(103, 102, 80n), srow(104, 103, 120n)]); + const scan = await verifyWindowsRecordedTreeClaimants([{ pid: 101, fingerprint: SCAN_TICKS(0n) }], scanDeps(tx)); + assert.equal(scan.clean, false); + assert.deepEqual(scan.livePids, [102, 103, 104]); +}); + +test("scan: a denied identity read of a live recorded pid stays a claimant and propagates to children", async () => { + const tx = new ScriptedScan([srow(101, 4, 0n), srow(102, 101, 40n)]); + tx.holdOverrides.set(101, { pid: 101, status: "denied", error: 5 }); + const scan = await verifyWindowsRecordedTreeClaimants([{ pid: 101, fingerprint: SCAN_TICKS(0n) }], scanDeps(tx)); + assert.equal(scan.clean, false); + assert.deepEqual(scan.livePids, [101, 102]); +}); + +test("scan: a hold error outcome for a live recorded pid stays a claimant", async () => { + const tx = new ScriptedScan([srow(101, 4, 0n)]); + tx.holdOverrides.set(101, { pid: 101, status: "error", error: 87 }); + const scan = await verifyWindowsRecordedTreeClaimants([{ pid: 101, fingerprint: SCAN_TICKS(0n) }], scanDeps(tx)); + assert.equal(scan.clean, false); + assert.deepEqual(scan.livePids, [101]); +}); + +test("scan: a missing hold outcome fails the scan instead of proving clean", async () => { + const tx = new ScriptedScan([srow(101, 4, 0n)]); + tx.holdResults.set(1, []); + await assert.rejects( + verifyWindowsRecordedTreeClaimants([{ pid: 101, fingerprint: SCAN_TICKS(0n) }], scanDeps(tx)), + (error: unknown) => { + assert.ok(error instanceof WindowsTreeCleanupError); + assert.equal(error.code, "process_tree_hold_invalid"); + assert.match(error.message, /0 responses for 1 requested pid/); + return true; + }, + ); + assert.equal(tx.closeCount, 1, "the scan helper is still closed exactly once"); +}); + +test("scan: an enumerate failure rejects and never reports clean", async () => { + const tx = new ScriptedScan([srow(101, 4, 0n)]); + tx.rows = [srow(101, 4, 0n)]; + const original = tx.enumerate.bind(tx); + tx.enumerate = async () => { + await original(); + throw new Error("process_tree_transport_timeout: enum begin"); + }; + await assert.rejects( + verifyWindowsRecordedTreeClaimants([{ pid: 101, fingerprint: SCAN_TICKS(0n) }], scanDeps(tx)), + /process_tree_transport_timeout/, + ); +}); + +// -- review-round-2 S1: evidence from multiple recorded roots combines -------- + +test("scan S1: a pid foreign to one recorded root but a live child of another dead recorded member stays occupied", async () => { + // Recorded roots 101 (old fp 100) and 102 (no fingerprint). 102 is dead; + // the current 101 (parent 102, generation 300) is proven foreign relative + // to the OLD 101 record, but it is a live child of dead recorded 102 — + // ours-or-unattributable under the contract. Claimant evidence wins. + const tx = new ScriptedScan([ + { pid: 101, parentPid: 102, creationFileTime: "300", startTicks: "300" }, + ]); + const scan = await verifyWindowsRecordedTreeClaimants( + [{ pid: 101, fingerprint: "100" }, { pid: 102, fingerprint: null }], + scanDeps(tx), + ); + assert.equal(scan.clean, false); + assert.deepEqual(scan.livePids, [101]); + assert.ok(scan.observedPids.includes(101), "the claimant is reported as observed evidence"); +}); + +test("scan S1: the same cross-root claim with target order reversed stays occupied", async () => { + const tx = new ScriptedScan([ + { pid: 101, parentPid: 102, creationFileTime: "300", startTicks: "300" }, + ]); + const scan = await verifyWindowsRecordedTreeClaimants( + [{ pid: 102, fingerprint: null }, { pid: 101, fingerprint: "100" }], + scanDeps(tx), + ); + assert.equal(scan.clean, false); + assert.deepEqual(scan.livePids, [101]); +}); + +test("scan S1: the reused pid's descendants stay occupied with it (dead recorded ancestor)", async () => { + const tx = new ScriptedScan([ + { pid: 101, parentPid: 102, creationFileTime: "300", startTicks: "300" }, + { pid: 103, parentPid: 101, creationFileTime: "400", startTicks: "400" }, + ]); + const scan = await verifyWindowsRecordedTreeClaimants( + [{ pid: 101, fingerprint: "100" }, { pid: 102, fingerprint: "200" }], + scanDeps(tx), + ); + assert.equal(scan.clean, false); + assert.deepEqual(scan.livePids, [101, 103]); +}); + +test("scan S1: when every recorded generation is proven foreign the tree releases (positive)", async () => { + const tx = new ScriptedScan([ + { pid: 101, parentPid: 4, creationFileTime: "300", startTicks: "300" }, + { pid: 102, parentPid: 101, creationFileTime: "400", startTicks: "400" }, + { pid: 103, parentPid: 102, creationFileTime: "500", startTicks: "500" }, + ]); + const scan = await verifyWindowsRecordedTreeClaimants( + [{ pid: 101, fingerprint: "100" }, { pid: 102, fingerprint: "200" }], + scanDeps(tx), + ); + assert.equal(scan.clean, true); + assert.deepEqual(scan.livePids, []); +}); + +// -- review-round-2 S2: observation evidence survives the scan exit ---------- + +test("scan S2: a false result reports claimants and requested candidates as observed evidence, proven-foreign excluded", async () => { + // Root 50001 is dead; child 51001 and grandchild 51002 are live claimants. + // The scan never holds them (dead-root propagation needs no hold), yet both + // must appear in observedPids for durable retention; the original target is + // included as well. + const tx = new ScriptedScan([ + { pid: 51001, parentPid: 50001, creationFileTime: "200", startTicks: "200" }, + { pid: 51002, parentPid: 51001, creationFileTime: "300", startTicks: "300" }, + ]); + const scan = await verifyWindowsRecordedTreeClaimants([{ pid: 50001, fingerprint: "100" }], scanDeps(tx)); + assert.equal(scan.clean, false); + assert.deepEqual(scan.livePids, [51001, 51002]); + assert.deepEqual(scan.observedPids, [50001, 51001, 51002]); +}); + +test("scan S2: a rejected scan carries the requested-but-unresolved candidate, not just the roots", async () => { + // The foreign root is held and proven foreign (excluded from evidence); + // the strictly-later child is requested, and its hold returns nothing — + // the protocol failure must carry [root, child] so the manager can retain + // the observation. + const tx = new ScriptedScan([ + { pid: 50001, parentPid: 4, creationFileTime: "300", startTicks: "300" }, + { pid: 51001, parentPid: 50001, creationFileTime: "400", startTicks: "400" }, + ]); + tx.holdOverrides.set(51001, undefined as unknown as HoldOutcome); + tx.hold = async (pids: readonly number[]) => { + tx.holds.push([...pids]); + if (pids[0] === 51001) return []; + return [{ pid: pids[0]!, status: "ok", creationFileTime: "300", creationTicks: "300" }]; + }; + await assert.rejects( + verifyWindowsRecordedTreeClaimants([{ pid: 50001, fingerprint: "100" }], scanDeps(tx)), + (error: unknown) => { + assert.ok(error instanceof WindowsTreeCleanupError); + assert.equal(error.code, "process_tree_hold_invalid"); + assert.deepEqual(error.memberPids, [50001, 51001], "the requested candidate joins the roots in the evidence"); + return true; + }, + ); +}); + +// -- review-round-3 T1: recorded foreign targets obey the same proof --------- + +test("scan T1-A: a recorded child replaced OLDER than its recorded parent's replacement stays occupied", async () => { + // Both 101 and 102 are recorded; both current holders match their rows but + // differ from the recorded fingerprints. Current 101 (300) predates current + // 102 (500): it is an old-generation child of recorded 102 or + // unattributable — being a replacement of old 101 does not erase that. + const tx = new ScriptedScan([ + { pid: 101, parentPid: 102, creationFileTime: "300", startTicks: "300" }, + { pid: 102, parentPid: 4, creationFileTime: "500", startTicks: "500" }, + ]); + const scan = await verifyWindowsRecordedTreeClaimants([{ pid: 101, fingerprint: "100" }, { pid: 102, fingerprint: "200" }], scanDeps(tx)); + assert.equal(scan.clean, false); + assert.deepEqual(scan.livePids, [101]); +}); + +test("scan T1-A: equal-precision replacements across recorded roots stay occupied", async () => { + const tx = new ScriptedScan([ + { pid: 101, parentPid: 102, creationFileTime: "300", startTicks: "300" }, + { pid: 102, parentPid: 4, creationFileTime: "300", startTicks: "300" }, + ]); + const scan = await verifyWindowsRecordedTreeClaimants([{ pid: 101, fingerprint: "100" }, { pid: 102, fingerprint: "200" }], scanDeps(tx)); + assert.equal(scan.clean, false); + assert.deepEqual(scan.livePids, [101]); +}); + +test("scan T1-A: the demoted recorded child propagates claimants to later grandchildren", async () => { + const tx = new ScriptedScan([ + { pid: 103, parentPid: 101, creationFileTime: "700", startTicks: "700" }, + { pid: 102, parentPid: 4, creationFileTime: "500", startTicks: "500" }, + { pid: 101, parentPid: 102, creationFileTime: "300", startTicks: "300" }, + ]); + const scan = await verifyWindowsRecordedTreeClaimants([{ pid: 101, fingerprint: "100" }, { pid: 102, fingerprint: "200" }], scanDeps(tx)); + assert.equal(scan.clean, false); + assert.deepEqual(scan.livePids, [101, 103]); +}); + +test("scan T1-A: target order reversal keeps the same cross-root occupancy", async () => { + const tx = new ScriptedScan([ + { pid: 102, parentPid: 4, creationFileTime: "500", startTicks: "500" }, + { pid: 101, parentPid: 102, creationFileTime: "300", startTicks: "300" }, + ]); + const scan = await verifyWindowsRecordedTreeClaimants([{ pid: 102, fingerprint: "200" }, { pid: 101, fingerprint: "100" }], scanDeps(tx)); + assert.equal(scan.clean, false); + assert.deepEqual(scan.livePids, [101]); +}); + +test("scan T1-A positive: a strictly-later proven edge between replaced recorded pids releases", async () => { + const tx = new ScriptedScan([ + { pid: 101, parentPid: 102, creationFileTime: "600", startTicks: "600" }, + { pid: 102, parentPid: 4, creationFileTime: "500", startTicks: "500" }, + ]); + const scan = await verifyWindowsRecordedTreeClaimants([{ pid: 101, fingerprint: "100" }, { pid: 102, fingerprint: "200" }], scanDeps(tx)); + assert.equal(scan.clean, true); + assert.deepEqual(scan.livePids, []); + assert.deepEqual(tx.holds, [[101], [102]], "both replacements verified once; cached evidence, no repeat holds"); +}); + +test("scan T1-B: a recorded root replaced between row and hold cannot use the row's lineage", async () => { + // Row says generation 100 (parent 4); the held handle reports 300: the row + // describes the OLD process and cannot vouch for the replacement — the + // pid stays occupied and its row-child 400 is not excluded through it. + const tx = new ScriptedScan([ + { pid: 101, parentPid: 4, creationFileTime: "100", startTicks: "100" }, + { pid: 103, parentPid: 101, creationFileTime: "400", startTicks: "400" }, + ]); + tx.holdOverrides.set(101, { pid: 101, status: "ok", creationFileTime: "300", creationTicks: "300" }); + const scan = await verifyWindowsRecordedTreeClaimants([{ pid: 101, fingerprint: "100" }, { pid: 102, fingerprint: "200" }], scanDeps(tx)); + assert.equal(scan.clean, false); + assert.deepEqual(scan.livePids, [101, 103]); +}); + +test("scan T1-B: a mixed-generation row (old CIM, new ticks) cannot vouch for the replacement either", async () => { + const tx = new ScriptedScan([ + { pid: 101, parentPid: 4, creationFileTime: "100", startTicks: "300" }, + { pid: 103, parentPid: 101, creationFileTime: "400", startTicks: "400" }, + ]); + tx.holdOverrides.set(101, { pid: 101, status: "ok", creationFileTime: "300", creationTicks: "300" }); + const scan = await verifyWindowsRecordedTreeClaimants([{ pid: 101, fingerprint: "100" }, { pid: 102, fingerprint: "200" }], scanDeps(tx)); + assert.equal(scan.clean, false); + assert.deepEqual(scan.livePids, [101, 103]); +}); + +test("scan T1-B positive: a row/handle-consistent foreign replacement keeps proving its lineage", async () => { + const tx = new ScriptedScan([ + { pid: 101, parentPid: 4, creationFileTime: "300", startTicks: "300" }, + { pid: 103, parentPid: 101, creationFileTime: "400", startTicks: "400" }, + ]); + tx.holdOverrides.set(101, { pid: 101, status: "ok", creationFileTime: "300", creationTicks: "300" }); + const scan = await verifyWindowsRecordedTreeClaimants([{ pid: 101, fingerprint: "100" }, { pid: 102, fingerprint: "200" }], scanDeps(tx)); + assert.equal(scan.clean, true); + assert.deepEqual(tx.holds, [[101], [103]]); +});