From d618943c6b89c3ab03c8407eb288cad3c65c45ad Mon Sep 17 00:00:00 2001
From: Lex <sunsan05@Gmail.com>
Date: Mon, 17 Aug 2026 14:04:05 +0800
Subject: [PATCH] fix(test): watchdog silent stand-down on missing heartbeat +
 teardown heartbeats
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

The out-of-process watchdog treated a missing heartbeat file as healthy
(!mtime → return forever), so a CI runner reclaiming the tmpdir file left
the guard silently disarmed — the 2026-08-17 teardown hang burned the full
15m step timeout with no kill despite --progress arming the watchdog. Track
missing-since and kill after the timeout instead.

Also heartbeat the teardown phases (disposeApps per-app, cleanup, complete):
the 2026-08-17 hang sat after 'summary pass=229' with zero attribution; the
next hang names the teardown stage it froze in.
---
 .../test/server/httpapi-exercise/backend.ts     | 17 +++++++++++++++--
 .../test/server/httpapi-exercise/index.ts       |  8 +++++++-
 .../test/server/httpapi-exercise/watchdog.ts    | 15 +++++++++++++--
 3 files changed, 35 insertions(+), 5 deletions(-)

diff --git a/packages/opencode/test/server/httpapi-exercise/backend.ts b/packages/opencode/test/server/httpapi-exercise/backend.ts
index 5d13f3d76a..e89b9dd86b 100644
--- a/packages/opencode/test/server/httpapi-exercise/backend.ts
+++ b/packages/opencode/test/server/httpapi-exercise/backend.ts
@@ -57,10 +57,23 @@ type CachedApp = BackendApp & { readonly dispose: () => Promise<void> }
 
 const appCache: Partial<Record<string, CachedApp>> = {}
 
-export async function disposeApps() {
+export async function disposeApps(heartbeat?: (label: string) => void) {
   const apps = Object.values(appCache)
   for (const key of Object.keys(appCache)) delete appCache[key]
-  await Promise.all(apps.flatMap((app) => (app === undefined ? [] : [app.dispose()])))
+  heartbeat?.(`teardown: disposing ${apps.filter((app) => app !== undefined).length} app(s)`)
+  await Promise.all(
+    apps.flatMap((app, i) =>
+      app === undefined
+        ? []
+        : [
+            app.dispose().then(
+              () => heartbeat?.(`teardown: app[${i}] disposed`),
+              (err) => heartbeat?.(`teardown: app[${i}] dispose error: ${err}`),
+            ),
+          ],
+    ),
+  )
+  heartbeat?.("teardown: all apps disposed")
 }
 
 function app(modules: Runtime, options: CallOptions) {
diff --git a/packages/opencode/test/server/httpapi-exercise/index.ts b/packages/opencode/test/server/httpapi-exercise/index.ts
index ad9f3a2080..3e68d25a5a 100644
--- a/packages/opencode/test/server/httpapi-exercise/index.ts
+++ b/packages/opencode/test/server/httpapi-exercise/index.ts
@@ -2151,7 +2151,13 @@ const llmScenarios = new Set([
 ])
 
 const main = Effect.gen(function* () {
-  yield* Effect.addFinalizer(() => Effect.promise(() => disposeApps()).pipe(Effect.andThen(cleanupExercisePaths)))
+  yield* Effect.addFinalizer(() =>
+    Effect.promise(() => disposeApps(options.heartbeat)).pipe(
+      Effect.andThen(Effect.sync(() => options.heartbeat?.("teardown: cleanupExercisePaths"))),
+      Effect.andThen(cleanupExercisePaths),
+      Effect.andThen(Effect.sync(() => options.heartbeat?.("teardown: complete"))),
+    ),
+  )
   const parsed = parseOptions(Bun.argv.slice(2))
   const options: Options = parsed.progress ? { ...parsed, heartbeat: startProgressWatchdog() } : parsed
   const modules = yield* Effect.promise(() => runtime())
diff --git a/packages/opencode/test/server/httpapi-exercise/watchdog.ts b/packages/opencode/test/server/httpapi-exercise/watchdog.ts
index e4c896e4f8..c13cb2f54f 100644
--- a/packages/opencode/test/server/httpapi-exercise/watchdog.ts
+++ b/packages/opencode/test/server/httpapi-exercise/watchdog.ts
@@ -24,6 +24,11 @@ const pid = Number(process.env.WATCHDOG_PID)
 const file = process.env.WATCHDOG_FILE
 const timeoutMs = Number(process.env.WATCHDOG_TIMEOUT_MS)
 const pollMs = Number(process.env.WATCHDOG_POLL_MS)
+// A missing heartbeat file is treated as stalled (tracked via missingSince),
+// never as healthy: CI runners can reclaim tmpdir files, and silently
+// standing down when the file disappears is what let the 2026-08-17
+// teardown hang burn the full 15m step timeout without a kill.
+let missingSince = 0
 setInterval(() => {
   let alive = true
   try {
@@ -36,12 +41,18 @@ setInterval(() => {
   try {
     mtime = fs.statSync(file).mtimeMs
   } catch {}
-  if (!mtime || Date.now() - mtime <= timeoutMs) return
+  if (mtime) {
+    missingSince = 0
+    if (Date.now() - mtime <= timeoutMs) return
+  } else {
+    if (!missingSince) missingSince = Date.now()
+    if (Date.now() - missingSince <= timeoutMs) return
+  }
   let last = "<none>"
   try {
     last = fs.readFileSync(file, "utf8")
   } catch {}
-  console.error("[watchdog] no progress for " + Math.round((Date.now() - mtime) / 1000) + "s; last activity: " + last + " — killing pid " + pid)
+  console.error("[watchdog] no progress for " + Math.round((Date.now() - (mtime || missingSince)) / 1000) + "s; last activity: " + last + " — killing pid " + pid)
   try {
     process.kill(pid, "SIGKILL")
   } catch {}