fix(ci): remove the shared root causes of flaky Windows CI

scripts/ci/run-bun-test-batches.sh

@@ -326,6 +326,18 @@ fallback_ms="$(
 )"
 readonly FALLBACK_MS="$fallback_ms"
 
+# One private Bun transpiler cache for every batch in this shard. Left unset, each batch's
+# preload creates an empty cache of its own, so the first child a batch spawns re-transpiles
+# its whole module graph from nothing (0.67s cold against 0.20s warm for the journal/CLI graph
+# on macOS). On Windows runners that cold load is what pushed first spawns to 15-40s and past
+# their deadlines. mktemp -d creates a fresh directory only this job can write, which is the
+# protected explicit cache createIsolatedTestEnvironment accepts; an operator's value is kept.
+if [[ -z "${BUN_RUNTIME_TRANSPILER_CACHE_PATH+x}" ]]; then
+  shard_transpiler_cache="$(mktemp -d -t ocx-bun-transpiler-cache.XXXXXX)"
+  trap 'rm -rf -- "$shard_transpiler_cache"' EXIT
+  export BUN_RUNTIME_TRANSPILER_CACHE_PATH="$shard_transpiler_cache"
+fi
+
 assignment_file="$(mktemp -t ocx-bun-test-shards.XXXXXX)"
 printf '%s\n' "${GENERAL_FILES[@]}" \
   | awk -F '\t' -v table="$durations_source" -v fallback="$FALLBACK_MS" '

scripts/test-layout/layout.json

@@ -1525,6 +1525,7 @@
     "responses-spill-inspection.test.ts": "responses",
     "responses-spill-orphan-sweep.test.ts": "responses",
     "responses-spill-shutdown-clock.test.ts": "responses",
+    "responses-state-flush-race.test.ts": "responses",
     "responses-state-snapshot-select.test.ts": "responses",
     "responses-state-snapshot-stubs.test.ts": "responses", "responses-state-store-false.test.ts": "responses",
     "responses-state-write-amplification.test.ts": "responses",
@@ -1849,6 +1850,7 @@
     "windows-process-priority.test.ts": "windows",
     "windows-scheduler-install-verification.test.ts": "windows",
     "windows-secret-acl.test.ts": "windows", "windows-secret-acl-removal-flight.test.ts": "windows",
+    "windows-config-read-acl-memo.test.ts": "windows",
     "windows-service-mutation-lock.test.ts": "windows",
     "windows-service-wrappers.test.ts": "windows",
     "windows-text-decoding.test.ts": "windows",

scripts/test-temp.ts

@@ -210,6 +210,35 @@ export function removeTestTempTree(path: string, options: RemoveTreeOptions = {}
   }
 }
 
+/**
+ * The same policy, but the waits yield to the event loop.
+ *
+ * The synchronous loop is only correct when the handle holder is another process. When the
+ * holder is async work in THIS process -- an un-awaited quota prime holding a claim database
+ * across an icacls child or a fetch -- `Bun.sleepSync` stops the loop that work needs to reach
+ * its `finally`, so every retry fails until the budget is gone. Awaiting lets the owner finish.
+ */
+export async function removeTestTempTreeAsync(
+  path: string,
+  options: Omit<RemoveTreeOptions, "sleep"> & { sleep?: (milliseconds: number) => Promise<void> } = {},
+): Promise<void> {
+  const delays = options.delays ?? removeRetrySchedule(options.budgetMs);
+  const remove = options.remove ?? (target => rmSync(target, { recursive: true, force: true }));
+  const sleep = options.sleep ?? (milliseconds => Bun.sleep(milliseconds));
+  for (let attempt = 0; attempt <= delays.length; attempt += 1) {
+    try {
+      remove(path);
+      return;
+    } catch (error) {
+      const transient = TRANSIENT_REMOVE_CODES.has(errorCode(error));
+      const finalAttempt = !transient || attempt === delays.length;
+      if (finalAttempt && quarantineFinalEperm(path, error, options.lockDiagnostic, options.lockEnv)) return;
+      if (finalAttempt) throw error;
+      await sleep(delays[attempt]!);
+    }
+  }
+}
+
 /** Stamp a newly created root so future runs can prove its OpenCodex test ownership. */
 export function writeTestTempOwner(root: string, runId?: string): void {
   const owner: TestTempOwner = {

src/cli/auxiliary-listener-preflight.ts

@@ -0,0 +1,37 @@
+import type { OcxConfig } from "../types";
+import { AuxiliaryListenerBindError } from "../server/ports";
+import { createServer } from "node:net";
+
+/**
+ * Reject impossible auxiliary binds before startup creates private state. On Windows
+ * that publication requires several synchronous ACL subprocesses. This is only an
+ * early rejection: startServer still owns the actual bind and transactional rollback
+ * if another process takes the port between this probe and startup.
+ */
+export async function assertAuxiliaryListenerPortsAvailable(config: OcxConfig): Promise<void> {
+  const loopback = config.unauthenticatedLoopbackListener;
+  const management = config.runtimeRole === "hub" ? config.hub?.managementIngress : undefined;
+  const targets = [
+    { listener: "unauthenticatedLoopbackListener", port: loopback?.enabled ? loopback.port : undefined },
+    { listener: "hub.managementIngress", port: management?.enabled ? management.port : undefined },
+  ] as const;
+  // Each probe releases its socket, so two listeners configured for one port would both
+  // pass and startup would still fail on the second bind. Reject the collision directly.
+  if (targets[0].port !== undefined && targets[0].port === targets[1].port) {
+    throw new AuxiliaryListenerBindError(
+      "hub.managementIngress", targets[1].port, "127.0.0.1",
+      new Error("port is also configured for unauthenticatedLoopbackListener"),
+    );
+  }
+  for (const { listener, port } of targets) {
+    // The port-less companion shares the public bind; it has no independent target.
+    if (port === undefined) continue;
+    await new Promise<void>((resolve, reject) => {
+      const probe = createServer(socket => socket.destroy());
+      const fail = (cause: unknown) => reject(new AuxiliaryListenerBindError(listener, port, "127.0.0.1", cause));
+      probe.once("error", fail);
+      probe.once("listening", () => probe.close(error => error ? fail(error) : resolve()));
+      try { probe.listen({ port, host: "127.0.0.1" }); } catch (error) { fail(error); }
+    });
+  }
+}

src/cli/index.ts

@@ -283,6 +283,7 @@ async function chooseListenPort(
       `Port ${preferred} is reserved for unauthenticatedLoopbackListener; choose a different proxy port.`,
     );
   }
+  await (await import("./auxiliary-listener-preflight")).assertAuxiliaryListenerPortsAvailable(config);
   // Soft start: brief prefer-retry then ephemeral hop.
   // Explicit `--port` (service wrappers / update restart): wait for the pinned port
   // to free without killing any listener (healthy ocx / foreign). Never hop.

src/codex/auth-api/pool-mode-gate.ts

@@ -25,6 +25,7 @@ import type { PoolQuotaResult } from "./pool-quota-probe";
 import { fetchMainAccountInfoAttempt, fetchMainAccountInfo, MAIN_CACHE_TTL } from "./main-account-probe";
 import { fetchPoolAccountQuota, PoolQuotaProbeBusyError, POOL_CACHE_TTL, POOL_QUOTA_REFRESH_CONCURRENCY } from "./pool-quota-probe";
 import { getRuntimeConfig, configuredPoolAccount, mapWithConcurrency } from "./runtime-config";
+import { trackBackgroundWork } from "../../lib/background-work";
 
 let primeInFlight: Promise<void> | null = null;
 let lastPriorityFailbackPrimeAt: number | undefined;
@@ -259,7 +260,9 @@ export async function primeCodexPoolQuotas(
       && now - lastPriorityFailbackPrimeAt < CODEX_PRIORITY_FAILBACK_REFRESH_MS) return;
     lastPriorityFailbackPrimeAt = now;
   }
-  primeInFlight = (async () => {
+  // Tracked because most callers do not await it, yet it holds the native-main claim
+  // database open; whoever removes that directory settles it first (see background-work).
+  primeInFlight = trackBackgroundWork(`codex-pool-quota-prime:${reason}`, (async () => {
     const pool = (runtimeConfig.codexAccounts ?? []).filter(isSelectableCodexPoolAccount);
     const stale = pool.filter(a => {
       const q = getAccountQuota(a.id);
@@ -344,7 +347,7 @@ export async function primeCodexPoolQuotas(
     if (process.env.OPENCODEX_DEBUG_QUOTA === "1") {
       console.warn(`[codex-quota] prime done (reason=${reason}, pool=${pool.length}, refreshed=${stale.length})`);
     }
-  })().finally(() => { primeInFlight = null; });
+  })().finally(() => { primeInFlight = null; }));
   return primeInFlight;
 }
 

src/codex/auth-context.ts

@@ -51,6 +51,7 @@ import {
 // The half-open TRANSIENT-HOLD lease (#4701). Not the quota-cooldown probe lease imported from
 // ./routing above -- different module, different domain, and a request never holds both.
 import { releaseTransientProbe } from "../routing/probe-lease";
+import { trackBackgroundWork } from "../lib/background-work";
 import {
   codexConversationIdentity,
   recordCodexThreadLineage,
@@ -1547,9 +1548,10 @@ async function resolveCodexAuthContextUnannotated(
     if (options.primeCodexPoolQuotas) {
       void options.primeCodexPoolQuotas(config, reason).catch(() => {});
     } else {
-      import("./auth-api")
+      // Tracked from the import on: the prime registers itself only once the module resolves.
+      void trackBackgroundWork(`codex-pool-quota-prime-import:${reason}`, import("./auth-api")
         .then(({ primeCodexPoolQuotas }) => primeCodexPoolQuotas(config, reason))
-        .catch(() => {});
+        .catch(() => {}));
     }
   }
   // Snapshot (not just the deadline) so a refused request can report WHY it is cooled:

src/codex/desktop-compatibility/windows-key-protection.ts

@@ -4,6 +4,10 @@ import { resolveTrustedWindowsPowerShellExe } from "../../lib/windows-elevation"
 const MAX_INPUT = 65_536;
 const MAX_OUTPUT = 131_072;
 const PURPOSE = "opencodex/codex-desktop-compatibility/authority/v1";
+// A cold Windows PowerShell that must also load System.Security has been measured past 10s on a
+// loaded machine (Windows CI run 38591626343). The call is async and off the request path, so a
+// wider bound costs nothing while healthy and turns a slow start into success instead of a refusal.
+const DPAPI_TIMEOUT_MS = 20_000;
 const SCRIPT = [
   "$ErrorActionPreference='Stop'",
   "Add-Type -AssemblyName System.Security",
@@ -31,7 +35,7 @@ function runDpapi(operation: "protect" | "unprotect", data: Uint8Array): Promise
   if (data.byteLength === 0 || data.byteLength > MAX_INPUT) return Promise.reject(new Error("desktop_compatibility_key_size"));
   return new Promise((resolve, reject) => {
     const child = execFile(resolveTrustedWindowsPowerShellExe(), ["-NoProfile", "-NonInteractive", "-Command", SCRIPT], {
-      timeout: 10_000, maxBuffer: MAX_OUTPUT, windowsHide: true, encoding: "utf8",
+      timeout: DPAPI_TIMEOUT_MS, maxBuffer: MAX_OUTPUT, windowsHide: true, encoding: "utf8",
     }, (error, stdout) => {
       // Do not include PowerShell stderr, command output or the input in a propagated error.
       if (error || !/^[A-Za-z0-9+/]+={0,2}$/.test(stdout.trim())) {

src/config/load-degrade.ts

@@ -52,9 +52,12 @@ import { hasWarnedInheritedFastWireConflict, markWarnedInheritedFastWireConflict
 
 export function hardenExistingSecret(path: string): void {
   if (existsSync(path) && !isPassiveRead()) {
-    try { chmodSync(path, 0o600); } catch { /* best-effort */ }
     if (process.platform === "win32") {
+      // Preserve the ACL helper's metadata observation across reads. Windows chmod does
+      // not establish a private DACL and can invalidate that observation before every load.
       hardenSecretPath(path, { required: false });
+    } else {
+      try { chmodSync(path, 0o600); } catch { /* best-effort */ }
     }
   }
 }

src/config/paths.ts

@@ -4,6 +4,7 @@ import { join, resolve } from "node:path";
 import { flushWindowsSecretAclReapsBeforeRemoval, hardenSecretDirAsync, windowsSecretAclApplies } from "../lib/windows-secret-acl";
 import { assertNotRealHomeUnderTest } from "../lib/test-home-guard";
 import { isPassiveRead } from "../lib/passive-read";
+import { trackBackgroundWork } from "../lib/background-work";
 
 /**
  * Expand a leading `~` in user-supplied paths without interpreting shell
@@ -37,17 +38,23 @@ export function hardenConfigDir(): void {
   // have changed the protected directory used by the test-home boundary.
   assertNotRealHomeUnderTest(dir);
   if (!existsSync(dir)) return;
-  try { chmodSync(dir, 0o700); } catch { /* best-effort */ }
+  // Windows privacy is enforced by NTFS ACLs. chmod only changes DOS attributes there;
+  // doing it before the ACL lookup can invalidate its ctime-based successful observation.
+  if (!windowsSecretAclApplies()) {
+    try { chmodSync(dir, 0o700); } catch { /* best-effort */ }
+  }
   if (windowsSecretAclApplies() && !configDirHardeningFlights.has(dir)) {
     // This is an optional read-path harden. Waiting synchronously here used to stop the Bun
     // event loop (including /healthz) for the full icacls timeout. Required mutation paths keep
     // their own awaited/fail-closed hardening; ordinary config reads only start one soft flight.
-    const flight = hardenSecretDirAsync(dir, { required: false })
+    // Tracked: icacls holds the directory open, and a config write or removal that lands on
+    // it before the child exits fails with a sharing violation on Windows.
+    const flight = trackBackgroundWork("config-dir-acl-harden", hardenSecretDirAsync(dir, { required: false })
       .then(() => undefined)
       .catch(() => undefined)
       .finally(() => {
         if (configDirHardeningFlights.get(dir) === flight) configDirHardeningFlights.delete(dir);
-      });
+      }));
     configDirHardeningFlights.set(dir, flight);
   }
 }

src/lib/background-work.ts

@@ -0,0 +1,65 @@
+/**
+ * Fire-and-forget work the process started and nobody awaits.
+ *
+ * A lazy quota prime or a startup reconcile is deliberately left running behind a
+ * request so the request never waits on it. The work still owns files: the pool
+ * prime holds the native-main claim database open across ACL hardening and a WHAM
+ * read. Whoever later removes the directory those files live in has to wait for the
+ * owner. Without a handle on the work there was nothing to wait for: test teardown
+ * spun a synchronous retry loop, which blocks the very event loop the owner needs to
+ * reach its `finally` and close the file. On Windows every overlap became a 15 s EBUSY.
+ *
+ * Registering costs one Set entry per in-flight task and changes nothing about how
+ * the work runs. Settling is opt-in: test teardown calls it; requests never do.
+ */
+
+interface BackgroundWork {
+  readonly label: string;
+  readonly settled: Promise<void>;
+}
+
+const inFlight = new Set<BackgroundWork>();
+
+/** Record `work` until it settles and return it unchanged. Rejections stay the caller's. */
+export function trackBackgroundWork<T>(label: string, work: Promise<T>): Promise<T> {
+  const entry: BackgroundWork = {
+    label,
+    settled: work.then(() => { inFlight.delete(entry); }, () => { inFlight.delete(entry); }),
+  };
+  inFlight.add(entry);
+  return work;
+}
+
+/** Labels of work still running, for diagnostics. */
+export function pendingBackgroundWork(): string[] {
+  return [...inFlight].map(entry => entry.label);
+}
+
+export interface BackgroundWorkSettlement {
+  readonly settled: boolean;
+  readonly pending: string[];
+}
+
+/**
+ * Wait until no tracked work is in flight, including work started by work that was
+ * already running. A bounded wait reports what is still running instead of throwing,
+ * so the caller decides whether that is fatal.
+ */
+export async function settleBackgroundWork(options: { timeoutMs?: number } = {}): Promise<BackgroundWorkSettlement> {
+  const deadline = options.timeoutMs === undefined ? Infinity : Date.now() + options.timeoutMs;
+  while (inFlight.size > 0) {
+    const remaining = deadline - Date.now();
+    if (remaining <= 0) break;
+    const batch = Promise.all([...inFlight].map(entry => entry.settled)).then(() => false);
+    if (remaining === Infinity) {
+      await batch;
+      continue;
+    }
+    let timer: ReturnType<typeof setTimeout> | undefined;
+    const expired = new Promise<boolean>(resolve => { timer = setTimeout(() => resolve(true), remaining); });
+    const timedOut = await Promise.race([batch, expired]);
+    clearTimeout(timer);
+    if (timedOut) break;
+  }
+  return { settled: inFlight.size === 0, pending: pendingBackgroundWork() };
+}

src/responses/state.ts

@@ -710,11 +710,6 @@ export function sweepOrphanedResponseSpills(): number {
 type SnapshotWriteOutcome = "stable" | "unstable" | "failed";
 
 async function writeBoundedSnapshot(path: string, attemptLimit: number): Promise<SnapshotWriteOutcome> {
-  // Serialize writers so concurrent flush + debounce cannot race on temps / ACL (#612).
-  const previous = persistGate;
-  let release!: () => void;
-  persistGate = new Promise<void>(resolve => { release = resolve; });
-  await previous;
   try {
     for (let attempt = 0; attempt < attemptLimit; attempt += 1) {
       const revision = stateRevision;
@@ -751,13 +746,19 @@ async function writeBoundedSnapshot(path: string, attemptLimit: number): Promise
         lastSnapshotTarget = resolveWriteTarget(path);
       }
       persistAttemptHookForTests?.();
-      if (revision === stateRevision) return "stable";
+      if (revision === stateRevision) {
+        // Cancel only the timer whose mutation this stable revision includes,
+        // before an awaiting caller's microtask can record a newer revision.
+        if (persistTimer) clearTimeout(persistTimer);
+        persistTimer = null;
+        pendingPersistPath = null;
+        drainPendingSpillUnlinks();
+        return "stable";
+      }
     }
     return "unstable";
   } catch {
     return "failed";
-  } finally {
-    release();
   }
 }
 
@@ -792,21 +793,30 @@ function schedulePersistAt(path: string, replace = false): void {
 }
 
 async function persistNow(path: string, awaitFollowUp = false): Promise<void> {
-  if (persistTimer) {
-    clearTimeout(persistTimer);
-    persistTimer = null;
-  }
-  pendingPersistPath = null;
-  const attemptLimit = awaitFollowUp ? MAX_SNAPSHOT_REWRITE_ATTEMPTS : 1;
-  let outcome = await writeBoundedSnapshot(path, attemptLimit);
-  if (outcome === "unstable" && awaitFollowUp) {
-    if (persistTimer) clearTimeout(persistTimer);
-    persistTimer = null;
+  // Include follow-up scheduling in the gate: releasing after only the disk write
+  // lets shutdown observe an empty timer before the background pass re-arms it.
+  const previous = persistGate;
+  let release!: () => void;
+  persistGate = new Promise<void>(resolve => { release = resolve; });
+  await previous;
+  try {
+    if (persistTimer) {
+      clearTimeout(persistTimer);
+      persistTimer = null;
+    }
     pendingPersistPath = null;
-    outcome = await writeBoundedSnapshot(path, attemptLimit);
+    const attemptLimit = awaitFollowUp ? MAX_SNAPSHOT_REWRITE_ATTEMPTS : 1;
+    let outcome = await writeBoundedSnapshot(path, attemptLimit);
+    if (outcome === "unstable" && awaitFollowUp) {
+      if (persistTimer) clearTimeout(persistTimer);
+      persistTimer = null;
+      pendingPersistPath = null;
+      outcome = await writeBoundedSnapshot(path, attemptLimit);
+    }
+    if (outcome === "unstable" && !awaitFollowUp) schedulePersistAt(path, true);
+  } finally {
+    release();
   }
-  if (outcome === "stable") drainPendingSpillUnlinks();
-  else if (outcome === "unstable" && !awaitFollowUp) schedulePersistAt(path, true);
 }
 
 function schedulePersist(): void {
@@ -816,15 +826,8 @@ function schedulePersist(): void {
 }
 
 async function flushResponseSnapshot(): Promise<void> {
-  if (persistTimer) {
-    await persistNow(pendingPersistPath ?? snapshotPath(), true);
-    return;
-  }
-  // No pending timer: still await any in-flight write so shutdown does not race (#612).
+  // The gate includes the background pass's timer decision, not just its write.
   await persistGate;
-  // A bounded background pass may have scheduled its same-path follow-up while
-  // this flush was waiting on the single-flight gate. Shutdown owns one awaited
-  // bounded follow-up rather than returning behind that unref'd timer.
   if (persistTimer) await persistNow(pendingPersistPath ?? snapshotPath(), true);
 }