diff --git a/capsules/assistant/browser/agent-stream.js b/capsules/assistant/browser/agent-stream.js index 9bddc346b..4409fc179 100644 --- a/capsules/assistant/browser/agent-stream.js +++ b/capsules/assistant/browser/agent-stream.js @@ -2073,13 +2073,17 @@ async function startLiveTurnForPrompt(userText, { resumeTurn = null } = {}) { lastScrollTop = scroller.scrollTop; lastMaxTop = scroller.scrollHeight - scroller.clientHeight; }; + /* Layout can clamp a saved pin when the scroll range shrinks. A reader + above that reachable position has moved up; a clamp alone keeps follow. */ + const stayedAtPin = (top) => + scroller.scrollTop >= Math.min(top, scroller.scrollHeight - scroller.clientHeight) - 1; /* Land on the end now and once more after layout settles, unless the reader moved up in between or another turn took the transcript. */ const pinToEnd = () => { scroller.scrollTop = scroller.scrollHeight; const top = scroller.scrollTop; window.requestAnimationFrame(() => { - if (generation === ctx.streamGeneration && scroller.scrollTop >= top - 1) { + if (generation === ctx.streamGeneration && stayedAtPin(top)) { scroller.scrollTop = scroller.scrollHeight; if (endScrollTop === top) { endScrollTop = scroller.scrollTop; @@ -2928,7 +2932,7 @@ async function startLiveTurnForPrompt(userText, { resumeTurn = null } = {}) { /* The last flush and the rich render below make the reply taller; keep a reader who was following at the end. One who scrolled up since the stream ended (its listeners are gone) stays where they are. */ - const pinned = endScrollTop !== null && !!scroller && scroller.scrollTop >= endScrollTop - 1; + const pinned = endScrollTop !== null && !!scroller && stayedAtPin(endScrollTop); /* Streaming painted plain text (live tokens never run Markdown, see adaptQuality). The finished reply, stopped early or not, now gets the render a reopened chat gives it: the whole text through the history diff --git a/scripts/assistant-reply-layout-browser-smoke.mjs b/scripts/assistant-reply-layout-browser-smoke.mjs index 8c369b1e9..105a1b94c 100755 --- a/scripts/assistant-reply-layout-browser-smoke.mjs +++ b/scripts/assistant-reply-layout-browser-smoke.mjs @@ -187,6 +187,8 @@ try { server.listen(0, "127.0.0.1"); await once(server, "listening"); browser = await chromium.launch({ headless: true, ...(process.env.BRAVE_BIN ? { executablePath: process.env.BRAVE_BIN } : {}) }); page = await browser.newPage({ viewport: { width: 1280, height: 800 } }); + const cdp = await page.context().newCDPSession(page); + await cdp.send("Emulation.setCPUThrottlingRate", { rate: 4 }); page.on("pageerror", error => pageErrors.push(`${stage}: ${error}`)); await page.goto(`http://127.0.0.1:${server.address().port}/`); const room = page.frameLocator("iframe").frameLocator("iframe"); @@ -224,7 +226,17 @@ try { seen.status = await expectNewestAboveComposer(".agent-progress", "the status line"); await typeLongFollowUp(); seen.statusTyping = await expectNewestAboveComposer(".agent-progress", "the status line, while a follow-up is typed"); - await input.fill(""); + + // Let layout clamp the scroll position between the stream's final pin + // and its idle Markdown render. A layout clamp is not a reader scrolling up. + await frame.evaluate(() => { + const scheduleIdle = window.requestIdleCallback; + window.requestIdleCallback = (callback, options) => { + window.requestIdleCallback = scheduleIdle; + window.__finishReply = () => scheduleIdle(callback, options); + return 0; + }; + }); stage = "streamed reply"; fixture.events = [ @@ -233,7 +245,24 @@ try { ]; await room.locator(".agent-msg-agent.is-streaming").waitFor({ state: "detached", timeout: 15000 }); await room.locator(".agent-msg-agent").last().filter({ hasText: "Streamed step 25" }).waitFor({ timeout: 5000 }); + await frame.waitForFunction(() => typeof window.__finishReply === "function", null, { timeout: 5000 }); + const beforeShrink = await frame.evaluate(() => document.querySelector("#agent-harness-stream").scrollTop); + await page.setViewportSize({ width: 1280, height: 1600 }); + await input.fill(""); + await frame.waitForFunction((top) => { + const scroller = document.querySelector("#agent-harness-stream"); + return scroller.scrollTop < top - 1 && scroller.scrollHeight - scroller.scrollTop - scroller.clientHeight <= 1; + }, beforeShrink, { timeout: 5000 }); + await frame.evaluate(() => new Promise(done => requestAnimationFrame(() => requestAnimationFrame(done)))); + await frame.evaluate(() => { window.__finishReply(); delete window.__finishReply; }); + await room.locator('.agent-msg-agent[data-stream-phase="presentation_done"]').last().waitFor({ timeout: 5000 }); seen.reply = await expectNewestAboveComposer(".agent-msg-agent", "the newest reply"); + await cdp.send("Emulation.setCPUThrottlingRate", { rate: 1 }); + await page.setViewportSize({ width: 1280, height: 800 }); + await frame.evaluate(() => { + const scroller = document.querySelector("#agent-harness-stream"); + scroller.scrollTop = scroller.scrollHeight; + }); await typeLongFollowUp(); seen.replyTyping = await expectNewestAboveComposer(".agent-msg-agent", "the newest reply, while a follow-up is typed"); diff --git a/scripts/ci-installed-journeys.py b/scripts/ci-installed-journeys.py index 21a4e8d93..f376620d2 100644 --- a/scripts/ci-installed-journeys.py +++ b/scripts/ci-installed-journeys.py @@ -398,6 +398,7 @@ def __init__(self, home, data, address, environment, gib, platform=sys.platform) self.stop = threading.Event() self.error = None self.failure = None + self.census_gap = None self.thread = None self.outside = [os.getpid()] self.engine = None @@ -490,6 +491,87 @@ def events(self): def poll(self): return 1 if self.error or self.pid in self.fds and pidfd_exited(self.fds[self.pid]) else None + def record_census_gap(self, pid, fd): + """Capture the failed sample before cleanup, without changing its verdict.""" + if self.census_gap is not None: + return + observed = {"pid": pid, "sampled_at": time.monotonic()} + directory = Path("/proc") / str(pid) + for name, limit in (("stat", 4096), ("cmdline", 8192), ("cgroup", 4096)): + try: + if name == "stat": + text = bounded_text(directory / name, limit) + fields = text[text.rfind(")") + 2:].split() + observed.update(ppid=int(fields[1]), start_ticks=int(fields[19]), state=fields[0]) + elif name == "cmdline": + text = bounded_text(directory / name, limit).rstrip("\0") + observed[name] = text.split("\0") if text else [] + else: + observed[name] = cgroup_membership(pid) + except (OSError, RuntimeError, ValueError, IndexError) as error: + observed[name + "_error"] = PUBLIC_ERROR_TYPES.get(type(error), "redacted") + retained = self.known.get(pid, {}) + detail = {"retained": retained.get("census_sample", {}), "observed": observed, + "role": retained.get("role"), "exe_sha256": retained.get("exe_sha256"), + "expected_cgroup": self.binding["ControlGroup"]} + try: + pids = [int(value) for value in bounded_text(self.group / "cgroup.procs", 8192).split()] + if len(pids) > 512: + raise RuntimeError("finite journey process census exceeds its bound") + detail["expected_cgroup_procs"] = pids + except (OSError, RuntimeError, ValueError) as error: + detail["expected_cgroup_procs_error"] = PUBLIC_ERROR_TYPES.get(type(error), "redacted") + parent_pid = observed.get("ppid", detail["retained"].get("ppid", retained.get("parent"))) + parent = {"pid": parent_pid} + if parent_pid is not None and parent_pid > 0: + try: + if parent_pid in self.fds: + parent["pidfd_exited"] = pidfd_exited(self.fds[parent_pid]) + else: + parent_fd = os.pidfd_open(parent_pid, 0) + try: + parent["pidfd_exited"] = pidfd_exited(parent_fd) + finally: + os.close(parent_fd) + except (OSError, RuntimeError) as error: + parent["pidfd_error"] = PUBLIC_ERROR_TYPES.get(type(error), "redacted") + try: + parent["cgroup"] = cgroup_membership(parent_pid) + except (OSError, RuntimeError, ValueError) as error: + parent["cgroup_error"] = PUBLIC_ERROR_TYPES.get(type(error), "redacted") + detail["parent"] = parent + try: + detail["pidfd_exited"] = pidfd_exited(fd) + except (OSError, RuntimeError) as error: + detail["pidfd_error"] = PUBLIC_ERROR_TYPES.get(type(error), "redacted") + detail["same_start_ticks"] = (observed["start_ticks"] == retained["start_ticks"] + if "start_ticks" in observed and "start_ticks" in retained else None) + # Raw argv and non-unit cgroup names stay with the private Runtime log. + # CI uploads JSON and PNG only; this file deliberately uses .private.log. + try: + private_json(self.control / "census-gap.private.log", detail) + detail["private_capture"] = "saved" + except (OSError, RuntimeError) as error: + detail["private_capture"] = PUBLIC_ERROR_TYPES.get(type(error), "redacted") + for key in ("retained", "observed", "parent"): + sample = dict(detail[key]) + if "cmdline" in sample: + # Only the bound executable's name and fixed harness verbs leave + # the runner. Arbitrary positional values can contain credentials. + executable = Path(retained.get("executable", "")).name + sample["cmdline"] = [executable if index == 0 and Path(arg).name == executable else + arg if arg in ("gateway", "init", "daemon", "--internal-local-llama-guard") + else "" for index, arg in enumerate(sample["cmdline"])] + if "cgroup" in sample and sample["cgroup"] != self.binding["ControlGroup"]: + sample["cgroup_sha256"] = hashlib.sha256(sample["cgroup"].encode()).hexdigest() + sample["cgroup"] = "/" + "/".join( + part if re.fullmatch(r"[A-Za-z0-9_.@-]{1,64}\.(slice|scope|service|mount|socket)", part) + else "" for part in sample["cgroup"].split("/")[1:]) + if key == "retained": + sample.update(role=retained.get("role"), exe_sha256=retained.get("exe_sha256")) + detail[key] = sample + self.census_gap = detail + def observe_once(self, require_runtime=True): if (self.metric("memory.max") != str(self.budget) or self.metric("memory.swap.max") != "0" or self.metric("pids.max") != "512"): @@ -585,9 +667,16 @@ def observe_once(self, require_runtime=True): "model_provider" if executable == (self.data / "bin/model-provider").resolve() else "llama_server" if executable.name == "llama-server" else "other") self.known[pid] = identity + self.known[pid]["census_sample"] = { + "pid": pid, "ppid": identity["parent"], "start_ticks": identity["start_ticks"], + "state": identity["state"], "cmdline": command, "cgroup": membership, + "sampled_at": time.monotonic(), + } current[pid] = self.known[pid] - if any(pid not in current and not pidfd_exited(fd) for pid, fd in self.fds.items()): - raise RuntimeError("finite journey live child left its unit census") + for pid, fd in self.fds.items(): + if pid not in current and not pidfd_exited(fd): + self.record_census_gap(pid, fd) + raise RuntimeError("finite journey live child left its unit census") if require_runtime and self.pid not in current: raise RuntimeError("finite journey Runtime left its unit") memory = [self.metric(name) for name in ("memory.current", "memory.peak")] @@ -708,6 +797,8 @@ def finish(self): "actions": self.actions, "outside_runtime_unit": True} if self.failure: record["observer_failure"] = self.failure + if self.census_gap: + record["census_gap"] = self.census_gap if runtime_exited: record["runtime_exited_before_cleanup"] = True phase = "unit_binding" diff --git a/scripts/ci-release-policy-test.py b/scripts/ci-release-policy-test.py index 63c6d96fa..cc9d7432d 100644 --- a/scripts/ci-release-policy-test.py +++ b/scripts/ci-release-policy-test.py @@ -2722,6 +2722,7 @@ def finite_owner(self, root): owner.stop = mock.Mock() owner.stop.is_set.return_value = False owner.thread, owner.error, owner.failure, owner.launch_requested = None, None, None, True + owner.census_gap = None owner.baseline_events = {"oom": 0, "oom_kill": 0} owner.show = mock.Mock(return_value=dict(owner.binding)) owner.command = mock.Mock() @@ -2939,6 +2940,187 @@ def test_finite_census_accepts_same_birth_exec_and_never_rebinds_an_exited_pidfd self.assertNotIn(103, owner.observe_once()) self.assertEqual(opened.call_count, len(rows)) + def test_finite_census_gap_names_child_without_changing_failure(self): + for observation in ("present", "gone", "reused", "denied"): + with self.subTest(observation=observation), tempfile.TemporaryDirectory() as temp: + owner, _ = self.finite_owner(Path(temp)) + rows = self.process_rows(owner) + with self.observations(owner, rows), mock.patch.object(os, "getuid", return_value=1001), \ + mock.patch.object(os, "pidfd_open", side_effect=lambda pid, flags: pid + 1000, create=True): + owner.observe_once() + rows.pop(103) + reader = self.globals["bounded_text"] + def read(path, limit): + if Path(path).parent.name != "103": + return reader(path, limit) + if observation in ("gone", "denied"): + raise (FileNotFoundError if observation == "gone" else PermissionError)("private detail") + if Path(path).name == "stat": + fields = ["Z", "102", "103"] + ["0"] * 17 + fields[19] = "999" if observation == "reused" else "803" + return "103 (private name) " + " ".join(fields) + if Path(path).name == "cmdline": + return "/private/operator/llama-server\0--token\0bare-secret\0init\0" + self.fail("unexpected diagnostic read") + self.globals["bounded_text"] = read + with self.assertRaisesRegex(RuntimeError, "live child left its unit census"): + owner.observe_once() + detail = owner.census_gap + self.assertEqual(detail["retained"]["pid"], 103) + self.assertEqual(detail["retained"]["ppid"], 102) + self.assertEqual(detail["retained"]["start_ticks"], 803) + self.assertEqual(detail["retained"]["role"], "llama_server") + self.assertEqual(detail["retained"]["cgroup"], owner.binding["ControlGroup"]) + self.assertFalse(detail["pidfd_exited"]) + self.assertEqual(detail["expected_cgroup"], owner.binding["ControlGroup"]) + if observation in ("present", "reused"): + self.assertEqual(detail["observed"]["ppid"], 102) + self.assertEqual(detail["observed"]["state"], "Z") + self.assertEqual(detail["same_start_ticks"], observation == "present") + self.assertEqual(detail["observed"]["cmdline"], + ["llama-server", "", "", "init"]) + else: + self.assertEqual(detail["observed"]["stat_error"], + "FileNotFoundError" if observation == "gone" else "PermissionError") + encoded = json.dumps(detail) + for secret in ("bare-secret", "--token", "/private/operator", "private name", "private detail"): + self.assertNotIn(secret, encoded) + private = owner.control / "census-gap.private.log" + self.assertEqual(private.stat().st_mode & 0o777, 0o600) + self.assertEqual(json.loads(private.read_text())["same_start_ticks"], + detail["same_start_ticks"]) + with mock.patch.dict(self.globals, {"private_json": mock.Mock()}) as globals_: + owner.record_census_gap(103, 1103) + globals_["private_json"].assert_not_called() + self.assertIs(owner.census_gap, detail) + owner.launch_requested = False + with mock.patch.object(os, "close"): + self.assertEqual(owner.finish()["census_gap"], detail) + + def test_finite_census_diagnostics_keep_exit_and_capture_errors_separate(self): + for fault in ("exited", "pidfd", "private_write"): + with self.subTest(fault=fault), tempfile.TemporaryDirectory() as temp: + owner, _ = self.finite_owner(Path(temp)) + rows = self.process_rows(owner) + with self.observations(owner, rows), mock.patch.object(os, "getuid", return_value=1001), \ + mock.patch.object(os, "pidfd_open", side_effect=lambda pid, flags: pid + 1000, create=True): + owner.observe_once() + rows.pop(103) + polls = iter((False, True)) + def exited(fd): + if fd != 1103: + return False + value = next(polls) + if value and fault == "pidfd": + raise RuntimeError("private pidfd detail") + return value + self.globals["pidfd_exited"] = exited + self.globals["cgroup_membership"] = lambda pid: ("/foreign/operator/session.key" if pid == 103 else + "/outside" if pid in owner.outside else owner.binding["ControlGroup"]) + # Force bounded stat/argv reads to fail; cgroup still reports + # the foreign path independently of those observations. + reader = self.globals["bounded_text"] + self.globals["bounded_text"] = lambda path, limit: ( + "" if Path(path).parent.name == "103" else reader(path, limit)) + writer = self.globals["private_json"] + if fault == "private_write": + self.globals["private_json"] = mock.Mock(side_effect=PermissionError("private write detail")) + with self.assertRaisesRegex(RuntimeError, "live child left its unit census"): + owner.observe_once() + detail = owner.census_gap + self.assertNotIn("operator", json.dumps(detail)) + self.assertNotIn("private", json.dumps(detail).replace("private_capture", "")) + self.assertEqual(detail["observed"]["cgroup"], "///") + self.assertEqual(len(detail["observed"]["cgroup_sha256"]), 64) + self.assertEqual(detail["observed"]["cmdline"], []) + self.assertEqual(detail.get("pidfd_error"), "RuntimeError" if fault == "pidfd" else None) + self.assertEqual(detail.get("pidfd_exited"), None if fault == "pidfd" else True) + self.assertEqual(detail["private_capture"], "PermissionError" if fault == "private_write" else "saved") + self.globals["private_json"] = writer + + def test_finite_census_gap_keeps_unit_names_in_public_receipt(self): + paths = { + "/init.scope": "/init.scope", + "/user.slice/user-1001.slice/user@1001.service/init.scope": + "/user.slice/user-1001.slice/user@1001.service/init.scope", + "/system.slice/cron.service": "/system.slice/cron.service", + "/private-operator/system.slice/cron.service": "//system.slice/cron.service", + "/a.mount/a.socket/" + "b" * 64 + ".scope/" + "c" * 65 + ".service": + "/a.mount/a.socket/" + "b" * 64 + ".scope/", + "/bad!.service/cron.service.extra": "//", + } + for path, public in paths.items(): + with self.subTest(path=path), tempfile.TemporaryDirectory() as temp: + owner, _ = self.finite_owner(Path(temp)) + rows = self.process_rows(owner) + with self.observations(owner, rows), mock.patch.object(os, "getuid", return_value=1001), \ + mock.patch.object(os, "pidfd_open", side_effect=lambda pid, flags: pid + 1000, create=True): + owner.observe_once() + rows.pop(103) + membership = self.globals["cgroup_membership"] + self.globals["cgroup_membership"] = lambda pid: path if pid in (102, 103) else membership(pid) + owner.record_census_gap(103, 1103) + # Use the same JSON serialization as the uploaded receipt. + detail = json.loads(json.dumps(owner.census_gap)) + for sample in ("observed", "parent"): + self.assertEqual(detail[sample]["cgroup"], public) + self.assertEqual(detail[sample]["cgroup_sha256"], hashlib.sha256(path.encode()).hexdigest()) + self.assertNotIn("private-operator", json.dumps(detail)) + + def test_finite_census_gap_rereads_unit_and_parent_despite_missing_retained_keys(self): + for fault in ("none", "known", "sample", "parent_exited", "parent_gone", "parent_denied", "unit_gone"): + with self.subTest(fault=fault), tempfile.TemporaryDirectory() as temp: + owner, _ = self.finite_owner(Path(temp)) + rows = self.process_rows(owner) + with self.observations(owner, rows), mock.patch.object(os, "getuid", return_value=1001), \ + mock.patch.object(os, "pidfd_open", side_effect=lambda pid, flags: pid + 1000, create=True): + owner.observe_once() + rows.pop(103) + if fault == "known": + del owner.known[103] + elif fault == "sample": + del owner.known[103]["census_sample"] + reader = self.globals["bounded_text"] + reads = [] + def read(path, limit): + if Path(path).name == "cgroup.procs": + reads.append(path) + if fault == "unit_gone" and len(reads) == 2: + raise FileNotFoundError("private unit detail") + return "99 100 101 102" if len(reads) == 1 else "99 100 101 102 103" + if Path(path) == Path("/proc/103/stat"): + fields = ["S", "102", "103"] + ["0"] * 17 + fields[19] = "803" + return "103 (private child) " + " ".join(fields) + return reader(path, limit) + def membership(pid): + if pid == 102 and len(reads) == 2 and fault in ("parent_gone", "parent_denied"): + raise (FileNotFoundError if fault == "parent_gone" else PermissionError)("private parent") + return "/outside" if pid == 900 else owner.binding["ControlGroup"] + def exited(fd): + if len(reads) != 2: + return False + if fd == 1102 and fault in ("parent_gone", "parent_denied"): + raise (ProcessLookupError if fault == "parent_gone" else PermissionError)("private parent") + return fd == 1102 and fault == "parent_exited" + self.globals.update(bounded_text=read, cgroup_membership=membership, pidfd_exited=exited) + with self.assertRaisesRegex(RuntimeError, "live child left its unit census"): + owner.observe_once() + detail = owner.census_gap + self.assertEqual(len(reads), 2) + self.assertEqual(detail["parent"]["pid"], 102) + if fault in ("parent_gone", "parent_denied"): + self.assertIn("pidfd_error", detail["parent"]) + self.assertIn("cgroup_error", detail["parent"]) + else: + self.assertEqual(detail["parent"]["pidfd_exited"], fault == "parent_exited") + self.assertEqual(detail["parent"]["cgroup"], owner.binding["ControlGroup"]) + if fault == "unit_gone": + self.assertEqual(detail["expected_cgroup_procs_error"], "FileNotFoundError") + else: + self.assertEqual(detail["expected_cgroup_procs"], [99, 100, 101, 102, 103]) + self.assertNotIn("private parent", json.dumps(detail)) + def test_finite_pidfd_binding_closes_on_identity_race_or_unreadable_child(self): for fault in ("reused", "disappeared", "denied"): with self.subTest(fault=fault), tempfile.TemporaryDirectory() as temp: