diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index aabf35d..36e1b72 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -126,8 +126,11 @@ jobs: - name: Build and run pipe_probe (cef_host against a stub plugin) # Reuses the cef_host build tree the example build just configured. # pipe_probe drives the host over the pipe: handshake, create, first - # frame, a navigation, shutdown. The second run turns the GPU off and - # checks the frames came through OnPaint, the software path. + # frame, a navigation, shutdown; then a host whose UI thread it + # suspends before kOpShutdown, which has to exit on its own. It serves + # its pages on 127.0.0.1, so the network can't slow it down. The + # second run turns the GPU off and checks the frames came through + # OnPaint, the software path. shell: pwsh timeout-minutes: 10 run: | @@ -149,11 +152,13 @@ jobs: } - name: Runtime smoke test (example app, real cef_host) # Runs the app against the cef_host it just built: first frame, eval, a - # JS channel, resize, freeze/thaw, and a crash loop on one of two tiles - # sharing a host. Once as the runner composites, once with the GPU off + # JS channel, resize, freeze/thaw, then a crash loop and a hung + # renderer on one of several tiles sharing a host, and a replaced GPU + # process. Once as the runner composites, once with the GPU off # (software paint). The plugin's and hosts' log lines go to a file - # (FLUTTER_CEF_LOG_FILE); the crash-loop lines are printed, and the - # whole log when a mode fails. See example/lib/windows_smoke_probe.dart. + # (FLUTTER_CEF_LOG_FILE); the crash-loop and liveness lines are + # printed, and the whole log when a mode fails. See + # example/lib/windows_smoke_probe.dart. shell: pwsh timeout-minutes: 20 run: | @@ -185,8 +190,8 @@ jobs: Write-Output "== plugin and host log, $mode compositing" Get-Content $log } else { - Write-Output "== host crash-loop lines, $mode compositing" - Select-String -Path $log -Pattern "renderer terminated|giving up|several browsers|browser gone" | ForEach-Object { $_.Line } + Write-Output "== host crash-loop and liveness lines, $mode compositing" + Select-String -Path $log -Pattern "renderer terminated|giving up|several browsers|browser gone|liveness ping" | ForEach-Object { $_.Line } } } if ($modeFailed) { $failed = $true } diff --git a/CHANGELOG.md b/CHANGELOG.md index a4b15f3..c4a194c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,12 @@ ## Unreleased +* **Windows: a hung renderer ends only its tile**, as on macOS: a renderer + that leaves the liveness ping unanswered for 15 s gets + `processGone('crashed')` for its own tile, and the other tiles on its host + carry on. The renderer answers the ping, not the page, so a page that breaks + `window.cefQuery` or `JSON` isn't taken for hung. +* **Windows `cef_host` exits on its own when stuck at shutdown**: 6 s after a + shutdown request, or 30 s once `CefShutdown` is running, as on macOS. * **Windows logs to a file on request**: set `FLUTTER_CEF_LOG_FILE` to a path and the plugin appends its own and every `cef_host`'s log lines there (they otherwise go only to `OutputDebugString`). diff --git a/example/lib/windows_smoke_probe.dart b/example/lib/windows_smoke_probe.dart index 42de905..ab7ccb5 100644 --- a/example/lib/windows_smoke_probe.dart +++ b/example/lib/windows_smoke_probe.dart @@ -17,7 +17,13 @@ // * dispose; // * a tile whose renderer keeps crashing ends alone: two tiles share a host, // one is sent to chrome://kill until the host gives up on it, and it gets -// processGone('crashed') while the other keeps painting and answering. +// processGone('crashed') while the other keeps painting and answering; +// * a tile whose renderer hangs ends alone, too: its page loops forever, the +// liveness ping goes unanswered, and it gets processGone('crashed'). A +// page on the same host that only breaks evals is left alone, because the +// ping is answered by the renderer, not the page; +// * a GPU process that Chromium replaces (chrome://gpucrash) leaves the +// host's tiles painting. // // The result is a `CEF_PROBE_RESULT PASS|FAIL` line on stdout and, because a // Windows GUI app's stdout isn't reliably captured, also in the file named by @@ -27,6 +33,7 @@ // set FLUTTER_CEF_PROBE_OUT=%TEMP%\smoke.txt // build\windows\x64\runner\Debug\flutter_cef_example.exe import 'dart:async'; +import 'dart:convert'; import 'dart:io'; import 'package:flutter/material.dart'; @@ -43,10 +50,17 @@ const _html = '''
'''; -// The crash-loop case's two tiles are served at different sites, so site -// isolation gives each its own renderer process and killing one can't take -// the other with it. +// A page with nothing to animate: it presents only when something changes. +const _static = + '' + '

static

'; + +// The crash-loop and hang cases' tiles are served at different sites, so site +// isolation gives each its own renderer process, and killing or hanging one +// can't take the others with it. const _sentinelUrl = 'https://flutter-cef-sentinel.test/'; +const _hangUrl = 'https://flutter-cef-hang.test/'; +const _noEvalsUrl = 'https://flutter-cef-no-evals.test/'; void main() => runApp(const MaterialApp(home: ProbeApp())); @@ -209,6 +223,8 @@ class _ProbeAppState extends State { } await c.dispose(); await _crashLoops(); + await _hungRenderer(); + await _gpuProcessReplaced(); _finish(); } @@ -340,6 +356,216 @@ class _ProbeAppState extends State { await victim.dispose(); } + /// A renderer that stops answering ends only its own tile, and a page that + /// merely can't answer evals is left alone. + Future _hungRenderer() async { + const tag = 'hung renderer'; + const group = 'windows-smoke-hang'; + final sentinel = CefWebController(hostGroup: group); + final hung = CefWebController(hostGroup: group); + final noEvals = CefWebController(hostGroup: group); + final clock = Stopwatch()..start(); + void event(String what) => + _log(' $tag +${clock.elapsedMilliseconds}ms $what'); + final loaded = {}; + String? sentinelGone; + String? noEvalsGone; + final hungGone = Completer(); + sentinel.onProcessGone = (r) => sentinelGone ??= r; + noEvals.onProcessGone = (r) { + event('evals-breaking tile processGone($r)'); + noEvalsGone ??= r; + }; + hung.onProcessGone = (r) { + event('hung tile processGone($r)'); + if (!hungGone.isCompleted) hungGone.complete(r); + }; + for (final c in [sentinel, hung, noEvals]) { + c.onPageFinished = (_) => loaded.add(c); + } + try { + for (final (c, url, html) in [ + (sentinel, _sentinelUrl, _html), + (hung, _hangUrl, _static), + (noEvals, _noEvalsUrl, _static), + ]) { + await c.create( + url: url, + html: html, + htmlBaseUrl: url, + width: 320, + height: 240, + ); + } + _check( + '$tag: the tiles load and paint', + await _waitFor(() => loaded.length == 3, const Duration(seconds: 60)) && + await _presentsPast(hung, 0) && + await _presentsPast(noEvals, 0), + ); + + // Every eval's reply goes through JSON.stringify in the page, so once + // that throws the page can't answer one. Broken after this eval has + // answered. + final one = await noEvals + .runJavaScriptReturningResult( + '(setTimeout(() => { JSON.stringify = () => { throw 0; }; }, 0), 1)', + ) + .timeout(const Duration(seconds: 10)); + _check( + '$tag: an eval answers before the page breaks them', + '$one' == '1', + one, + ); + final broken = Stopwatch()..start(); + // This page never returns to its event loop, so its renderer's main + // thread can't answer the ping. + await hung.executeJavaScript('setTimeout(() => { for (;;) {} }, 0)'); + event('page hung'); + + // The sweep pings a tile that has shown no new frame for 10 s, and a + // renderer that leaves the ping unanswered for 15 s is hung. + final gone = await hungGone.future.timeout( + const Duration(seconds: 60), + onTimeout: () => '', + ); + _check( + '$tag: the hung tile gets processGone(crashed)', + gone == 'crashed', + gone, + ); + // By 45 s the sweep has pinged the evals-breaking page too. Had the page + // been asked rather than its renderer, it would be gone by now. + final rest = const Duration(seconds: 45) - broken.elapsed; + if (rest > Duration.zero) await Future.delayed(rest); + _check( + '$tag: a page that can\'t answer evals is left alone', + noEvalsGone == null, + noEvalsGone, + ); + + final before = await _presents(sentinel); + _check( + '$tag: the other tiles keep painting', + await _presentsPast( + sentinel, + before, + within: const Duration(seconds: 10), + ), + await sentinel.sessionStats(), + ); + final four = await sentinel + .runJavaScriptReturningResult('2 + 2') + .timeout(const Duration(seconds: 10)); + _check('$tag: the other tiles answer evals', '$four' == '4', four); + _check( + '$tag: the other tiles get no processGone', + sentinelGone == null, + sentinelGone, + ); + } catch (e, st) { + _check('$tag: ran to completion', false, '$e\n$st'); + } + await sentinel.dispose(); + await hung.dispose(); + await noEvals.dispose(); + } + + /// A GPU process that Chromium replaces leaves the host's tiles painting, + /// so Windows, unlike macOS, doesn't end a host for it. + Future _gpuProcessReplaced() async { + const tag = 'gpu process replaced'; + const group = 'windows-smoke-gpu'; + final a = CefWebController(hostGroup: group); + final b = CefWebController(hostGroup: group); + String? gone; + a.onProcessGone = (r) => gone ??= 'a: $r'; + b.onProcessGone = (r) => gone ??= 'b: $r'; + try { + await a.create( + url: _sentinelUrl, + html: _html, + htmlBaseUrl: _sentinelUrl, + width: 320, + height: 240, + ); + await b.create( + url: 'about:blank', + html: _static, + width: 320, + height: 240, + ); + _check( + '$tag: the tiles paint', + await _presentsPast(a, 0, within: const Duration(seconds: 60)) && + await _presentsPast(b, 0, within: const Duration(seconds: 60)), + ); + final before = await _gpuProcesses(); + // chrome://gpucrash is a debug URL: Chromium crashes the GPU process + // and starts a new one. + await b.navigate('chrome://gpucrash'); + // Replaced: one of the GPU processes from before is gone, and a new one + // is running. + bool replaced(Set after) => + !after.containsAll(before) && !before.containsAll(after); + var after = before; + final sw = Stopwatch()..start(); + while (sw.elapsed < const Duration(seconds: 15) && !replaced(after)) { + await Future.delayed(const Duration(milliseconds: 500)); + after = await _gpuProcesses(); + } + _check( + '$tag: chrome://gpucrash replaces the GPU process', + replaced(after), + 'before $before, after $after', + ); + final presented = await _presents(a); + await Future.delayed(const Duration(seconds: 2)); + final since = await _presents(a) - presented; + _check( + '$tag: the tiles keep painting', + since > 30, + '$since presents in 2 s', + ); + final c = CefWebController(hostGroup: group); + await c.create(url: 'about:blank', html: _html, width: 320, height: 240); + _check( + '$tag: a new tile on the host paints', + await _presentsPast(c, 0, within: const Duration(seconds: 30)), + await c.sessionStats(), + ); + await c.dispose(); + _check('$tag: no processGone', gone == null, gone); + } catch (e, st) { + _check('$tag: ran to completion', false, '$e\n$st'); + } + await a.dispose(); + await b.dispose(); + } + + /// The pids of the running GPU processes of every cef_host. + Future> _gpuProcesses() async { + const script = + "Get-CimInstance Win32_Process -Filter \"Name='cef_host.exe'\" | " + "Where-Object { \$_.CommandLine -like '*--type=gpu-process*' } | " + "ForEach-Object { \$_.ProcessId }"; + // -EncodedCommand takes UTF-16LE, base64: no quoting to get wrong. + final utf16 = []; + for (final u in script.codeUnits) { + utf16 + ..add(u & 0xff) + ..add(u >> 8); + } + final r = await Process.run('powershell', [ + '-NoProfile', + '-EncodedCommand', + base64Encode(utf16), + ]); + return { + for (final w in '${r.stdout}'.split(RegExp(r'\s+'))) ?int.tryParse(w), + }; + } + void _finish() { final result = 'CEF_PROBE_RESULT ${_pass ? "PASS" : "FAIL"}'; // ignore: avoid_print diff --git a/packages/flutter_cef_windows/CHANGELOG.md b/packages/flutter_cef_windows/CHANGELOG.md index 6ffc035..151f06a 100644 --- a/packages/flutter_cef_windows/CHANGELOG.md +++ b/packages/flutter_cef_windows/CHANGELOG.md @@ -1,5 +1,23 @@ ## Unreleased +* A renderer that leaves the liveness ping unanswered ends only its own tile + (`processGone("crashed")`); it used to end the whole host. The ping is + answered by the renderer's main thread (a process message), not the page, so + a page that breaks `window.cefQuery` or `JSON.stringify` isn't taken for + hung, and a page can't answer in the ping's place. As on macOS. +* `cef_host` ends itself when it is still running 6 s after a shutdown + request, or 30 s after its message loop quit (a stuck `CefShutdown`), + logging `still running after shutdown (); exiting now` to stderr. As on + macOS. The watchdog's thread starts with the host, so a wedged thread + holding the loader lock can't keep it from starting. +* A replaced GPU process doesn't end the host, unlike on macOS: Windows tiles + keep painting after Chromium relaunches it. +* CI: `pipe_probe` serves its pages on 127.0.0.1 instead of example.com, + prints each host's stdout and stderr, and checks that a host whose UI + thread is suspended at `kOpShutdown` exits on its own about 6 s later. The + smoke test hangs one tile's renderer on a shared host (only that tile goes; + a page that breaks evals beside it stays) and replaces a host's GPU process + (its tiles keep painting). * `FLUTTER_CEF_LOG_FILE=` appends the plugin's and every `cef_host`'s log lines to that file, each with a millisecond tick, as well as sending them to `OutputDebugString`. diff --git a/packages/flutter_cef_windows/README.md b/packages/flutter_cef_windows/README.md index 90f9f67..822b7ec 100644 --- a/packages/flutter_cef_windows/README.md +++ b/packages/flutter_cef_windows/README.md @@ -34,10 +34,18 @@ renderer that crashes 4 times within 10 s ends only its own tile tiles crash-loop within 10 s of each other, the host's child processes can't start, so the host exits and every tile on it gets `processGone('crashed')`. -The plugin reports `paintStalled` for a tile that never paints. Liveness -differs from macOS in two ways: a renderer that leaves a JS ping unanswered -for 15 s (`FLUTTER_CEF_HANG_MS`) ends the whole host rather than just its -tile, and a replaced GPU process isn't detected. +The plugin reports `paintStalled` for a tile that never paints. A tile whose +renderer hangs is handled as on macOS: a tile that has shown no new frame for +10 s is pinged, and a renderer that leaves the ping unanswered for 15 s +(`FLUTTER_CEF_HANG_MS`) ends only its own tile (`processGone('crashed')`). The +renderer answers the ping, not the page, so a page that breaks +`window.cefQuery` or `JSON` isn't taken for hung. One difference: macOS ends a +host whose GPU process Chromium had to replace, because its tiles never paint +again. On Windows they keep painting, so the host carries on. + +A `cef_host` that is still running 6 s after it was asked to shut down (30 s +once its message loop has quit and `CefShutdown` is running) ends itself, as +on macOS, and writes `still running after shutdown` to its stderr. ## Sandbox diff --git a/packages/flutter_cef_windows/native/cef_host/PROTOCOL.md b/packages/flutter_cef_windows/native/cef_host/PROTOCOL.md index 0cf68ff..1dfe3f6 100644 --- a/packages/flutter_cef_windows/native/cef_host/PROTOCOL.md +++ b/packages/flutter_cef_windows/native/cef_host/PROTOCOL.md @@ -264,11 +264,22 @@ thread (marshal from the reader thread). - Steady-state liveness: every 2 s the plugin checks each painted, visible tile. One with no frame for 10 s (`FLUTTER_CEF_LIVENESS_MS`) gets a 0x37 repaint and a ping: `kOpEvalReturning` with id `0xFFFFFFFF` (Dart's ids never - reach it; the reply is consumed, not forwarded). An idle page answers; a - renderer that leaves the ping unanswered for 15 s (`FLUTTER_CEF_HANG_MS`) - ends its host. Tiles blocked on a JS dialog, with DevTools opened, or on an - agent-control host are not pinged. macOS also detects a replaced GPU - process; Windows doesn't. + reach it; the reply is consumed, not forwarded). The host doesn't evaluate + it in the page: it sends the renderer a process message, which the + renderer's main thread answers, and replies `0xFFFFFFFF:{"ok":true,"v":1}` + (a page's own `eval:` reply under that id is refused). A renderer that + leaves the ping unanswered for 15 s (`FLUTTER_CEF_HANG_MS`) is hung: the + plugin emits `processGone("crashed")` for that session alone and disposes + it, and the host's other browsers carry on. Tiles blocked on a JS dialog, + with DevTools opened, or on an agent-control host are not pinged. macOS + also ends a host whose GPU process was replaced; on Windows the tiles keep + painting after one is, so it isn't watched. +- Shutdown: `kOpShutdown`, the pipe closing and a crash-loop host exit all + arm a watchdog in the host, which ends the process (exit code 0) if it is + still running 6 s later, or 30 s after its message loop has quit, logging + `[cef_host] still running after shutdown (); exiting now` to stderr. + A wedged UI thread never runs the shutdown, and a host left running keeps + its profile locked. The timings are the macOS host's. - Renderer crash loops are counted per browser, as on macOS. A crash reloads the page; 4 crashes of one browser within 10 s end only that browser (kOpBrowserGone "crashed", no more reloads): the plugin emits diff --git a/packages/flutter_cef_windows/native/cef_host/cef_host_policy.h b/packages/flutter_cef_windows/native/cef_host/cef_host_policy.h index 420776f..c972b3f 100644 --- a/packages/flutter_cef_windows/native/cef_host/cef_host_policy.h +++ b/packages/flutter_cef_windows/native/cef_host/cef_host_policy.h @@ -89,6 +89,25 @@ class RendererCrashPolicy { std::map bursts_; // by wire id }; +// ---- Hard exit after shutdown ----------------------------------------------- +// +// A host whose UI thread is wedged never runs the shutdown it was sent, so it +// lingers and keeps its profile's lock. A shutdown request therefore also +// starts a watchdog that ends the process kHardExitAfterShutdown later. Once +// the message loop has quit, CefShutdown is running, which takes a second or +// so and longer on a first launch, so the deadline moves out to +// kHardExitAfterTeardown from then. The same timings as macOS. +constexpr std::chrono::seconds kHardExitAfterShutdown{6}; +constexpr std::chrono::seconds kHardExitAfterTeardown{30}; + +// The watchdog's wait: sleeps until `deadline()` has passed, reading it again +// after every sleep, so a deadline moved out meanwhile is kept. `now()` and +// `deadline()` are milliseconds on one clock; `sleep_ms(ms)` sleeps. +template +void WaitForDeadline(NowFn now, DeadlineFn deadline, SleepFn sleep_ms) { + for (int64_t left; (left = deadline() - now()) > 0;) sleep_ms(left); +} + // ---- Page-sourced payloads -------------------------------------------------- // // The plugin treats a frame body over 64 MiB as a desynced stream and drops diff --git a/packages/flutter_cef_windows/native/cef_host/cef_host_win.cc b/packages/flutter_cef_windows/native/cef_host/cef_host_win.cc index c49bb44..ffe5d73 100644 --- a/packages/flutter_cef_windows/native/cef_host/cef_host_win.cc +++ b/packages/flutter_cef_windows/native/cef_host/cef_host_win.cc @@ -138,6 +138,15 @@ std::set g_allowed_schemes; // translation (OnBeforeCommandLineProcessing in main.mm). std::string g_cdp_io_pipes; +// The plugin's liveness ping: kOpEvalReturning under this id, which Dart never +// issues. DoEvalReturning asks the renderer itself (kPingMessage, answered +// from the renderer's main thread with kPongMessage), not the page, so a hung +// renderer doesn't answer and a page can't stop a live one from answering (by +// replacing window.cefQuery or JSON, say). As on macOS. +constexpr uint32_t kLivenessPingId = 0xFFFFFFFFu; +constexpr char kPingMessage[] = "flutter_cef.ping"; +constexpr char kPongMessage[] = "flutter_cef.pong"; + // JS channels: on each MAIN-frame load OnLoadStart injects a // window..postMessage shim for each of the browser's Slot::channels, // routed to the browser process over window.cefQuery (the CefMessageRouter @@ -1329,6 +1338,14 @@ class HostClient : public CefClient, callback->Failure(403, "subframe"); return true; } + // Only the renderer answers the liveness ping (OnProcessMessageReceived + // below), never the page. + static const std::string kPingReply = + "eval:" + std::to_string(kLivenessPingId) + ":"; + if (r.rfind(kPingReply, 0) == 0) { + callback->Failure(403, "reserved eval id"); + return true; + } SendUtf8(slot_->browser_id, kOpEvalResult, policy::CapEvalResult(r.substr(5))); callback->Success(CefString()); @@ -1367,6 +1384,14 @@ class HostClient : public CefClient, CefRefPtr frame, CefProcessId source_process, CefRefPtr message) override { + // The renderer answered the liveness ping (see DoEvalReturning); reply as + // the plugin's ping eval would have. + if (message->GetName().ToString() == kPongMessage) { + if (frame && frame->IsMain()) + SendUtf8(slot_->browser_id, kOpEvalResult, + std::to_string(kLivenessPingId) + ":{\"ok\":true,\"v\":1}"); + return true; + } return router_->OnProcessMessageReceived(browser, frame, source_process, message); } @@ -1716,6 +1741,14 @@ class HostApp : public CefApp, CefRefPtr frame, CefProcessId source_process, CefRefPtr message) override { + // The liveness ping: answered from this (the renderer's main) thread, + // which a hung page or renderer never gets back to. + if (message->GetName().ToString() == kPingMessage) { + if (frame) + frame->SendProcessMessage(PID_BROWSER, + CefProcessMessage::Create(kPongMessage)); + return true; + } return render_router_ && render_router_->OnProcessMessageReceived(browser, frame, source_process, message); @@ -1978,6 +2011,12 @@ void DoJsDialogResp(const std::shared_ptr& slot, uint32_t id, bool ok, void DoEvalReturning(Slot& slot, uint32_t id, const std::string& code) { CefRefPtr frame = slot.browser->GetMainFrame(); if (!frame) return; + if (id == kLivenessPingId) { + // Asked of the renderer itself, not the page. + frame->SendProcessMessage(PID_RENDERER, + CefProcessMessage::Create(kPingMessage)); + return; + } std::string js = "window.cefQuery({request:'eval:" + std::to_string(id) + ":'+(function(){try{return JSON.stringify({ok:true,v:(" + code + @@ -2224,6 +2263,59 @@ void FinishShutdownQuit() { CefQuitMessageLoop(); } +// Ends the process if it is still running a while after a shutdown was +// requested (policy::kHardExitAfterShutdown; see cef_host_policy.h). Armed +// once, wherever a shutdown is requested; any thread. As the macOS host's +// ArmHardExit, except that its thread starts with the process +// (StartHardExitWatchdog): starting a thread takes the loader lock, which a +// wedged thread may hold. +std::once_flag g_hard_exit_armed; +HANDLE g_hard_exit_event = nullptr; // set when armed +std::atomic g_hard_exit_at_ms{0}; // GetTickCount64 clock +std::atomic g_hard_exit_why{""}; + +int64_t TickMs() { return static_cast(GetTickCount64()); } + +void StartHardExitWatchdog() { + g_hard_exit_event = CreateEventW(nullptr, TRUE, FALSE, nullptr); + if (!g_hard_exit_event) { + LogErr("[cef_host] CreateEvent failed (%lu): no hard exit after shutdown", + GetLastError()); + return; + } + std::thread([] { + WaitForSingleObject(g_hard_exit_event, INFINITE); + policy::WaitForDeadline( + TickMs, [] { return g_hard_exit_at_ms.load(); }, + [](int64_t ms) { Sleep(static_cast(ms)); }); + LogErr("[cef_host] still running after shutdown (%s); exiting now", + g_hard_exit_why.load()); + // Not exit(): that runs DLL detach code, which can block on a lock the + // wedged thread holds. + TerminateProcess(GetCurrentProcess(), 0); + }).detach(); +} + +// `why` must be a string literal. +void ArmHardExit(const char* why) { + std::call_once(g_hard_exit_armed, [why] { + g_hard_exit_why = why; + g_hard_exit_at_ms = + TickMs() + std::chrono::milliseconds(policy::kHardExitAfterShutdown) + .count(); + if (g_hard_exit_event) SetEvent(g_hard_exit_event); + }); +} + +// The message loop has quit and CefShutdown is next: allow it +// policy::kHardExitAfterTeardown from now. +void ExtendHardExitForTeardown() { + ArmHardExit("teardown"); + g_hard_exit_at_ms = + TickMs() + + std::chrono::milliseconds(policy::kHardExitAfterTeardown).count(); +} + // Cookie-store flush completion: once the on-disk jar is written, quit. class ShutdownFlushCallback : public CefCompletionCallback { public: @@ -2249,6 +2341,7 @@ class ShutdownFlushCallback : public CefCompletionCallback { // grace. The macOS host does not need this. void DoShutdown() { CEF_REQUIRE_UI_THREAD(); + ArmHardExit("shutdown"); std::vector> slots; { std::lock_guard lock(g_slots_mutex); @@ -2343,6 +2436,7 @@ void IpcReadLoop() { SetDocumentStart(wire_id, document_start::ParsePayload(p, plen)); break; case kOpShutdown: + ArmHardExit("kOpShutdown"); CefPostTask(TID_UI, base::BindOnce(&DoShutdown)); return; case kOpResize: { @@ -2600,6 +2694,7 @@ void IpcReadLoop() { } // Plugin died / pipe closed: quit. (Orphan-kill backstop is the plugin's // Job Object — JOB_OBJECT_LIMIT_KILL_ON_JOB_CLOSE — not a parent watch.) + ArmHardExit("IPC closed"); CefPostTask(TID_UI, base::BindOnce(&DoShutdown)); } @@ -2639,6 +2734,7 @@ extern "C" CEF_BOOTSTRAP_EXPORT int RunConsoleMain( if (code >= 0) return code; // ---- Browser process from here on. ---- + StartHardExitWatchdog(); const std::vector args = Utf8Args(); std::string ipc_name = GetSwitch(args, "--ipc="); std::string profile_dir = GetSwitch(args, "--profile-dir="); @@ -2796,6 +2892,7 @@ extern "C" CEF_BOOTSTRAP_EXPORT int RunConsoleMain( } CefRunMessageLoop(); + ExtendHardExitForTeardown(); // Teardown: invalidate the pipe FIRST (atomic exchange), THEN close — so a // late SendFrame from a CEF thread can't write into a recycled handle diff --git a/packages/flutter_cef_windows/native/cef_host/test/pipe_probe.cc b/packages/flutter_cef_windows/native/cef_host/test/pipe_probe.cc index f28fc8b..b553d71 100644 --- a/packages/flutter_cef_windows/native/cef_host/test/pipe_probe.cc +++ b/packages/flutter_cef_windows/native/cef_host/test/pipe_probe.cc @@ -6,23 +6,37 @@ // kOpCreated -> first kOpPresent within 20s -> kOpNavigate to a second URL -> // url/title/loadState events -> kOpShutdown -> clean host exit. // -// No CEF dependency — just Win32 + cef_host_protocol.h. Build: +// Then a wedged host: a second cef_host loads a page, the probe suspends its +// UI thread and sends kOpShutdown, which that thread never runs. The host's +// hard-exit watchdog has to end it about 6 s later, exit code 0, with its +// "still running after shutdown" line on stderr. +// +// The pages come from a server the probe runs on 127.0.0.1, so a slow network +// can't slow a load or a shutdown down. Each host's stdout and stderr are +// printed after it exits. +// +// No CEF dependency — just Win32, Winsock and cef_host_protocol.h. Build: // cmake --build build --target pipe_probe // Run (hostDir must contain cef_host.exe + cef_host.dll + the CEF runtime): // pipe_probe.exe [url1] [url2] // Exits 0 + prints "PIPE_PROBE PASS" on success; nonzero otherwise. +#include #include +#include #include #include #include +#include #include #include #include #include "../cef_host_protocol.h" +#pragma comment(lib, "ws2_32.lib") + using namespace flutter_cef; namespace { @@ -30,6 +44,12 @@ namespace { HANDLE g_pipe = INVALID_HANDLE_VALUE; ULONGLONG g_t0 = 0; +// The running host's stdout and stderr, collected by a reader thread; printed +// by Fail and after each host. +std::mutex g_host_out_mutex; +std::string g_host_out; +HANDLE g_host_out_done = nullptr; // set when the host's output pipe closes + double Now() { return (GetTickCount64() - g_t0) / 1000.0; } void Say(const char* fmt, ...) { @@ -42,11 +62,106 @@ void Say(const char* fmt, ...) { fflush(stdout); } +// Everything the host has written to stdout and stderr so far. +std::string HostOutput() { + std::lock_guard lock(g_host_out_mutex); + return g_host_out; +} + +void PrintHostOutput() { + const std::string out = HostOutput(); + Say("host stdout+stderr (%zu bytes)%s", out.size(), out.empty() ? "" : ":"); + size_t start = 0; + while (start < out.size()) { + size_t end = out.find('\n', start); + if (end == std::string::npos) end = out.size(); + std::string line = out.substr(start, end - start); + if (!line.empty() && line.back() == '\r') line.pop_back(); + printf(" [host] %s\n", line.c_str()); + start = end + 1; + } + fflush(stdout); +} + [[noreturn]] void Fail(const char* why) { + PrintHostOutput(); Say("PIPE_PROBE FAIL: %s", why); ExitProcess(1); } +// ---- The probe's pages, served on 127.0.0.1 ---- + +std::string PageBody(const std::string& path) { + if (path == "/one") + return "pipe_probe one" + "page one"; + if (path == "/two") + return "pipe_probe two" + "page two"; + return std::string(); +} + +// One connection: read the request head, answer it, close. Its own thread, so +// a connection Chromium opens ahead of time and never uses can't hold up the +// next. +void ServeConnection(SOCKET c) { + std::string req; + char buf[4096]; + while (req.find("\r\n\r\n") == std::string::npos && req.size() < 65536) { + const int n = recv(c, buf, sizeof(buf), 0); + if (n <= 0) break; + req.append(buf, static_cast(n)); + } + // "GET /path HTTP/1.1" + std::string path; + const size_t sp1 = req.find(' '); + const size_t sp2 = + sp1 == std::string::npos ? std::string::npos : req.find(' ', sp1 + 1); + if (sp2 != std::string::npos) path = req.substr(sp1 + 1, sp2 - sp1 - 1); + std::string body = PageBody(path); + const char* status = "200 OK"; + if (body.empty()) { + status = "404 Not Found"; + body = "not found"; + } + const std::string resp = + std::string("HTTP/1.1 ") + status + + "\r\nContent-Type: text/html; charset=utf-8\r\nContent-Length: " + + std::to_string(body.size()) + + "\r\nCache-Control: no-store\r\nConnection: close\r\n\r\n" + body; + send(c, resp.data(), static_cast(resp.size()), 0); + shutdown(c, SD_SEND); + closesocket(c); +} + +// Starts the page server on an ephemeral loopback port; returns the port. +int StartPageServer() { + WSADATA wsa; + if (WSAStartup(MAKEWORD(2, 2), &wsa) != 0) Fail("WSAStartup failed"); + SOCKET listener = socket(AF_INET, SOCK_STREAM, IPPROTO_TCP); + if (listener == INVALID_SOCKET) Fail("socket failed"); + sockaddr_in addr = {}; + addr.sin_family = AF_INET; + addr.sin_addr.s_addr = htonl(INADDR_LOOPBACK); + addr.sin_port = 0; + if (bind(listener, reinterpret_cast(&addr), sizeof(addr)) != 0 || + listen(listener, SOMAXCONN) != 0) + Fail("bind/listen on 127.0.0.1 failed"); + int len = sizeof(addr); + if (getsockname(listener, reinterpret_cast(&addr), &len) != 0) + Fail("getsockname failed"); + std::thread([listener] { + for (;;) { + SOCKET c = accept(listener, nullptr, nullptr); + if (c == INVALID_SOCKET) return; + std::thread(ServeConnection, c).detach(); + } + }).detach(); + return ntohs(addr.sin_port); +} + +// ---- The pipe ---- + bool ReadAllPipe(void* buf, size_t len) { uint8_t* p = static_cast(buf); size_t off = 0; @@ -121,30 +236,19 @@ const char* OpName(uint8_t op) { } } -} // namespace +// ---- A host ---- -int main(int argc, char** argv) { - g_t0 = GetTickCount64(); - if (argc < 2) { - fprintf(stderr, - "usage: pipe_probe.exe [url1] [url2]\n"); - return 2; - } - std::string host_dir = argv[1]; - std::string url1 = argc > 2 ? argv[2] : "https://example.com/"; - std::string url2 = - argc > 3 ? argv[3] : "https://www.iana.org/help/example-domains"; - - // Global watchdog: nothing in this probe may take 3 minutes. - std::thread([] { - Sleep(180000); - printf("PIPE_PROBE FAIL: global watchdog (180s)\n"); - ExitProcess(3); - }).detach(); +struct Host { + HANDLE process = nullptr; + HANDLE main_thread = nullptr; // runs CefRunMessageLoop: the UI thread +}; - // 1. Pipe server (the plugin's role): byte-stream, single instance. - std::string pipe_name = - "\\\\.\\pipe\\flutter_cef_probe_" + std::to_string(GetCurrentProcessId()); +// Creates this run's pipe server, spawns cef_host.exe against it with its +// stdout and stderr on a pipe the probe reads, and waits for it to connect. +Host SpawnHost(const std::string& host_dir, const char* tag) { + const std::string suffix = + std::to_string(GetCurrentProcessId()) + "_" + tag; + std::string pipe_name = "\\\\.\\pipe\\flutter_cef_probe_" + suffix; g_pipe = CreateNamedPipeA( pipe_name.c_str(), PIPE_ACCESS_DUPLEX, PIPE_TYPE_BYTE | PIPE_READMODE_BYTE | PIPE_WAIT, 1, 1 << 20, 1 << 20, 0, @@ -152,35 +256,130 @@ int main(int argc, char** argv) { if (g_pipe == INVALID_HANDLE_VALUE) Fail("CreateNamedPipe failed"); Say("pipe server up: %s", pipe_name.c_str()); - // 2. Spawn cef_host.exe --ipc=... --profile-dir=... --ephemeral. char tmp[MAX_PATH] = {}; GetTempPathA(MAX_PATH, tmp); - std::string profile = std::string(tmp) + "flutter_cef_probe_prof_" + - std::to_string(GetCurrentProcessId()); - std::string exe = host_dir + "\\cef_host.exe"; + const std::string profile = + std::string(tmp) + "flutter_cef_probe_prof_" + suffix; + + // The host's output pipe. Only its write end is inherited, and only by the + // host. + SECURITY_ATTRIBUTES sa = {sizeof(sa), nullptr, TRUE}; + HANDLE out_read = nullptr, out_write = nullptr; + if (!CreatePipe(&out_read, &out_write, &sa, 0) || + !SetHandleInformation(out_read, HANDLE_FLAG_INHERIT, 0)) + Fail("CreatePipe for the host's output failed"); + SIZE_T attr_size = 0; + InitializeProcThreadAttributeList(nullptr, 1, 0, &attr_size); + std::vector attr_buf(attr_size); + auto* attrs = + reinterpret_cast(attr_buf.data()); + if (!InitializeProcThreadAttributeList(attrs, 1, 0, &attr_size) || + !UpdateProcThreadAttribute(attrs, 0, PROC_THREAD_ATTRIBUTE_HANDLE_LIST, + &out_write, sizeof(out_write), nullptr, + nullptr)) + Fail("ProcThreadAttributeList failed"); + STARTUPINFOEXA six = {}; + six.StartupInfo.cb = sizeof(six); + six.StartupInfo.dwFlags = STARTF_USESTDHANDLES; + six.StartupInfo.hStdInput = nullptr; + six.StartupInfo.hStdOutput = out_write; + six.StartupInfo.hStdError = out_write; + six.lpAttributeList = attrs; + + const std::string exe = host_dir + "\\cef_host.exe"; std::string cmd = "\"" + exe + "\" --ipc=" + pipe_name + " --profile-dir=" + profile + " --ephemeral"; - // Plain spawn, no handle inheritance needed. (The one transport gotcha - // lives host-side: the pipe must be used with OVERLAPPED I/O — a - // synchronous handle serializes read/write on the file object, which - // deadlocked SendFrame against the host's blocking reader and froze the - // CEF UI thread; see cef_host_win.cc OverlappedIo.) - STARTUPINFOA si = {}; - si.cb = sizeof(si); + // (The one transport gotcha lives host-side: the pipe must be used with + // OVERLAPPED I/O — a synchronous handle serializes read/write on the file + // object, which deadlocked SendFrame against the host's blocking reader and + // froze the CEF UI thread; see cef_host_win.cc OverlappedIo.) PROCESS_INFORMATION pi = {}; - if (!CreateProcessA(exe.c_str(), &cmd[0], nullptr, nullptr, FALSE, 0, - nullptr, host_dir.c_str(), &si, &pi)) - Fail("CreateProcess(cef_host.exe) failed"); - CloseHandle(pi.hThread); + const BOOL created = CreateProcessA( + exe.c_str(), &cmd[0], nullptr, nullptr, TRUE, + EXTENDED_STARTUPINFO_PRESENT, nullptr, host_dir.c_str(), + &six.StartupInfo, &pi); + DeleteProcThreadAttributeList(attrs); + CloseHandle(out_write); + if (!created) Fail("CreateProcess(cef_host.exe) failed"); Say("spawned cef_host.exe pid=%lu", pi.dwProcessId); - // 3. Wait for the host to connect. + { + std::lock_guard lock(g_host_out_mutex); + g_host_out.clear(); + } + g_host_out_done = CreateEventA(nullptr, TRUE, FALSE, nullptr); + std::thread([out_read, done = g_host_out_done] { + char buf[4096]; + DWORD n = 0; + while (ReadFile(out_read, buf, sizeof(buf), &n, nullptr) && n > 0) { + std::lock_guard lock(g_host_out_mutex); + g_host_out.append(buf, n); + } + CloseHandle(out_read); + SetEvent(done); + }).detach(); + if (!ConnectNamedPipe(g_pipe, nullptr) && GetLastError() != ERROR_PIPE_CONNECTED) Fail("ConnectNamedPipe failed"); Say("host connected to pipe"); + return Host{pi.hProcess, pi.hThread}; +} + +// After the host has exited: waits (briefly) for the last of its output, then +// prints it. +void FinishHostOutput() { + if (WaitForSingleObject(g_host_out_done, 5000) != WAIT_OBJECT_0) + Say("the host's output pipe is still open 5s after it exited"); + PrintHostOutput(); +} + +void SendCreate(uint32_t wire, const std::string& url) { + // kOpCreateBrowser: {u32 w}{u32 h}{f64 dpr}{utf8 url}. + std::vector p(16 + url.size()); + WriteU32BE(p.data(), 1024); + WriteU32BE(p.data() + 4, 768); + WriteF64BE(p.data() + 8, 1.0); + memcpy(p.data() + 16, url.data(), url.size()); + SendFrame(wire, kOpCreateBrowser, p.data(), static_cast(p.size())); + Say("-> kOpCreateBrowser wire=%u 1024x768@1.0 url=%s", wire, url.c_str()); +} + +void CheckReady(const Frame& f) { + if (f.payload.size() < 2) Fail("kOpReady payload too short"); + uint8_t flags = f.payload[0], ver = f.payload[1]; + Say("<- kOpReady flags=%u protocolVersion=%u", flags, ver); + if (ver != kCefHostProtocolVersion) Fail("protocol version mismatch"); +} - // ---- Event pump / assertions ---- +// Reads frames until the host closes the pipe, so its writes never block. +void DrainToEof() { + Frame f; + while (ReadFrame(&f)) { + if (f.op == kOpLog) Say("<- kOpLog %s", PayloadStr(f).c_str()); + } + Say("pipe EOF"); + CloseHandle(g_pipe); + g_pipe = INVALID_HANDLE_VALUE; +} + +// Waits up to `timeout_ms` for the host to exit; its exit code. +DWORD AwaitExit(const Host& host, DWORD timeout_ms, const char* what) { + if (WaitForSingleObject(host.process, timeout_ms) != WAIT_OBJECT_0) { + TerminateProcess(host.process, 9); + Fail(what); + } + DWORD exit_code = 0; + GetExitCodeProcess(host.process, &exit_code); + return exit_code; +} + +constexpr char kHardExitLine[] = "still running after shutdown"; + +// The gate: create, first frame, a navigation, a clean shutdown. +void RunGate(const std::string& host_dir, const std::string& url1, + const std::string& url2) { + Host host = SpawnHost(host_dir, "gate"); const uint32_t kWire = 1; bool got_ready = false, got_created = false, got_present = false; bool page1_settled = false, sent_navigate = false; @@ -192,25 +391,12 @@ int main(int argc, char** argv) { Frame f; while (ReadFrame(&f)) { switch (f.op) { - case kOpReady: { - if (f.payload.size() < 2) Fail("kOpReady payload too short"); - uint8_t flags = f.payload[0], ver = f.payload[1]; - Say("<- kOpReady flags=%u protocolVersion=%u", flags, ver); - if (ver != kCefHostProtocolVersion) Fail("protocol version mismatch"); + case kOpReady: + CheckReady(f); got_ready = true; - // kOpCreateBrowser wire=1: {u32 w}{u32 h}{f64 dpr}{utf8 url}. - std::vector p(16 + url1.size()); - WriteU32BE(p.data(), 1024); - WriteU32BE(p.data() + 4, 768); - WriteF64BE(p.data() + 8, 1.0); - memcpy(p.data() + 16, url1.data(), url1.size()); - SendFrame(kWire, kOpCreateBrowser, p.data(), - static_cast(p.size())); + SendCreate(kWire, url1); t_create = Now(); - Say("-> kOpCreateBrowser wire=%u 1024x768@1.0 url=%s", kWire, - url1.c_str()); break; - } case kOpCreated: Say("<- kOpCreated wire=%u", f.wire_id); got_created = true; @@ -266,8 +452,7 @@ int main(int argc, char** argv) { case kOpUrl: { std::string url = PayloadStr(f); Say("<- kOpUrl %s", url.c_str()); - if (sent_navigate && url.find(url2.substr(0, url2.find('/', 8))) != - std::string::npos) + if (sent_navigate && url.compare(0, url2.size(), url2) == 0) nav_url_seen = true; break; } @@ -320,22 +505,103 @@ int main(int argc, char** argv) { presents); SendFrame(0, kOpShutdown, nullptr, 0); Say("-> kOpShutdown"); + DrainToEof(); + // Longer than the host's own teardown allowance (30 s), so a slow + // CefShutdown is reported as the watchdog's exit, not as a hang here. + const DWORD exit_code = + AwaitExit(host, 40000, "host did not exit within 40s of shutdown"); + Say("host exited code=%lu", exit_code); + FinishHostOutput(); + if (exit_code != 0) Fail("host exit code != 0"); + if (HostOutput().find(kHardExitLine) != std::string::npos) + Fail("the host's hard-exit watchdog ended it: shutdown didn't finish"); + CloseHandle(host.main_thread); + CloseHandle(host.process); +} - // Drain until EOF so the host's writes never block, then await exit. - while (ReadFrame(&f)) { - if (f.op == kOpLog) Say("<- kOpLog %s", PayloadStr(f).c_str()); - } - Say("pipe EOF"); - DWORD wait = WaitForSingleObject(pi.hProcess, 20000); - if (wait != WAIT_OBJECT_0) { - TerminateProcess(pi.hProcess, 9); - Fail("host did not exit within 20s of shutdown"); +// A host whose UI thread is wedged when kOpShutdown arrives exits on its own. +void RunWedgedShutdown(const std::string& host_dir, const std::string& url) { + Say("== a host whose UI thread is wedged at shutdown"); + Host host = SpawnHost(host_dir, "wedge"); + const uint32_t kWire = 1; + bool presented = false, settled = false; + Frame f; + while (!(presented && settled)) { + if (!ReadFrame(&f)) Fail("pipe closed before the page loaded"); + switch (f.op) { + case kOpReady: + CheckReady(f); + SendCreate(kWire, url); + break; + case kOpCreateFailed: + Fail("kOpCreateFailed"); + break; + case kOpPresent: + presented = true; + break; + case kOpLoadState: + if (f.payload.size() >= 1 && f.payload[0] == 0) settled = true; + break; + case kOpLog: + Say("<- kOpLog(wire=%u) %s", f.wire_id, PayloadStr(f).c_str()); + break; + default: + break; + } } - DWORD exit_code = 0; - GetExitCodeProcess(pi.hProcess, &exit_code); - CloseHandle(pi.hProcess); - Say("host exited code=%lu", exit_code); + Say("page loaded and painted"); + // Let the load's last tasks run, so the UI thread is suspended idle rather + // than inside something the watchdog thread might need. + Sleep(500); + if (SuspendThread(host.main_thread) == static_cast(-1)) + Fail("SuspendThread failed"); + Say("suspended the host's UI thread"); + SendFrame(0, kOpShutdown, nullptr, 0); + const double t_shutdown = Now(); + Say("-> kOpShutdown"); + DrainToEof(); + const DWORD exit_code = AwaitExit( + host, 20000, "a host with a wedged UI thread did not exit within 20s"); + const double took = Now() - t_shutdown; + Say("host exited code=%lu, %.3fs after kOpShutdown", exit_code, took); + FinishHostOutput(); + const std::string want = + std::string("[cef_host] ") + kHardExitLine + " (kOpShutdown); exiting now"; + if (HostOutput().find(want) == std::string::npos) + Fail("no hard-exit line from the host"); if (exit_code != 0) Fail("host exit code != 0"); + // 6 s after the host read kOpShutdown, give or take the clocks' ticks. + if (took < 5.0 || took > 15.0) Fail("the hard exit wasn't about 6s late"); + CloseHandle(host.main_thread); + CloseHandle(host.process); +} + +} // namespace + +int main(int argc, char** argv) { + g_t0 = GetTickCount64(); + if (argc < 2) { + fprintf(stderr, + "usage: pipe_probe.exe [url1] [url2]\n"); + return 2; + } + std::string host_dir = argv[1]; + + // Global watchdog: nothing in this probe may take 3 minutes. + std::thread([] { + Sleep(180000); + printf("PIPE_PROBE FAIL: global watchdog (180s)\n"); + ExitProcess(3); + }).detach(); + + const std::string origin = + "http://127.0.0.1:" + std::to_string(StartPageServer()); + Say("serving the probe's pages at %s", origin.c_str()); + const std::string url1 = argc > 2 ? argv[2] : origin + "/one"; + const std::string url2 = argc > 3 ? argv[3] : origin + "/two"; + + RunGate(host_dir, url1, url2); + RunWedgedShutdown(host_dir, url1); Say("PIPE_PROBE PASS"); return 0; } diff --git a/packages/flutter_cef_windows/test/native/policy_test.cc b/packages/flutter_cef_windows/test/native/policy_test.cc index 1141bfa..57f3495 100644 --- a/packages/flutter_cef_windows/test/native/policy_test.cc +++ b/packages/flutter_cef_windows/test/native/policy_test.cc @@ -128,6 +128,48 @@ void TestCrashLoopBurstsAgeOut() { CHECK(f.OnRendererTerminated(1, At(t0, 3)) == Crash::kReload); } +// The hard-exit watchdog's wait against a fake clock. `extend_at` > 0 moves +// the deadline out to extend_at + the teardown allowance during the first +// sleep, as the host does when its message loop quits; `early_ms` wakes the +// first sleep that much early. Returns when the wait ended; counts sleeps. +int64_t HardExitAt(int64_t start, int64_t extend_at, int64_t early_ms, + int* sleeps) { + const auto ms = [](std::chrono::seconds s) { + return static_cast( + std::chrono::duration_cast(s).count()); + }; + int64_t now = start; + int64_t deadline = ms(policy::kHardExitAfterShutdown); + *sleeps = 0; + policy::WaitForDeadline( + [&] { return now; }, [&] { return deadline; }, + [&](int64_t left) { + if (++*sleeps == 1) { + if (extend_at > 0) + deadline = extend_at + ms(policy::kHardExitAfterTeardown); + left -= early_ms; + } + now += left; + }); + return now; +} + +void TestHardExit() { + // The same timings as the macOS host. + CHECK(policy::kHardExitAfterShutdown == std::chrono::seconds(6)); + CHECK(policy::kHardExitAfterTeardown == std::chrono::seconds(30)); + int sleeps = 0; + // Nothing moves the deadline: the process ends 6 s after the request. + CHECK(HardExitAt(0, 0, 0, &sleeps) == 6000 && sleeps == 1); + // The message loop quits 2 s in, while the watchdog sleeps: on waking it + // sees the teardown deadline and waits for that instead. + CHECK(HardExitAt(0, 2000, 0, &sleeps) == 32000 && sleeps == 2); + // A sleep that ends early is followed by one for the rest. + CHECK(HardExitAt(0, 0, 1000, &sleeps) == 6000 && sleeps == 2); + // A deadline already behind: no wait at all. + CHECK(HardExitAt(7000, 0, 0, &sleeps) == 7000 && sleeps == 0); +} + void TestPayloadCaps() { // Under the cap: untouched. CHECK(policy::CapText("hello", 10) == "hello"); @@ -304,6 +346,7 @@ int main() { TestCrashLoopOneBrowser(); TestCrashLoopSeveralBrowsers(); TestCrashLoopBurstsAgeOut(); + TestHardExit(); TestPayloadCaps(); TestSchemes(); TestFrameRate(); diff --git a/packages/flutter_cef_windows/windows/flutter_cef_plugin.cpp b/packages/flutter_cef_windows/windows/flutter_cef_plugin.cpp index d35cd5c..7cd1582 100644 --- a/packages/flutter_cef_windows/windows/flutter_cef_plugin.cpp +++ b/packages/flutter_cef_windows/windows/flutter_cef_plugin.cpp @@ -14,7 +14,6 @@ #include #include #include -#include #include #include #include @@ -1148,6 +1147,16 @@ void FlutterCefPlugin::DisposeSession(const std::string& session_id) { if (!host_key.empty()) DetachFromHost(host_key, browser_id); } +// `session_id` is taken by value: callers pass the session's own id, which +// DisposeSession frees. Disposing the host's last session shuts the host +// down, as any dispose does. As macOS reportBrowserGone. +void FlutterCefPlugin::ReportBrowserGone(std::string session_id, + const std::string& reason) { + EmitEvent("processGone", session_id, + {{Ev("reason"), flutter::EncodableValue(reason)}}); + DisposeSession(session_id); +} + void FlutterCefPlugin::DetachFromHost(const std::string& host_key, uint32_t browser_id) { auto hit = hosts_.find(host_key); @@ -1845,29 +1854,18 @@ void FlutterCefPlugin::HandleSessionFrame( // still goes out after it. if (!s->visible) SendOrQueue(s, kOpSetVisible, {uint8_t{0}}); break; - case kOpCreateFailed: { - // This browser's create failed; the host process is otherwise healthy — - // drop just this ONE session (emit processGone, close its browser, keep - // the host for its siblings, as macOS onBrowserGone does). Copy the id - // first: DisposeSession frees `s`, dangling session_id (an alias of - // s->id). - const std::string sid = session_id; - EmitEvent("processGone", sid, - {{Ev("reason"), flutter::EncodableValue("createFailed")}}); - DisposeSession(sid); + case kOpCreateFailed: + // This browser's create failed; the host process is otherwise healthy, + // so only this session goes. + ReportBrowserGone(session_id, "createFailed"); break; - } case kOpBrowserGone: { // cef_host gave up on this browser (its renderer keeps crashing). The // host and its other browsers are fine: drop just this session, as for - // a failed create. Disposing it shuts the host down when it was the - // host's last browser. - const std::string sid = session_id; + // a failed create. std::string reason = PayloadString(payload); if (reason.empty()) reason = "crashed"; - EmitEvent("processGone", sid, - {{Ev("reason"), flutter::EncodableValue(reason)}}); - DisposeSession(sid); + ReportBrowserGone(session_id, reason); break; } case kOpTargetId: @@ -2112,11 +2110,13 @@ void FlutterCefPlugin::OnWatchdogTimer(UINT_PTR timer_id) { // signal. This sweep (a port of CefProfileHost's) covers the steady state. // Staleness alone isn't a hang — a static page idles without frames — so a // stale tile is nudged with a repaint and its renderer pinged with an eval -// under liveness::kPingId, an id Dart never issues. A healthy page answers -// the ping; a renderer that leaves it unanswered for FLUTTER_CEF_HANG_MS is -// hung, and its host is ended so every tile on it reports processGone -// ("crashed") and recovers. GPU-process replacement, which macOS also checks, -// is not detected here. +// under liveness::kPingId, an id Dart never issues, which cef_host puts to +// the renderer itself rather than the page. A live renderer answers it; one +// that leaves it unanswered for FLUTTER_CEF_HANG_MS is hung, and only its +// tile ends, with processGone("crashed"): its siblings on the host are fine. +// macOS also ends a host whose GPU process was replaced, because its views +// never paint again; on Windows they keep painting (the smoke test's +// chrome://gpucrash case), so there is nothing to watch for. void FlutterCefPlugin::EnsureLivenessTimer() { if (liveness_timer_active_ || !message_window_) return; @@ -2134,7 +2134,8 @@ void FlutterCefPlugin::OnLivenessTimer() { const uint64_t stale_ms = LivenessStalenessMs(); const uint64_t hang_ms = LivenessHangMs(); const auto ns = [](uint64_t ms) { return ms * 1000000ull; }; - std::set hung_hosts; + // Ended after the loop: ReportBrowserGone erases sessions. + std::vector hung; for (auto& kv : sessions_) { Session* s = kv.second.get(); // Frozen tiles have no browser, and the first-present watchdog owns tiles @@ -2158,7 +2159,7 @@ void FlutterCefPlugin::OnLivenessTimer() { // renderer. if (liveness::Ping(ns(now), ns(s->ping_sent_ms), ns(s->ping_replied_ms), ns(stale_ms), ns(hang_ms)) == liveness::PingAction::kHung) { - hung_hosts.insert(s->host_key); + hung.push_back(s->id); continue; } const bool nudged = s->nudged_at_ms != 0; @@ -2186,14 +2187,14 @@ void FlutterCefPlugin::OnLivenessTimer() { SendOrQueue(s, kOpEvalReturning, std::move(p)); } } - // Outside the loop: FailHost erases sessions. - for (const auto& key : hung_hosts) { - auto it = hosts_.find(key); - if (it == hosts_.end()) continue; - Log("host '" + key + "': a renderer left a liveness ping unanswered for " + - std::to_string(hang_ms / 1000) + "s — ending the host"); - if (it->second->process) it->second->process->Terminate(); - FailHost(key, "crashed"); + for (const auto& sid : hung) { + auto it = sessions_.find(sid); + if (it == sessions_.end()) continue; + Log("host '" + it->second->host_key + "': browser " + + std::to_string(it->second->browser_id) + + "'s renderer left the liveness ping unanswered for " + + std::to_string(hang_ms / 1000) + "s — reporting it gone"); + ReportBrowserGone(sid, "crashed"); } } diff --git a/packages/flutter_cef_windows/windows/flutter_cef_plugin.h b/packages/flutter_cef_windows/windows/flutter_cef_plugin.h index d38013a..f8fd809 100644 --- a/packages/flutter_cef_windows/windows/flutter_cef_plugin.h +++ b/packages/flutter_cef_windows/windows/flutter_cef_plugin.h @@ -257,6 +257,10 @@ class FlutterCefPlugin : public flutter::Plugin { const std::string& allowed_schemes, bool agent_control); void DisposeSession(const std::string& session_id); + // One session can't go on while its host is otherwise fine (its create + // failed, its renderer kept crashing or hung): emit processGone(`reason`) + // for it and dispose it. The host carries on for its other sessions. + void ReportBrowserGone(std::string session_id, const std::string& reason); // freezeSession / thawSession (see the .cpp). void FreezeSession( const flutter::EncodableMap& args, diff --git a/packages/flutter_cef_windows/windows/liveness_policy.h b/packages/flutter_cef_windows/windows/liveness_policy.h index 9ed83ad..3e78350 100644 --- a/packages/flutter_cef_windows/windows/liveness_policy.h +++ b/packages/flutter_cef_windows/windows/liveness_policy.h @@ -8,9 +8,10 @@ // // A static page legitimately presents nothing while idle, so staleness alone // is not a hang. The sweep first nudges a stale tile (kOpInvalidate) and pings -// its renderer with an eval the page can't see. A healthy page answers the -// ping; a renderer that leaves it unanswered for `hang_ns` is hung, and the -// plugin ends the host so the tiles are recreated. +// its renderer: an eval id that cef_host answers from the renderer's main +// thread, not the page. A live renderer answers the ping; one that leaves it +// unanswered for `hang_ns` is hung, and the plugin ends that tile alone with +// processGone("crashed"). #ifndef FLUTTER_PLUGIN_FLUTTER_CEF_LIVENESS_POLICY_H_ #define FLUTTER_PLUGIN_FLUTTER_CEF_LIVENESS_POLICY_H_